utils_v2 Namespace Reference

Classes
class	Chi2Test
class	ComparisonError
class	KolmogorovTest
class	StatisticalTest
	StatisticalTests. More...

Functions
def	auth_download_file (url, chunk_size=1048576)
def	auth_wget (url)
	-----------------— Recursife file downloader --------------------— More...
def	check_disk_for_space (work_path, size_needed)
def	get_relval_cmssw_version (file)
def	get_relval_id (file)
def	get_relval_max_version (files)
def	get_relval_version (file)
	----------------— Make files pairs: RelVal utils ------------------— More...
def	get_relvaldata_cmssw_version (file)
def	get_relvaldata_id (file)
	--------------— Make files pairs: RelValData utils -----------------— More...
def	get_relvaldata_max_version (files)
def	get_relvaldata_version (file)
def	get_size_to_download (work_path, files_with_urls)
def	get_version (filename)
def	init_database (db_path)
	Utils. More...
def	is_relvaldata (files)
	--------------------— Make file pairs -----------------------— More...
def	make_file_pairs (files1, files2)
def	recursive_search_online (url, rel1, frags1, rel2, frags2)
def	search_on_disk (work_path, rel1, frags1, rel2, frags2)
def	show_status_bar (total_size)

Variables
	comparison_errors
	Exception definitions. More...
	tests

Detailed Description

Help functions for ValidationMatrix_v2.py.

Author:  Albertas Gimbutas,  Vilnius University (LT)
e-mail:  albertasgim@gmail.com

Function Documentation

auth_download_file()

def utils_v2.auth_download_file	(		url,
			chunk_size = 1048576
	)

Definition at line 200 of file utils_v2.py.

def auth_download_file(url, chunk_size=1048576):
    filename = basename(url)
    file_path = join(auth_download_file.work_dir, filename)
 
    file = open(file_path, 'wb')
    opener = build_opener(X509CertOpen())
    url_file = opener.open(Request(url))
    chunk = url_file.read(chunk_size)
    while chunk:
        file.write(chunk)
        auth_download_file.q.put((1,))   # reports, that downloaded 1MB
        chunk = url_file.read(chunk_size)
    print('\rDownloaded: %s  ' % (filename,))
    file.close()
 
 

References join(), and edm.print().

auth_wget()

def utils_v2.auth_wget

(

url

)

-----------------— Recursife file downloader --------------------—

Definition at line 186 of file utils_v2.py.

def auth_wget(url):
    try:
        opener = build_opener(X509CertOpen())
        return opener.open(Request(url)).read()
    except HTTPError as e:
        print('\nError: DQM GUI is temporarily unavailable. Probably maintainance hours. '+\
                'Please try again later. Original error message: ``%s``. \nExiting...\n' % (e,))
        exit()
    except BadStatusLine as e:
        print('\nYou do not have permissions to access DQM GUI. Please check if your certificates '+\
            'in ``~/.globus`` directory are configured correctly. Exitting...') 
        exit()
 
 

References beamvalidation.exit(), edm.print(), and readEcalDQMStatus.read.

Referenced by recursive_search_online().

check_disk_for_space()

def utils_v2.check_disk_for_space	(		work_path,
			size_needed
	)

Checks afs file system for space.

Definition at line 462 of file utils_v2.py.

def check_disk_for_space(work_path, size_needed):
    '''Checks afs file system for space.'''
    pass
    # try:
    #     fs_proc = subprocess.Popen(['fs', 'listquota', work_path], stdout=subprocess.PIPE)
    # except OSError:
    #     return
    # fs_response = fs_proc.communicate()[0]
    # quota, used = re.findall('([\d]+)', fs_response)[:2]
    # free_space = int(quota) - int(used)
    # if free_space * 1024 < size_needed:
    #     print '\nNot enougth free space on disk.',
    #     print 'Free space: %d MB. Need: %d MB. Exiting...\n' % (free_space / 1024, size_needed /1048576)
    #     exit()
    # elif size_needed:
    #     print 'Free space on disk: %d MB.\n' % (free_space / 1024,)
 
 

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

get_relval_cmssw_version()

def utils_v2.get_relval_cmssw_version

(

file

)

Definition at line 92 of file utils_v2.py.

def get_relval_cmssw_version(file):
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
    if cmssw_release and gr_r_version:
        return (cmssw_release[0], gr_r_version[0])
 

Referenced by get_version().

get_relval_id()

def utils_v2.get_relval_id

(

file

)

Returns unique relval ID (dataset name) for a given file.

Definition at line 98 of file utils_v2.py.

def get_relval_id(file):
    """Returns unique relval ID (dataset name) for a given file."""
    dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
    return dataset_name[0]
 

get_relval_max_version()

def utils_v2.get_relval_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 80 of file utils_v2.py.

def get_relval_max_version(files):
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relval_version(files[0])
    for file in files:
        file_v = get_relval_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file
 

References get_relval_version().

get_relval_version()

def utils_v2.get_relval_version

(

file

)

----------------— Make files pairs: RelVal utils ------------------—

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 73 of file utils_v2.py.

def get_relval_version(file):
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))
 

References createfilelist.int.

Referenced by get_relval_max_version().

get_relvaldata_cmssw_version()

def utils_v2.get_relvaldata_cmssw_version

(

file

)

Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 42 of file utils_v2.py.

def get_relvaldata_cmssw_version(file):
    """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
    cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
    gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
    if not gr_r_version:
        gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
    if cmssw_release and gr_r_version:
        return (cmssw_release[0], gr_r_version[0])
 

Referenced by get_version().

get_relvaldata_id()

def utils_v2.get_relvaldata_id

(

file

)

--------------— Make files pairs: RelValData utils -----------------—

Returns unique relvaldata ID for a given file.

Definition at line 32 of file utils_v2.py.

def get_relvaldata_id(file):
    """Returns unique relvaldata ID for a given file."""
    run_id = re.search('R\d{9}', file)
    run = re.search('_RelVal_([\w\d]*)-v\d__', file)
    if not run:
        run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
    if run_id and run:
        return (run_id.group(), run.group(1))
    return None
 

get_relvaldata_max_version()

def utils_v2.get_relvaldata_max_version

(

files

)

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 60 of file utils_v2.py.

def get_relvaldata_max_version(files):
    """Returns file with maximum version at a) beggining of the file,
    e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
    max_file = files[0]
    max_v = get_relvaldata_version(files[0])
    for file in files:
        file_v = get_relvaldata_version(file)
        if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
            max_file = file
            max_v = file_v
    return max_file
 

References get_relvaldata_version().

get_relvaldata_version()

def utils_v2.get_relvaldata_version

(

file

)

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 51 of file utils_v2.py.

def get_relvaldata_version(file):
    """Returns tuple (CMSSW version, run version) for specified file."""
    cmssw_version = re.findall('DQM_V(\d*)_', file)
    run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
    if not run_version:
        run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
    if cmssw_version and run_version:
        return (int(cmssw_version[0]), int(run_version[0]))
 

References createfilelist.int.

Referenced by get_relvaldata_max_version().

get_size_to_download()

def utils_v2.get_size_to_download	(		work_path,
			files_with_urls
	)

Returns file list to download and total size to download.

Definition at line 446 of file utils_v2.py.

def get_size_to_download(work_path, files_with_urls):
    """Returns file list to download and total size to download."""
    opener = build_opener(X509CertOpen())
    size_to_download = 0
    files_to_download = []
    for filename, url in files_with_urls:
        url_file = opener.open(Request(url))
        size = int(url_file.headers["Content-Length"])
        file_path = join(work_path, filename)
        if exists(file_path) and getsize(file_path) / 1024 == size / 1024:
            print("Exists on disk %s." % filename)
        else:
            size_to_download += size
            files_to_download.append(url)
    return size_to_download, files_to_download
 

References createfilelist.int, join(), and edm.print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

get_version()

def utils_v2.get_version

(

filename

)

Returns CMSSW and GR_R versions for the given filename.

Definition at line 435 of file utils_v2.py.

def get_version(filename):
    """Returns CMSSW and GR_R versions for the given filename."""
    if is_relvaldata([filename]):
        version_elems = get_relvaldata_cmssw_version(filename)
    else:
        relval_version = get_relval_cmssw_version(filename)
        version_elems = (relval_version[0], relval_version[1][0], relval_version[1][1])
    version_elems = [elem.strip('_').strip('RelVal_') for elem in version_elems]
    return '___'.join([elem for elem in version_elems if elem])
 
 

References get_relval_cmssw_version(), get_relvaldata_cmssw_version(), is_relvaldata(), join(), and digitizers_cfi.strip.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

init_database()

def utils_v2.init_database

(

db_path

)

Utils.

Definition at line 391 of file utils_v2.py.

def init_database(db_path):
    print('Initialising DB: %s...' % basename(db_path), end=' ')
    conn = sqlite3.connect(db_path)
 
    
    c = conn.cursor()
    c.execute("""CREATE TABLE IF NOT EXISTS ReleaseComparison (
                        id INTEGER PRIMARY KEY,
                        title TEXT,
                        release1 TEXT,
                        release2 TEXT,
                        statistical_test TEXT
                    );""")
    c.execute("""CREATE TABLE IF NOT EXISTS Directory (
                        id INTEGER PRIMARY KEY,
                        name TEXT,
                        parent_id INTEGER,
                        from_histogram_id INTEGER,
                        till_histogram_id INTEGER,
                        FOREIGN KEY (parent_id) REFERENCES Directory(id)
                        FOREIGN KEY (from_histogram_id) REFERENCES HistogramComparison(id)
                        FOREIGN KEY (till_histogram_id) REFERENCES HistogramComparison(id)
                    )""")
    c.execute("""CREATE TABLE IF NOT EXISTS RootFileComparison (
                        id INTEGER PRIMARY KEY,
                        filename1 TEXT,
                        filename2 TEXT,
                        release_comparison_id INTEGER,
                        directory_id INTEGER,
                        FOREIGN KEY (release_comparison_id) REFERENCES ReleaseComparison(id),
                        FOREIGN KEY (directory_id) REFERENCES Directory(id)
                    )""")
    c.execute("""CREATE TABLE IF NOT EXISTS HistogramComparison (
                        id INTEGER PRIMARY KEY,
                        name TEXT,
                        p_value REAL,
                        directory_id INTEGER,
                        FOREIGN KEY (directory_id) REFERENCES Directory(id)
                    )""")
 
    print('Done.')
    return db_path
 
 

References edm.print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

is_relvaldata()

def utils_v2.is_relvaldata

(

files

)

--------------------— Make file pairs -----------------------—

Definition at line 104 of file utils_v2.py.

def is_relvaldata(files):
    is_relvaldata_re = re.compile('_RelVal_')
    return any([is_relvaldata_re.search(filename) for filename in files])
 

References any().

Referenced by get_version(), and make_file_pairs().

make_file_pairs()

def utils_v2.make_file_pairs	(		files1,
			files2
	)

Definition at line 108 of file utils_v2.py.

def make_file_pairs(files1, files2):
    print('\n#################       Analyzing files       ###################')
    
    if is_relvaldata(files1):
        is_relval_data = True
        get_cmssw_version = get_relvaldata_cmssw_version
        get_id = get_relvaldata_id
        get_max_version = get_relvaldata_max_version
    else:
        is_relval_data = False
        get_cmssw_version = get_relval_cmssw_version
        get_id = get_relval_id
        get_max_version = get_relval_max_version
 
    
    versions1, versions2 = dict(), dict() # {version1: [file1, file2, ...], version2: [...], ...}
    for files, versions in (files1, versions1), (files2, versions2):
        for file in files:
            version = get_cmssw_version(file)
            if version:
                if version in versions:
                    versions[version].append(file)
                else:
                    versions[version] = [file]
 
    
    print('For RELEASE1 found file groups:')
    for version in versions1:
        print('   %s: %d files' % (str(version),  len(versions1[version])))
    if not versions1:
        print('None.')
 
    print('\nFor RELEASE2 found file groups:')
    for version in versions2:
        print('   %s: %d files' % (str(version),  len(versions2[version])))
    if not versions2:
        print('None.')
 
    if not len(versions1) or not len(versions2):
        print('\nNot enough file groups. Exiting...\n')
        exit()
 
    
    pairs = []
    for v1 in sorted(versions1, key=lambda x: len(versions1[x]), reverse=True):
        for v2 in sorted(versions2, key=lambda x: len(versions2[x]), reverse=True):
            if v1 == v2:
                continue
            
            print('\n#################     Pairing the files     ###################')
            print('%s (%d files)   VS   %s (%d files):\n' % (str(v1),
                    len(versions1[v1]), str(v2), len(versions2[v2])))
 
            
            for unique_id in set([get_id(file) for file in versions1[v1]]):
                if is_relval_data:
                    dataset_re = re.compile(unique_id[0] + '_')
                    run_re = re.compile(unique_id[1])
                    c1_files = [file for file in versions1[v1] if dataset_re.search(file) and run_re.search(file)]
                    c2_files = [file for file in versions2[v2] if dataset_re.search(file) and run_re.search(file)]
                else:
                    dataset_re = re.compile(unique_id + '_')
                    c1_files = [file for file in versions1[v1] if dataset_re.search(file)]
                    c2_files = [file for file in versions2[v2] if dataset_re.search(file)]
 
                if len(c1_files) > 0 and len(c2_files) > 0:
                    first_file = get_max_version(c1_files)
                    second_file = get_max_version(c2_files)
                    print('%s\n%s\n' % (first_file, second_file))
                    pairs.append((first_file, second_file))
 
            print("Got %d pairs." % (len(pairs)))
            if pairs:
                return pairs
    print('Found no file pairs. Exiting..\n')
    exit()
 

References mps_setup.append, beamvalidation.exit(), is_relvaldata(), edm.print(), and str.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

recursive_search_online()

def utils_v2.recursive_search_online	(		url,
			rel1,
			frags1,
			rel2,
			frags2
	)

Recursively searches for files, that matches the pattern.

Definition at line 216 of file utils_v2.py.

def recursive_search_online(url, rel1, frags1, rel2, frags2):
    """Recursively searches for files, that matches the pattern."""
    if not url:
        url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelValData/'
        g1, g2 = recursive_search_online(url, rel1, frags1, rel2, frags2)
        url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/'
        g3, g4 = recursive_search_online(url, rel1, frags1, rel2, frags2)
        g1.update(g3), g2.update(g4)
        return g1, g2
 
    domain = '://'.join(urlparse(url)[:2])
 
    
    href_re = re.compile(r"<a href='([-./\w]*)'>([-./\w]*)<")
 
    def compile_res(rel, frags):
        frags = frags.split(',')
        regexps = [s for s in frags if not s.startswith('!')]
        regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
        regexps += [rel + '-', '.root']
        return [re.compile(r) for r in regexps]
 
    res1 = compile_res(rel1, frags1)
    res2 = compile_res(rel2, frags2)
 
    
    hrefs = [(name, path) for path, name in href_re.findall(auth_wget(url))[1:]]
    files_with_urls1, files_with_urls2 = dict(), dict()
    for name, path in hrefs:
        if splitext(name)[1]: # If file
            if all([r.search(name) for r in res1]):
                files_with_urls1[name] = domain + path
            if all([r.search(name) for r in res2]):
                files_with_urls2[name] = domain + path
        else:
            print(domain + path)
            new_hrefs = href_re.findall(auth_wget(domain + path))[1:]
            hrefs.extend([(name, path) for path, name in new_hrefs])
    return files_with_urls1, files_with_urls2
 

References python.cmstools.all(), auth_wget(), join(), and edm.print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

search_on_disk()

def utils_v2.search_on_disk	(		work_path,
			rel1,
			frags1,
			rel2,
			frags2
	)

Definition at line 256 of file utils_v2.py.

def search_on_disk(work_path, rel1, frags1, rel2, frags2):
    if not work_path:
        print('No working directory specified. Use "--dir DIR" option to ' +\
              'specify working directory. Exiting...')
        exit()
    
    def compile_res(rel, frags):
        frags = frags.split(',')
        regexps = [s for s in frags if not s.startswith('!')]
        regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
        regexps += [rel + '-', '.root']
        return [re.compile(r) for r in regexps]
 
    res1 = compile_res(rel1, frags1)
    res2 = compile_res(rel2, frags2)
 
    
    files = listdir(work_path)
    files1, files2 = [], []
    for name in files:
        if splitext(name)[1]:
            if all([r.search(name) for r in res1]):
                files1.append(name)
            if all([r.search(name) for r in res2]):
                files2.append(name)
    return files1, files2
 
 

References python.cmstools.all(), beamvalidation.exit(), and edm.print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

show_status_bar()

def utils_v2.show_status_bar

(

total_size

)

Shows download status.

Definition at line 480 of file utils_v2.py.

def show_status_bar(total_size):
    """Shows download status."""
    q = show_status_bar.q
    total_size = total_size / (1024*1024)
    downloaded = 0
    while downloaded < total_size:
        try:
            o = q.get(timeout=20)
            downloaded += 1
            print('\r      %d/%d MB     %d%%     ' % (downloaded, total_size, 100*downloaded/total_size), end=' ')
            sys.stdout.flush()
        except Empty:
            time.sleep(1)
            break

Variable Documentation

comparison_errors

utils_v2.comparison_errors

Exception definitions.

Definition at line 285 of file utils_v2.py.

tests

utils_v2.tests

Definition at line 388 of file utils_v2.py.