Classes
class	Chi2Test

class	ComparisonError

class	KolmogorovTest

class	StatisticalTest
	StatisticalTests. More...

Functions
def	auth_download_file (url, chunk_size=1048576)

def	auth_wget (url)
	-----------------— Recursife file downloader --------------------— More...

def	check_disk_for_space (work_path, size_needed)

def	get_relval_cmssw_version (file)

def	get_relval_id (file)

def	get_relval_max_version (files)

def	get_relval_version (file)
	-------------—— Make files pairs: RelVal utils ---------------—— More...

def	get_relvaldata_cmssw_version (file)

def	get_relvaldata_id (file)
	-----------—— Make files pairs: RelValData utils --------------—— More...

def	get_relvaldata_max_version (files)

def	get_relvaldata_version (file)

def	get_size_to_download (work_path, files_with_urls)

def	get_version (filename)

def	init_database (db_path)
	Utils. More...

def	is_relvaldata (files)
	--------------------— Make file pairs -----------------------— More...

def	make_file_pairs (files1, files2)

def	recursive_search_online (url, rel1, frags1, rel2, frags2)

def	search_on_disk (work_path, rel1, frags1, rel2, frags2)

def	show_status_bar (total_size)

Variables
	comparison_errors
	Exception definitions. More...

	tests

Detailed Description

Help functions for ValidationMatrix_v2.py.

Author:  Albertas Gimbutas,  Vilnius University (LT)
e-mail:  albertasgim@gmail.com

Function Documentation

◆ auth_download_file()

def utils_v2.auth_download_file	(	url,
		chunk_size = `1048576`
	)

Definition at line 200 of file utils_v2.py.

References join(), and print().

 def auth_download_file(url, chunk_size=1048576):
     filename = basename(url)
     file_path = join(auth_download_file.work_dir, filename)
 
     file = open(file_path, 'wb')
     opener = build_opener(X509CertOpen())
     url_file = opener.open(Request(url))
     chunk = url_file.read(chunk_size)
     while chunk:
         file.write(chunk)
         auth_download_file.q.put((1,))   # reports, that downloaded 1MB
         chunk = url_file.read(chunk_size)
     print('\rDownloaded: %s  ' % (filename,))
     file.close()
 
 

◆ auth_wget()

def utils_v2.auth_wget ( url )

-----------------— Recursife file downloader --------------------—

Definition at line 186 of file utils_v2.py.

References beamvalidation.exit(), print(), and fileinputsource_cfi.read.

Referenced by recursive_search_online().

 def auth_wget(url):
     try:
         opener = build_opener(X509CertOpen())
         return opener.open(Request(url)).read()
     except HTTPError as e:
         print('\nError: DQM GUI is temporarily unavailable. Probably maintainance hours. '+\
                 'Please try again later. Original error message: ``%s``. \nExiting...\n' % (e,))
         exit()
     except BadStatusLine as e:
         print('\nYou do not have permissions to access DQM GUI. Please check if your certificates '+\
             'in ``~/.globus`` directory are configured correctly. Exitting...') 
         exit()
 
 

◆ check_disk_for_space()

def utils_v2.check_disk_for_space	(	work_path,
		size_needed
	)

Checks afs file system for space.

Definition at line 462 of file utils_v2.py.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def check_disk_for_space(work_path, size_needed):
     '''Checks afs file system for space.'''
     pass
     # try:
     #     fs_proc = subprocess.Popen(['fs', 'listquota', work_path], stdout=subprocess.PIPE)
     # except OSError:
     #     return
     # fs_response = fs_proc.communicate()[0]
     # quota, used = re.findall('([\d]+)', fs_response)[:2]
     # free_space = int(quota) - int(used)
     # if free_space * 1024 < size_needed:
     #     print '\nNot enougth free space on disk.',
     #     print 'Free space: %d MB. Need: %d MB. Exiting...\n' % (free_space / 1024, size_needed /1048576)
     #     exit()
     # elif size_needed:
     #     print 'Free space on disk: %d MB.\n' % (free_space / 1024,)
 
 

◆ get_relval_cmssw_version()

def utils_v2.get_relval_cmssw_version ( file )

Definition at line 92 of file utils_v2.py.

Referenced by get_version().

 def get_relval_cmssw_version(file):
     cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
     gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-([\w\d]*)_V\d*\w?(_[\w\d]*)?-v', file)
     if cmssw_release and gr_r_version:
         return (cmssw_release[0], gr_r_version[0])
 

◆ get_relval_id()

def utils_v2.get_relval_id ( file )

Returns unique relval ID (dataset name) for a given file.

Definition at line 98 of file utils_v2.py.

 def get_relval_id(file):
     """Returns unique relval ID (dataset name) for a given file."""
     dataset_name = re.findall('R\d{9}__([\w\d]*)__CMSSW_', file)
     return dataset_name[0]
 

◆ get_relval_max_version()

def utils_v2.get_relval_max_version ( files )

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 80 of file utils_v2.py.

References get_relval_version().

 def get_relval_max_version(files):
     """Returns file with maximum version at a) beggining of the file,
     e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
     max_file = files[0]
     max_v = get_relval_version(files[0])
     for file in files:
         file_v = get_relval_version(file)
         if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
             max_file = file
             max_v = file_v
     return max_file
 

◆ get_relval_version()

def utils_v2.get_relval_version ( file )

-------------—— Make files pairs: RelVal utils ---------------——

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 73 of file utils_v2.py.

References createfilelist.int.

Referenced by get_relval_max_version().

 def get_relval_version(file):
     """Returns tuple (CMSSW version, run version) for specified file."""
     cmssw_version = re.findall('DQM_V(\d*)_', file)
     run_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-[\w\d]*_V\d*\w?(?:_[\w\d]*)?-v(\d*)__', file)
     if cmssw_version and run_version:
         return (int(cmssw_version[0]), int(run_version[0]))
 

◆ get_relvaldata_cmssw_version()

def utils_v2.get_relvaldata_cmssw_version ( file )

Returns tuple (CMSSW release, GR_R version) for specified RelValData file.

Definition at line 42 of file utils_v2.py.

Referenced by get_version().

 def get_relvaldata_cmssw_version(file):
     """Returns tuple (CMSSW release, GR_R version) for specified RelValData file."""
     cmssw_release = re.findall('(CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?)-', file)
     gr_r_version = re.findall('-(GR_R_\d*_V\d*\w?)(?:_RelVal)?_', file)
     if not gr_r_version:
         gr_r_version = re.findall('CMSSW_\d*_\d*_\d*(?:_[\w\d]*)?-(\w*)_RelVal_', file)
     if cmssw_release and gr_r_version:
         return (cmssw_release[0], gr_r_version[0])
 

◆ get_relvaldata_id()

def utils_v2.get_relvaldata_id ( file )

-----------—— Make files pairs: RelValData utils --------------——

Returns unique relvaldata ID for a given file.

Definition at line 32 of file utils_v2.py.

 def get_relvaldata_id(file):
     """Returns unique relvaldata ID for a given file."""
     run_id = re.search('R\d{9}', file)
     run = re.search('_RelVal_([\w\d]*)-v\d__', file)
     if not run:
         run = re.search('GR_R_\d*_V\d*C?_([\w\d]*)-v\d__', file)
     if run_id and run:
         return (run_id.group(), run.group(1))
     return None
 

◆ get_relvaldata_max_version()

def utils_v2.get_relvaldata_max_version ( files )

Returns file with maximum version at a) beggining of the file,
e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max.

Definition at line 60 of file utils_v2.py.

References get_relvaldata_version().

 def get_relvaldata_max_version(files):
     """Returns file with maximum version at a) beggining of the file,
     e.g. DQM_V000M b) at the end of run, e.g. _run2012-vM. M has to be max."""
     max_file = files[0]
     max_v = get_relvaldata_version(files[0])
     for file in files:
         file_v = get_relvaldata_version(file)
         if file_v[1] > max_v[1] or ((file_v[1] == max_v[1]) and (file_v[0] > max_v[0])):
             max_file = file
             max_v = file_v
     return max_file
 

◆ get_relvaldata_version()

def utils_v2.get_relvaldata_version ( file )

Returns tuple (CMSSW version, run version) for specified file.

Definition at line 51 of file utils_v2.py.

References createfilelist.int.

Referenced by get_relvaldata_max_version().

 def get_relvaldata_version(file):
     """Returns tuple (CMSSW version, run version) for specified file."""
     cmssw_version = re.findall('DQM_V(\d*)_', file)
     run_version = re.findall('_RelVal_[\w\d]*-v(\d)__', file)
     if not run_version:
         run_version = re.findall('GR_R_\d*_V\d*C?_[\w\d]*-v(\d)__', file)
     if cmssw_version and run_version:
         return (int(cmssw_version[0]), int(run_version[0]))
 

◆ get_size_to_download()

def utils_v2.get_size_to_download	(	work_path,
		files_with_urls
	)

Returns file list to download and total size to download.

Definition at line 446 of file utils_v2.py.

References createfilelist.int, join(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def get_size_to_download(work_path, files_with_urls):
     """Returns file list to download and total size to download."""
     opener = build_opener(X509CertOpen())
     size_to_download = 0
     files_to_download = []
     for filename, url in files_with_urls:
         url_file = opener.open(Request(url))
         size = int(url_file.headers["Content-Length"])
         file_path = join(work_path, filename)
         if exists(file_path) and getsize(file_path) / 1024 == size / 1024:
             print("Exists on disk %s." % filename)
         else:
             size_to_download += size
             files_to_download.append(url)
     return size_to_download, files_to_download
 

◆ get_version()

def utils_v2.get_version ( filename )

Returns CMSSW and GR_R versions for the given filename.

Definition at line 435 of file utils_v2.py.

References get_relval_cmssw_version(), get_relvaldata_cmssw_version(), is_relvaldata(), join(), and nano_mu_digi_cff.strip.

 def get_version(filename):
     """Returns CMSSW and GR_R versions for the given filename."""
     if is_relvaldata([filename]):
         version_elems = get_relvaldata_cmssw_version(filename)
     else:
         relval_version = get_relval_cmssw_version(filename)
         version_elems = (relval_version[0], relval_version[1][0], relval_version[1][1])
     version_elems = [elem.strip('_').strip('RelVal_') for elem in version_elems]
     return '___'.join([elem for elem in version_elems if elem])
 
 

◆ init_database()

def utils_v2.init_database ( db_path )

Utils.

Definition at line 391 of file utils_v2.py.

References print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def init_database(db_path):
     print('Initialising DB: %s...' % basename(db_path), end=' ')
     conn = sqlite3.connect(db_path)
 
     
     c = conn.cursor()
     c.execute("""CREATE TABLE IF NOT EXISTS ReleaseComparison (
                         id INTEGER PRIMARY KEY,
                         title TEXT,
                         release1 TEXT,
                         release2 TEXT,
                         statistical_test TEXT
                     );""")
     c.execute("""CREATE TABLE IF NOT EXISTS Directory (
                         id INTEGER PRIMARY KEY,
                         name TEXT,
                         parent_id INTEGER,
                         from_histogram_id INTEGER,
                         till_histogram_id INTEGER,
                         FOREIGN KEY (parent_id) REFERENCES Directory(id)
                         FOREIGN KEY (from_histogram_id) REFERENCES HistogramComparison(id)
                         FOREIGN KEY (till_histogram_id) REFERENCES HistogramComparison(id)
                     )""")
     c.execute("""CREATE TABLE IF NOT EXISTS RootFileComparison (
                         id INTEGER PRIMARY KEY,
                         filename1 TEXT,
                         filename2 TEXT,
                         release_comparison_id INTEGER,
                         directory_id INTEGER,
                         FOREIGN KEY (release_comparison_id) REFERENCES ReleaseComparison(id),
                         FOREIGN KEY (directory_id) REFERENCES Directory(id)
                     )""")
     c.execute("""CREATE TABLE IF NOT EXISTS HistogramComparison (
                         id INTEGER PRIMARY KEY,
                         name TEXT,
                         p_value REAL,
                         directory_id INTEGER,
                         FOREIGN KEY (directory_id) REFERENCES Directory(id)
                     )""")
 
     print('Done.')
     return db_path
 
 

◆ is_relvaldata()

def utils_v2.is_relvaldata ( files )

--------------------— Make file pairs -----------------------—

Definition at line 104 of file utils_v2.py.

References any().

Referenced by get_version(), and make_file_pairs().

 def is_relvaldata(files):
     is_relvaldata_re = re.compile('_RelVal_')
     return any([is_relvaldata_re.search(filename) for filename in files])
 

◆ make_file_pairs()

def utils_v2.make_file_pairs	(	files1,
		files2
	)

Definition at line 108 of file utils_v2.py.

References mps_setup.append, beamvalidation.exit(), is_relvaldata(), print(), and str.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def make_file_pairs(files1, files2):
     print('\n#################       Analyzing files       ###################')
     
     if is_relvaldata(files1):
         is_relval_data = True
         get_cmssw_version = get_relvaldata_cmssw_version
         get_id = get_relvaldata_id
         get_max_version = get_relvaldata_max_version
     else:
         is_relval_data = False
         get_cmssw_version = get_relval_cmssw_version
         get_id = get_relval_id
         get_max_version = get_relval_max_version
 
     
     versions1, versions2 = dict(), dict() # {version1: [file1, file2, ...], version2: [...], ...}
     for files, versions in (files1, versions1), (files2, versions2):
         for file in files:
             version = get_cmssw_version(file)
             if version:
                 if version in versions:
                     versions[version].append(file)
                 else:
                     versions[version] = [file]
 
     
     print('For RELEASE1 found file groups:')
     for version in versions1:
         print('   %s: %d files' % (str(version),  len(versions1[version])))
     if not versions1:
         print('None.')
 
     print('\nFor RELEASE2 found file groups:')
     for version in versions2:
         print('   %s: %d files' % (str(version),  len(versions2[version])))
     if not versions2:
         print('None.')
 
     if not len(versions1) or not len(versions2):
         print('\nNot enough file groups. Exiting...\n')
         exit()
 
     
     pairs = []
     for v1 in sorted(versions1, key=lambda x: len(versions1[x]), reverse=True):
         for v2 in sorted(versions2, key=lambda x: len(versions2[x]), reverse=True):
             if v1 == v2:
                 continue
             
             print('\n#################     Pairing the files     ###################')
             print('%s (%d files)   VS   %s (%d files):\n' % (str(v1),
                     len(versions1[v1]), str(v2), len(versions2[v2])))
 
             
             for unique_id in set([get_id(file) for file in versions1[v1]]):
                 if is_relval_data:
                     dataset_re = re.compile(unique_id[0] + '_')
                     run_re = re.compile(unique_id[1])
                     c1_files = [file for file in versions1[v1] if dataset_re.search(file) and run_re.search(file)]
                     c2_files = [file for file in versions2[v2] if dataset_re.search(file) and run_re.search(file)]
                 else:
                     dataset_re = re.compile(unique_id + '_')
                     c1_files = [file for file in versions1[v1] if dataset_re.search(file)]
                     c2_files = [file for file in versions2[v2] if dataset_re.search(file)]
 
                 if len(c1_files) > 0 and len(c2_files) > 0:
                     first_file = get_max_version(c1_files)
                     second_file = get_max_version(c2_files)
                     print('%s\n%s\n' % (first_file, second_file))
                     pairs.append((first_file, second_file))
 
             print("Got %d pairs." % (len(pairs)))
             if pairs:
                 return pairs
     print('Found no file pairs. Exiting..\n')
     exit()
 

◆ recursive_search_online()

def utils_v2.recursive_search_online	(	url,
		rel1,
		frags1,
		rel2,
		frags2
	)

Recursively searches for files, that matches the pattern.

Definition at line 216 of file utils_v2.py.

References python.cmstools.all(), auth_wget(), join(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def recursive_search_online(url, rel1, frags1, rel2, frags2):
     """Recursively searches for files, that matches the pattern."""
     if not url:
         url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelValData/'
         g1, g2 = recursive_search_online(url, rel1, frags1, rel2, frags2)
         url = 'https://cmsweb.cern.ch/dqm/relval/data/browse/ROOT/RelVal/'
         g3, g4 = recursive_search_online(url, rel1, frags1, rel2, frags2)
         g1.update(g3), g2.update(g4)
         return g1, g2
 
     domain = '://'.join(urlparse(url)[:2])
 
     
     href_re = re.compile(r"<a href='([-./\w]*)'>([-./\w]*)<")
 
     def compile_res(rel, frags):
         frags = frags.split(',')
         regexps = [s for s in frags if not s.startswith('!')]
         regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
         regexps += [rel + '-', '.root']
         return [re.compile(r) for r in regexps]
 
     res1 = compile_res(rel1, frags1)
     res2 = compile_res(rel2, frags2)
 
     
     hrefs = [(name, path) for path, name in href_re.findall(auth_wget(url))[1:]]
     files_with_urls1, files_with_urls2 = dict(), dict()
     for name, path in hrefs:
         if splitext(name)[1]: # If file
             if all([r.search(name) for r in res1]):
                 files_with_urls1[name] = domain + path
             if all([r.search(name) for r in res2]):
                 files_with_urls2[name] = domain + path
         else:
             print(domain + path)
             new_hrefs = href_re.findall(auth_wget(domain + path))[1:]
             hrefs.extend([(name, path) for path, name in new_hrefs])
     return files_with_urls1, files_with_urls2
 

◆ search_on_disk()

def utils_v2.search_on_disk	(	work_path,
		rel1,
		frags1,
		rel2,
		frags2
	)

Definition at line 256 of file utils_v2.py.

References python.cmstools.all(), beamvalidation.exit(), and print().

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

 def search_on_disk(work_path, rel1, frags1, rel2, frags2):
     if not work_path:
         print('No working directory specified. Use "--dir DIR" option to ' +\
               'specify working directory. Exiting...')
         exit()
     
     def compile_res(rel, frags):
         frags = frags.split(',')
         regexps = [s for s in frags if not s.startswith('!')]
         regexps += ['^((?%s).)*$' % s for s in frags if s.startswith('!')]
         regexps += [rel + '-', '.root']
         return [re.compile(r) for r in regexps]
 
     res1 = compile_res(rel1, frags1)
     res2 = compile_res(rel2, frags2)
 
     
     files = listdir(work_path)
     files1, files2 = [], []
     for name in files:
         if splitext(name)[1]:
             if all([r.search(name) for r in res1]):
                 files1.append(name)
             if all([r.search(name) for r in res2]):
                 files2.append(name)
     return files1, files2
 
 

◆ show_status_bar()

def utils_v2.show_status_bar ( total_size )

Shows download status.

Definition at line 480 of file utils_v2.py.

 def show_status_bar(total_size):
     """Shows download status."""
     q = show_status_bar.q
     total_size = total_size / (1024*1024)
     downloaded = 0
     while downloaded < total_size:
         try:
             o = q.get(timeout=20)
             downloaded += 1
             print('\r      %d/%d MB     %d%%     ' % (downloaded, total_size, 100*downloaded/total_size), end=' ')
             sys.stdout.flush()
         except Empty:
             time.sleep(1)
             break

Variable Documentation

◆ comparison_errors

utils_v2.comparison_errors

Exception definitions.

Definition at line 285 of file utils_v2.py.

◆ tests

utils_v2.tests

Definition at line 388 of file utils_v2.py.

Classes

Functions

Variables

Detailed Description

Function Documentation

◆ auth_download_file()

◆ auth_wget()

◆ check_disk_for_space()

◆ get_relval_cmssw_version()

◆ get_relval_id()

◆ get_relval_max_version()

◆ get_relval_version()

◆ get_relvaldata_cmssw_version()

◆ get_relvaldata_id()

◆ get_relvaldata_max_version()

◆ get_relvaldata_version()

◆ get_size_to_download()

◆ get_version()

◆ init_database()

◆ is_relvaldata()

◆ make_file_pairs()

◆ recursive_search_online()

◆ search_on_disk()

◆ show_status_bar()

Variable Documentation

◆ comparison_errors

◆ tests