Inheritance diagram for ValidationMatrix_v2.ReleaseComparison:

Public Member Functions
def	__init__ (self, work_path=None, db_name=None, clear_db=False, dry=False, no_url=False, use_external=False)

def	compare (self, rel1, frags1, rel2, frags2, st_tests, url=None, title=None)

def	was_compared (self, release1, release2, st_test_name)

Public Attributes
	clear_db

	db_name
	Download needed files. More...

	dry

	no_url

	use_external_script_to_compare_files

	work_path
	Create working directory if not given. More...

Detailed Description

Generates release comparison information and stores it on SQLite3 .db file.

Definition at line 121 of file ValidationMatrix_v2.py.

Constructor & Destructor Documentation

def ValidationMatrix_v2.ReleaseComparison.__init__	(	self,
		work_path = `None`,
		db_name = `None`,
		clear_db = `False`,
		dry = `False`,
		no_url = `False`,
		use_external = `False`
	)

Definition at line 123 of file ValidationMatrix_v2.py.

     def __init__(self, work_path=None, db_name=None, clear_db=False, dry=False, no_url=False, use_external=False):
         self.work_path = work_path
         self.db_name = db_name
         self.clear_db = clear_db
         self.dry = dry
         self.no_url = no_url
         self.use_external_script_to_compare_files = use_external
 

Member Function Documentation

def ValidationMatrix_v2.ReleaseComparison.compare	(	self,
		rel1,
		frags1,
		rel2,
		frags2,
		st_tests,
		url = `None`,
		title = `None`
	)

Definition at line 142 of file ValidationMatrix_v2.py.

References estimatePileup.basename, utils_v2.check_disk_for_space(), ValidationMatrix_v2.ReleaseComparison.clear_db, compare_using_files_v2.RootFileComparison.db_name, ValidationMatrix_v2.ReleaseComparison.db_name, ValidationMatrix_v2.ReleaseComparison.dry, cmsRelvalreport.exit, utils_v2.get_size_to_download(), utils_v2.get_version(), utils_v2.init_database(), join(), list(), utils_v2.make_file_pairs(), genParticles_cff.map, ValidationMatrix_v2.merge_dbs(), ValidationMatrix_v2.ReleaseComparison.no_url, ValidationMatrix_v2.partial_db_name(), edm.print(), utils_v2.recursive_search_online(), MatrixUtil.remove(), utils_v2.search_on_disk(), ValidationMatrix_v2.ReleaseComparison.use_external_script_to_compare_files, compare_using_files_v2.RootFileComparison.was_compared(), ValidationMatrix_v2.ReleaseComparison.was_compared(), compare_using_files_v2.RootFileComparison.work_path, ValidationMatrix_v2.ReleaseComparison.work_path, and ComparisonHelper.zip().

     def compare(self, rel1, frags1, rel2, frags2, st_tests, url=None, title=None):
         print('\n#################     Searching for files     ###################')
         if self.no_url:
             print('Searching for files on disk at %s' % (self.work_path))
             files1, files2 = search_on_disk(self.work_path, rel1, frags1, rel2, frags2)
             file_pairs = make_file_pairs(files1, files2)
         else:
             print('Searching for files online at:')
             files_with_urls1, files_with_urls2 = recursive_search_online(url, rel1, frags1, rel2, frags2)
             file_pairs = make_file_pairs(files_with_urls1, files_with_urls2)
             files_with_urls1.update(files_with_urls2)
             files1, files2 = list(zip(*file_pairs))
             paired_files_with_urls = [(file, files_with_urls1[file]) for file in files1 + files2]
 
             if self.dry:
                 print('DRY: nothing to do. Exiting.')
                 exit()
 
             ## Create working directory if not given.
             if not self.work_path:
                 self.work_path = '%s___VS___%s' % (get_version(files1[0]), get_version(files2[0]))
                 if self.db_name:
                     self.db_name = join(self.work_path, self.db_name)
 
             if not exists(self.work_path):
                 print('\n###################      Preparing directory     ###################')
                 print('Creating working directory: %s ...' % self.work_path, end=' ')
                 makedirs(self.work_path)
                 print('Done.')
 
             print('\n#################     Downloading the files     ###################')
             total_size, files_to_download = get_size_to_download(self.work_path, paired_files_with_urls)
             check_disk_for_space(self.work_path, total_size)
 
             ## Download needed files.
             q = Queue()
             show_status_bar.q = q
             auth_download_file.q = q
             auth_download_file.work_dir = self.work_path
 
             Process(target=show_status_bar, args=(total_size,)).start()
             Pool(2).map(auth_download_file, files_to_download)
             if total_size:
                 print("Done.")
 
         ## Create database
         print('\n#################     Preparing Database     ###################')
         if not self.db_name:
             self.db_name = '%s___VS___%s.db' % (get_version(file_pairs[0][0]), get_version(file_pairs[0][1]))
 
         if self.clear_db:
             print('Clearing DB: %s...' % self.db_name, end=' ')
             open(join(self.work_path, self.db_name), 'w').close()
             print('Done.')
 
         ## Compare file pairs.
         self.db_name = init_database(join(self.work_path, self.db_name))
 
         # TODO: Use multiprocessing for this task.
         for st_test_name in st_tests.split(','):
             print('\n#################     Comparing Releases (%s)     ###################' % st_test_name)
             st_test = tests[st_test_name]()
 
             some_files_compared = False
             file_comparison_ids = []
             if self.use_external_script_to_compare_files:
                 # Compare files using compare_using_files_v2.py
                 arg_list = [list(pair) + [self.work_path, partial_db_name(self.db_name, i),
                                                 self.clear_db] for i, pair in enumerate(file_pairs)]
                 pool = Pool(7)
                 pool.map(call_compare_using_files, arg_list)
 
                 # Merge databases
                 print('\n#################     Merging DBs (%s)     ###################' % st_test_name)
                 for i, pair in enumerate(file_pairs):
                     tmp_db = partial_db_name(self.db_name, i)
                     print('Merging %s...' % (basename(tmp_db),), end=' ')
                     file_comparison_ids.append(merge_dbs(self.db_name, tmp_db))
                     remove(tmp_db)
                     print('Done.')
                     some_files_compared = True
             else:
                 file_comparison = RootFileComparison(self.db_name)
 
                 for file1, file2 in file_pairs:
                     # TODO: If files are not found desplay nice message.
                     # TODO: Maybe subprocces would control the unwanted reports of RootFileComparison.compare()
                     file1_path = join(self.work_path, file1)
                     file2_path = join(self.work_path, file2)
 
                     if not file_comparison.was_compared(file1, file2, st_test_name):
                         print("Comparing:\n%s\n%s\n" % (file1, file2))
                         file_comparison_id = file_comparison.compare(file1_path, file2_path, st_test)
                         file_comparison_ids.append(file_comparison_id)
                         some_files_compared = True
                     else:
                         print("Already compared:\n%s\n%s\n" % (file1, file2))
 
             ## Calculate statistics for the release.
             release1 = get_version(file_pairs[0][0])
             release2 = get_version(file_pairs[0][1])
             if some_files_compared:
                 release_comparison_id = self.was_compared(release1, release2, st_test_name)
                 conn = sqlite3.connect(self.db_name)
                 c = conn.cursor()
                 if not release_comparison_id:
                     print('Inserting release "%s  VS  %s" description.\n' % (release1, release2))
                     if not title:
                         title = "%s__VS__%s" % (release1, release2)
                     c.execute('''INSERT INTO ReleaseComparison(title, release1, release2,
                                    statistical_test) VALUES (?, ?, ?, ?)''', (title,
                                 release1, release2, st_test_name))
                     release_comparison_id = c.lastrowid
                 c.executemany('''UPDATE RootFileComparison SET release_comparison_id = ?
                         WHERE id == ?''', [(release_comparison_id, fid) for fid in file_comparison_ids])
                 conn.commit()
                 conn.close()
 
 

def ValidationMatrix_v2.ReleaseComparison.was_compared	(	self,
		release1,
		release2,
		st_test_name
	)

Definition at line 131 of file ValidationMatrix_v2.py.

References compare_using_files_v2.RootFileComparison.db_name, and ValidationMatrix_v2.ReleaseComparison.db_name.

Referenced by ValidationMatrix_v2.ReleaseComparison.compare().

     def was_compared(self, release1, release2, st_test_name):
         conn = sqlite3.connect(self.db_name)
         c = conn.cursor()
         c.execute('''SELECT id FROM ReleaseComparison WHERE release1=? AND
                 release2=? AND statistical_test=?''', (release1, release2, st_test_name))
         release_comparison_id = c.fetchone()
         conn.close()
         if release_comparison_id:
             return release_comparison_id[0]
         return False
 