CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
ValidationMatrix Namespace Reference

Functions

def add_to_blacklist
 
def call_compare_using_files
 
def check_root_files
 
def count_alive_processes
 
def do_comparisons_threaded
 
def do_html
 
def do_reports
 
def get_clean_fileanames
 
def get_filenames_from_pool
 
def get_roofiles_in_dir
 
def guess_blacklists
 
def guess_params
 
def name2globaltag
 
def name2run
 
def name2runskim
 
def name2sample
 
def name2version
 

Variables

string action = "store"
 —HASHING—## More...
 
string all_samples = ""
 
 argv = sys.argv
 
 default = ref_samples,
 
string dest = "ref_samples"
 
string help = "The samples that act as reference (comma separated list)"
 
 hlt = False
 
string in_dir = ""
 
int n_processes = 1
 
int n_threads = 1
 
string out_dir = ""
 
tuple parser = OptionParser(usage="usage: %prog [options]")
 
string ref_samples = ""
 
int run = -1
 
string stat_test = "Chi2"
 
string test_samples = ""
 
float test_threshold = 0.00001
 

Function Documentation

def ValidationMatrix.add_to_blacklist (   blacklist,
  pattern,
  target,
  blist_piece 
)

Definition at line 127 of file ValidationMatrix.py.

References cond::persistency.search().

Referenced by guess_blacklists().

127 
128 def add_to_blacklist(blacklist, pattern, target, blist_piece):
129  int_pattern=pattern
130  int_pattern=pattern.strip()
131  flip_condition=False
132  if int_pattern[0]=='!':
133  int_pattern=int_pattern[1:]
134  flip_condition=True
135 
136  condition = search(int_pattern,target)!=None
137  if flip_condition:
138  condition = not condition
139 
140  if condition:
141  #print "Found %s in %s" %(pattern,target)
142  if blacklist!="": # if not the first, add a comma
143  blacklist+=","
144  blacklist+=blist_piece
145  #else:
146  #print " NOT Found %s in %s" %(pattern,target)
147  return blacklist
148 
149 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:21
def ValidationMatrix.call_compare_using_files (   args) 
Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 274 of file ValidationMatrix.py.

References guess_blacklists(), name2globaltag(), name2version(), and print().

274 
275 def call_compare_using_files(args):
276  """Creates shell command to compare two files using compare_using_files.py
277  script and calls it."""
278  sample, ref_filename, test_filename, options = args
279  gt = name2globaltag(ref_filename)
280  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
281  command = " compare_using_files.py "
282  command+= "%s %s " %(ref_filename,test_filename)
283  command+= " -C -R "
284  if options.do_pngs:
285  command+= " -p "
286  command+= " -o %s_%s " %(sample, gt)
287  # Change threshold to an experimental and empirical value of 10^-5
288  command+= " --specify_run "
289  if options.stat_test in ["Bin2Bin", "BinToBin"]:
290  options.test_threshold = 0.9999
291  command+= " -t %s " %options.test_threshold
292  command+= " -s %s " %options.stat_test
293 
294  # Inspect the HLT directories
295  if options.hlt:
296  command+=" -d HLT "
297 
298  if options.hash_name:
299  command += " --hash_name "
300 
301  if options.blacklist_file:
302  command += " --use_black_file "
303 
304  if options.standalone:
305  command += " --standalone "
306  if len(blacklists[sample]) >0:
307  command+= '-B %s ' %blacklists[sample]
308  print("\nExecuting -- %s" %command)
309 
310  process=call([x for x in command.split(" ") if len(x)>0])
311  return process
312 
313 
314 #--------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.check_root_files (   names_list)

Definition at line 118 of file ValidationMatrix.py.

References print().

Referenced by get_clean_fileanames().

118 
119 def check_root_files(names_list):
120  for name in names_list:
121  if not name.endswith(".root"):
122  print("File %s does not seem to be a rootfile. Please check.")
123  return False
124  return True
125 
126 #-------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.count_alive_processes (   p_list)

Definition at line 269 of file ValidationMatrix.py.

269 
270 def count_alive_processes(p_list):
271  return len([p for p in p_list if p.returncode==None])
272 
273 #-------------------------------------------------------------------------------
def ValidationMatrix.do_comparisons_threaded (   options)

Definition at line 315 of file ValidationMatrix.py.

References get_clean_fileanames(), get_filenames_from_pool(), guess_params(), print(), cond::persistency.search(), and ComparisonHelper.zip().

315 
316 def do_comparisons_threaded(options):
317 
318  n_processes= int(options.n_processes)
319 
320  ref_filenames=[]
321  test_filenames=[]
322 
323  if len(options.all_samples)>0:
324  ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)
325  else:
326  ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
327 
328  # make the paths absolute
329  ref_filenames=list(map(os.path.abspath,ref_filenames))
330  test_filenames=list(map(os.path.abspath,test_filenames))
331 
332  samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
333 
334  if len(samples)==0:
335  print("No Samples found... Quitting")
336  return 0
337 
338 # blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
339 
340  # Launch the single comparisons
341  original_dir=os.getcwd()
342 
343  outdir=options.out_dir
344  if len(outdir)==0:
345  print("Creating automatic outdir:", end=' ')
346  outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
347  print(outdir)
348  if len(options.input_dir)==0:
349  print("Creating automatic indir:", end=' ')
350  options.input_dir=outdir
351  print(options.input_dir)
352 
353  if not os.path.exists(outdir):
354  os.mkdir(outdir)
355  os.chdir(outdir)
356 
357  # adjust the number of threads
358  n_comparisons=len(ref_filenames)
359  if n_comparisons < n_processes:
360  print("Less comparisons than possible processes: reducing n processes to", end=' ')
361  n_processes=n_comparisons
362  #elif n_processes/n_comparisons == 0:
363  #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",
364  #original_nprocesses=n_processes
365  #first=True
366  #n_bunches=0
367  #while first or n_processes > original_nprocesses:
368  #n_processes=n_comparisons/2
369  #if n_comparisons%2 !=0:
370  #n_processes+=1
371  #first=False
372 
373  #print n_processes
374  #print n_processes
375 
376  # Test if we treat data
377  skim_name=""
378  if search("20[01]",cmssw_version1)!=None:
379  skim_name=cmssw_version1.split("_")[-1]
380 
381  running_subprocesses=[]
382  process_counter=0
383  #print ref_filenames
384 
385  ## Compare all pairs of root files
386  pool = Pool(n_processes)
387  args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
388  pool.map(call_compare_using_files, args_iterable)
389  # move the pickles on the top, hack
390  os.system("mv */*pkl .")
391 
392  os.chdir("..")
#-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:21
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:41
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.do_html (   options,
  hashing_flag,
  standalone 
)

Definition at line 421 of file ValidationMatrix.py.

References do_reports(), directories2html.make_summary_table(), and print().

421 
422 def do_html(options, hashing_flag, standalone):
423 
424  if options.reports:
425  print("Preparing reports for the single files...")
426  do_reports(options.input_dir)
427  # Do the summary page
428  aggregation_rules={}
429  aggregation_rules_twiki={}
430  # check which aggregation rules are to be used
431  if options.hlt:
432  print("Aggregating directories according to HLT rules")
433  aggregation_rules=definitions.aggr_pairs_dict['HLT']
434  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
435  else:
436  aggregation_rules=definitions.aggr_pairs_dict['reco']
437  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
438  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone)
439 
440  # create summary html file
441  ofile = open("RelMonSummary.html","w")
442  ofile.write(table_html)
443  ofile.close()
444 
445 #-------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.do_reports (   indir)

Definition at line 393 of file ValidationMatrix.py.

References print().

Referenced by do_html().

393 
394 def do_reports(indir):
395  #print indir
396  os.chdir(indir)
397  pkl_list=[x for x in os.listdir("./") if ".pkl" in x]
398  running_subprocesses=[]
399  n_processes=int(options.n_processes)
400  process_counter=0
401  for pklfilename in pkl_list:
402  command = "compare_using_files.py "
403  command+= "-R "
404  if options.do_pngs:
405  command+= " -p "
406  command+= "-P %s " %pklfilename
407  command+= "-o %s " %pklfilename[:-4]
408  print("Executing %s" %command)
409  process=call([x for x in command.split(" ") if len(x)>0])
410  process_counter+=1
411  # add it to the list
412  running_subprocesses.append(process)
413  if process_counter>=n_processes:
414  process_counter=0
415  for p in running_subprocesses:
416  #print "Waiting for %s" %p.name
417  p.wait()
418 
419  os.chdir("..")
420 
#-------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.get_clean_fileanames (   ref_samples,
  test_samples 
)

Definition at line 254 of file ValidationMatrix.py.

References check_root_files(), beamvalidation.exit(), and print().

Referenced by do_comparisons_threaded().

254 
255 def get_clean_fileanames(ref_samples,test_samples):
256  # Process the samples starting from the names
257  ref_filenames=list(map(lambda s:s.strip(),ref_samples.split(",")))
258  test_filenames=list(map(lambda s:s.strip(),test_samples.split(",")))
259 
260  if len(ref_filenames)!=len(test_filenames):
261  print("The numebr of reference and test files does not seem to be the same. Please check.")
262  exit(2)
263 
264  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
265  exit(2)
266  return ref_filenames,test_filenames
267 
268 #-------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.get_filenames_from_pool (   all_samples)

Definition at line 200 of file ValidationMatrix.py.

References get_roofiles_in_dir(), utils.make_files_pairs(), print(), sistrip::SpyUtilities.range(), and ComparisonHelper.zip().

Referenced by do_comparisons_threaded().

200 
201 def get_filenames_from_pool(all_samples):
202 
203  # get a list of the files
204  files_list=get_roofiles_in_dir(all_samples)
205 
206  if len(files_list)==0:
207  print("Zero files found in directory %s!" %all_samples)
208  return [],[]
209 
210  # Are they an even number?
211  for name in files_list:
212  print("* ",name)
213  if len(files_list)%2!=0:
214  print("The numbuer of file is not even... Trying to recover a catastrophe.")
215 
216  files_list=make_files_pairs(files_list)
217 
218  # Try to couple them according to their sample
219  ref_filenames=[]
220  test_filenames=[]
221  #files_list.sort(key=name2version)
222  #files_list.sort(key=name2sample)
223  #files_list.sort(key=name2run)
224  for iname in range(len(files_list)):
225  filename=files_list[iname]
226  if iname%2==0:
227  ref_filenames.append(filename)
228  else:
229  test_filenames.append(filename)
230 
231  print("The guess would be the following:")
232  for ref,test in zip(ref_filenames,test_filenames):
233  refbasedir=os.path.dirname(ref)
234  testbasedir=os.path.dirname(test)
235  dir_to_print=refbasedir
236  if refbasedir!=testbasedir:
237  dir_to_print="%s and %s" %(refbasedir,testbasedir)
238  print("* Directory: %s " %dir_to_print)
239  refname=os.path.basename(ref)
240  testname=os.path.basename(test)
241  print(" o %s" %refname)
242  print(" o %s" %testname)
243 
244  #is_ok=ask_ok("Is that ok?")
245  #if not is_ok:
246  #print "Manual input needed then!"
247  #exit(2)
248 
249 
250  return ref_filenames,test_filenames
251 
252 
253 #-------------------------------------------------------------------------------
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:41
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:573
def ValidationMatrix.get_roofiles_in_dir (   directory)

Definition at line 191 of file ValidationMatrix.py.

References print().

Referenced by get_filenames_from_pool().

191 
192 def get_roofiles_in_dir(directory):
193  print(directory)
194  files_list = [s for s in os.listdir(directory) if s.endswith(".root")]
195  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
196 
197  return files_list_path
198 
199 #-------------------------------------------------------------------------------
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.guess_blacklists (   samples,
  ver1,
  ver2,
  hlt 
) 
Build a blacklist for each sample accordind to a set of rules

Definition at line 150 of file ValidationMatrix.py.

References add_to_blacklist(), print(), and cond::persistency.search().

Referenced by call_compare_using_files().

150 
151 def guess_blacklists(samples,ver1,ver2,hlt):
152  """Build a blacklist for each sample accordind to a set of rules
153  """
154  blacklists={}
155  for sample in samples:
156  blacklists[sample]="FED@1,AlcaBeamMonitor@1,HLT@1,AlCaReco@1"
157 
158  # HLT
159  if hlt: #HLT
160  blacklists[sample]+=",AlCaEcalPi0@2"
161  if not search("2010+|2011+|2012+|2015+",ver1):
162  print("We are treating MC files for the HLT")
163  for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
164  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
165  else:
166  print("We are treating Data files for the HLT")
167  # at the moment it does not make sense since hlt is ran already
168 
169  else: #RECO
170  #Monte Carlo
171  if not search("2010+|2011+|2012+",ver1):
172  print("We are treating MC files")
173 
174  for pattern,blist in definitions.mc_pattern_blist_pairs:
175  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
176 # print "MC RECO"
177  #print blacklists[sample]
178 
179  # Data
180  else:
181  print("We are treating Data files:")
182  blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"
183  for pattern,blist in definitions.data_pattern_blist_pairs:
184  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
185 # print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])
186 
187 
188  return blacklists
189 
190 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:21
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.guess_params (   ref_filenames,
  test_filenames 
)

Definition at line 69 of file ValidationMatrix.py.

References name2sample(), name2version(), print(), cond::persistency.search(), and ComparisonHelper.zip().

Referenced by do_comparisons_threaded().

69 
70 def guess_params(ref_filenames,test_filenames):
71 
72  if len(ref_filenames)*len(test_filenames)==0:
73  print("Empty reference and test filenames lists!")
74  return [],"",""
75 
76  samples=[]
77  ref_versions=[]
78  test_versions=[]
79 
80  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
81 
82  ref_sample=name2sample(ref)
83  ref_version=name2version(ref)
84  test_sample=name2sample(test)
85  test_version=name2version(test)
86 
87  print(" ## sample 1: %s vs sample 2: %s"%(ref_sample, test_sample))
88 
89  if ref_sample!=test_sample:
90  print("Files %s and %s do not seem to be relative to the same sample." %(ref, test))
91  # exit(2)
92 
93  # Slightly modify for data
94  if search("20[01]",ref_version)!=None:
95  ref_sample+=ref_version.split("_")[-1]
96  samples.append(ref_sample)
97 
98  # append the versions
99  ref_versions.append(ref_version)
100  test_versions.append(test_version)
101 
102  # Check if ref and test versions are always the same.
103  ref_versions=list(set(ref_versions))
104  test_versions=list(set(test_versions))
105 
106  #for versions in ref_versions,test_versions:
107  #if len(versions)!=1:
108  #print "More than one kind of CMSSW version selected (%s)" %versions
109  #exit(2)
110 
111  cmssw_version1=ref_versions[0]
112  cmssw_version2=test_versions[0]
113 
114  return samples,cmssw_version1,cmssw_version2
115 
116 
117 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:21
ComparisonHelper::zip
OutputIterator zip(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp)
Definition: L1TStage2CaloLayer1.h:41
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def ValidationMatrix.name2globaltag (   filename)

Definition at line 63 of file ValidationMatrix.py.

References submitPVValidationJobs.split().

Referenced by call_compare_using_files().

63 
64 def name2globaltag(filename):
65  namebase = os.path.basename(filename)
66  return namebase.split("__")[2].split("-")[1] #returns GT from file basename
67 
68 #-------------------------------------------------------------------------------
def ValidationMatrix.name2run (   filename)

Definition at line 51 of file ValidationMatrix.py.

References submitPVValidationJobs.split().

Referenced by name2runskim().

51 
52 def name2run(filename):
53  namebase=os.path.basename(filename)
54  return namebase.split("__")[0].split("_")[2]
def ValidationMatrix.name2runskim (   filename)

Definition at line 55 of file ValidationMatrix.py.

References name2run(), name2version(), and submitPVValidationJobs.split().

55 
56 def name2runskim(filename):
57  run=name2run(filename)
58  skim=name2version(filename).split("_")[-1]
59  # remove skim version
60  if "-v" in skim:
61  skim = skim[:skim.rfind('-v')]
62  return "%s_%s"%(run,skim)
def ValidationMatrix.name2sample (   filename)

Definition at line 43 of file ValidationMatrix.py.

Referenced by guess_params().

43 
44 def name2sample(filename):
45  namebase=os.path.basename(filename)
46  return namebase.split("__")[1]
def ValidationMatrix.name2version (   filename)

Definition at line 47 of file ValidationMatrix.py.

Referenced by call_compare_using_files(), guess_params(), and name2runskim().

47 
48 def name2version(filename):
49  namebase=os.path.basename(filename)
50  return namebase.split("__")[2]

Variable Documentation

string ValidationMatrix.action = "store"

—HASHING—##

– USE CSS files in web access, for stand-alone usage –##

–Blacklist File –##

Definition at line 466 of file ValidationMatrix.py.

string ValidationMatrix.all_samples = ""

Definition at line 451 of file ValidationMatrix.py.

ValidationMatrix.argv = sys.argv

Definition at line 25 of file ValidationMatrix.py.

ValidationMatrix.default = ref_samples,

Definition at line 468 of file ValidationMatrix.py.

string ValidationMatrix.dest = "ref_samples"

Definition at line 467 of file ValidationMatrix.py.

string ValidationMatrix.help = "The samples that act as reference (comma separated list)"

Definition at line 469 of file ValidationMatrix.py.

ValidationMatrix.hlt = False

Definition at line 459 of file ValidationMatrix.py.

Referenced by DQMHcalIsolatedBunchAlCaReco.analyze(), StudyTriggerHLT.analyze(), IsoTrackCalibration.analyze(), HcalIsoTrkSimAnalyzer.analyze(), StudyCaloResponse.analyze(), HcalIsoTrkAnalyzer.analyze(), IsoTrig.analyze(), CSCValidation.analyze(), LumiCalculator.beginRun(), AlCaIsolatedBunchFilter.filter(), AlCaIsolatedBunchSelector.filter(), AlCaIsoTracksProducerFilter.filter(), AlCaHBHEMuonFilter.filter(), AlCaLowPUHBHEMuonFilter.filter(), AlCaHEMuonFilter.filter(), AlCaIsoTracksFilter.filter(), main(), AlCaHcalHBHEMuonProducer.produce(), AlCaHcalIsotrkProducer.produce(), and AlCaIsoTracksProducer.select().

string ValidationMatrix.in_dir = ""

Definition at line 454 of file ValidationMatrix.py.

int ValidationMatrix.n_processes = 1

Definition at line 452 of file ValidationMatrix.py.

int ValidationMatrix.n_threads = 1

Definition at line 455 of file ValidationMatrix.py.

string ValidationMatrix.out_dir = ""

Definition at line 453 of file ValidationMatrix.py.

tuple ValidationMatrix.parser = OptionParser(usage="usage: %prog [options]")

Definition at line 463 of file ValidationMatrix.py.

string ValidationMatrix.ref_samples = ""

Definition at line 449 of file ValidationMatrix.py.

int ValidationMatrix.run = -1

Definition at line 456 of file ValidationMatrix.py.

string ValidationMatrix.stat_test = "Chi2"

Definition at line 457 of file ValidationMatrix.py.

string ValidationMatrix.test_samples = ""

Definition at line 450 of file ValidationMatrix.py.

float ValidationMatrix.test_threshold = 0.00001

Definition at line 458 of file ValidationMatrix.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5