CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
ValidationMatrix Namespace Reference

Functions

def add_to_blacklist
 
def call_compare_using_files
 
def check_root_files
 
def count_alive_processes
 
def do_comparisons_threaded
 
def do_html
 
def do_reports
 
def get_clean_fileanames
 
def get_filenames_from_pool
 
def get_roofiles_in_dir
 
def guess_blacklists
 
def guess_params
 
def name2run
 
def name2runskim
 
def name2sample
 
def name2version
 

Variables

string action = "store"
 —HASHING—## More...
 
string all_samples = ""
 
 argv = sys.argv
 
 default = ref_samples,
 
string dest = "ref_samples"
 
string help = "The samples that act as reference (comma separated list)"
 
 hlt = False
 
string in_dir = ""
 
int n_processes = 1
 
int n_threads = 1
 
string out_dir = ""
 
tuple parser = OptionParser(usage="usage: %prog [options]")
 
string ref_samples = ""
 
int run = -1
 
string stat_test = "Chi2"
 
string test_samples = ""
 
float test_threshold = 0.00001
 

Function Documentation

def ValidationMatrix.add_to_blacklist (   blacklist,
  pattern,
  target,
  blist_piece 
)

Definition at line 122 of file ValidationMatrix.py.

References dataformats.search().

Referenced by guess_blacklists().

122 
123 def add_to_blacklist(blacklist, pattern, target, blist_piece):
124  int_pattern=pattern
125  int_pattern=pattern.strip()
126  flip_condition=False
127  if int_pattern[0]=='!':
128  int_pattern=int_pattern[1:]
129  flip_condition=True
130 
131  condition = search(int_pattern,target)!=None
132  if flip_condition:
133  condition = not condition
134 
135  if condition:
136  #print "Found %s in %s" %(pattern,target)
137  if blacklist!="": # if not the first, add a comma
138  blacklist+=","
139  blacklist+=blist_piece
140  #else:
141  #print " NOT Found %s in %s" %(pattern,target)
142  return blacklist
143 
144 #-------------------------------------------------------------------------------
def ValidationMatrix.call_compare_using_files (   args) 
Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 272 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, guess_blacklists(), and name2version().

272 
273 def call_compare_using_files(args):
274  """Creates shell command to compare two files using compare_using_files.py
275  script and calls it."""
276  sample, ref_filename, test_filename, options = args
277  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
278  command = " compare_using_files.py "
279  command+= "%s %s " %(ref_filename,test_filename)
280  command+= " -C -R "
281  if options.do_pngs:
282  command+= " -p "
283  command+= " -o %s " %sample
284  # Change threshold to an experimental and empirical value of 10^-5
285  command+= " --specify_run "
286  command+= " -t %s " %options.test_threshold
287  command+= " -s %s " %options.stat_test
288 
289  # Inspect the HLT directories
290  if options.hlt:
291  command+=" -d HLT "
292 
293  if options.hash_name:
294  command += " --hash_name "
295 
296  if options.blacklist_file:
297  command += " --use_black_file "
298 
299  if len(blacklists[sample]) >0:
300  command+= '-B %s ' %blacklists[sample]
301  print "\nExecuting -- %s" %command
302 
303  process=call(filter(lambda x: len(x)>0,command.split(" ")))
304  return process
305 
306 
307 #--------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.check_root_files (   names_list)

Definition at line 113 of file ValidationMatrix.py.

Referenced by get_clean_fileanames().

113 
114 def check_root_files(names_list):
115  for name in names_list:
116  if not name.endswith(".root"):
117  print "File %s does not seem to be a rootfile. Please check."
118  return False
119  return True
120 
121 #-------------------------------------------------------------------------------
def ValidationMatrix.count_alive_processes (   p_list)

Definition at line 267 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

267 
268 def count_alive_processes(p_list):
269  return len(filter(lambda p: p.returncode==None,p_list))
270 
271 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.do_comparisons_threaded (   options)

Definition at line 308 of file ValidationMatrix.py.

References get_clean_fileanames(), get_filenames_from_pool(), guess_params(), list(), Association.map, and dataformats.search().

308 
309 def do_comparisons_threaded(options):
310 
311  n_processes= int(options.n_processes)
312 
313  ref_filenames=[]
314  test_filenames=[]
315 
316  if len(options.all_samples)>0:
317  ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)
318  else:
319  ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
320 
321  # make the paths absolute
322  ref_filenames=map(os.path.abspath,ref_filenames)
323  test_filenames=map(os.path.abspath,test_filenames)
324 
325  samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
326 
327  if len(samples)==0:
328  print "No Samples found... Quitting"
329  return 0
330 
331 # blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
332 
333  # Launch the single comparisons
334  original_dir=os.getcwd()
335 
336  outdir=options.out_dir
337  if len(outdir)==0:
338  print "Creating automatic outdir:",
339  outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
340  print outdir
341  if len(options.input_dir)==0:
342  print "Creating automatic indir:",
343  options.input_dir=outdir
344  print options.input_dir
345 
346  if not os.path.exists(outdir):
347  os.mkdir(outdir)
348  os.chdir(outdir)
349 
350  # adjust the number of threads
351  n_comparisons=len(ref_filenames)
352  if n_comparisons < n_processes:
353  print "Less comparisons than possible processes: reducing n processes to",
354  n_processes=n_comparisons
355  #elif n_processes/n_comparisons == 0:
356  #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",
357  #original_nprocesses=n_processes
358  #first=True
359  #n_bunches=0
360  #while first or n_processes > original_nprocesses:
361  #n_processes=n_comparisons/2
362  #if n_comparisons%2 !=0:
363  #n_processes+=1
364  #first=False
365 
366  #print n_processes
367  #print n_processes
368 
369  # Test if we treat data
370  skim_name=""
371  if search("20[01]",cmssw_version1)!=None:
372  skim_name=cmssw_version1.split("_")[-1]
373 
374  running_subprocesses=[]
375  process_counter=0
376  #print ref_filenames
377 
378  ## Compare all pairs of root files
379  pool = Pool(n_processes)
380  args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
381  pool.map(call_compare_using_files, args_iterable)
382  # move the pickles on the top, hack
383  os.system("mv */*pkl .")
384 
385  os.chdir("..")
#-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def ValidationMatrix.do_html (   options,
  hashing_flag 
)

Definition at line 414 of file ValidationMatrix.py.

References do_reports(), and directories2html.make_summary_table().

414 
415 def do_html(options, hashing_flag):
416 
417  if options.reports:
418  print "Preparing reports for the single files..."
419  do_reports(options.input_dir)
420  # Do the summary page
421  aggregation_rules={}
422  aggregation_rules_twiki={}
423  # check which aggregation rules are to be used
424  if options.hlt:
425  print "Aggregating directories according to HLT rules"
426  aggregation_rules=definitions.aggr_pairs_dict['HLT']
427  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
428  else:
429  aggregation_rules=definitions.aggr_pairs_dict['reco']
430  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
431  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag)
432 
433  # create summary html file
434  ofile = open("RelMonSummary.html","w")
435  ofile.write(table_html)
436  ofile.close()
437 
438 #-------------------------------------------------------------------------------
def ValidationMatrix.do_reports (   indir)

Definition at line 386 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

Referenced by do_html().

386 
387 def do_reports(indir):
388  #print indir
389  os.chdir(indir)
390  pkl_list=filter(lambda x:".pkl" in x, os.listdir("./"))
391  running_subprocesses=[]
392  n_processes=int(options.n_processes)
393  process_counter=0
394  for pklfilename in pkl_list:
395  command = "compare_using_files.py "
396  command+= "-R "
397  if options.do_pngs:
398  command+= " -p "
399  command+= "-P %s " %pklfilename
400  command+= "-o %s " %pklfilename[:-4]
401  print "Executing %s" %command
402  process=call(filter(lambda x: len(x)>0,command.split(" ")))
403  process_counter+=1
404  # add it to the list
405  running_subprocesses.append(process)
406  if process_counter>=n_processes:
407  process_counter=0
408  for p in running_subprocesses:
409  #print "Waiting for %s" %p.name
410  p.wait()
411 
412  os.chdir("..")
413 
#-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.get_clean_fileanames (   ref_samples,
  test_samples 
)

Definition at line 252 of file ValidationMatrix.py.

References check_root_files(), cmsRelvalreport.exit, and Association.map.

Referenced by do_comparisons_threaded().

252 
253 def get_clean_fileanames(ref_samples,test_samples):
254  # Process the samples starting from the names
255  ref_filenames=map(lambda s:s.strip(),ref_samples.split(","))
256  test_filenames=map(lambda s:s.strip(),test_samples.split(","))
257 
258  if len(ref_filenames)!=len(test_filenames):
259  print "The numebr of reference and test files does not seem to be the same. Please check."
260  exit(2)
261 
262  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
263  exit(2)
264  return ref_filenames,test_filenames
265 
266 #-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
def ValidationMatrix.get_filenames_from_pool (   all_samples)

Definition at line 198 of file ValidationMatrix.py.

References get_roofiles_in_dir(), and utils.make_files_pairs().

Referenced by do_comparisons_threaded().

198 
199 def get_filenames_from_pool(all_samples):
200 
201  # get a list of the files
202  files_list=get_roofiles_in_dir(all_samples)
203 
204  if len(files_list)==0:
205  print "Zero files found in directory %s!" %all_samples
206  return [],[]
207 
208  # Are they an even number?
209  for name in files_list:
210  print "* ",name
211  if len(files_list)%2!=0:
212  print "The numbuer of file is not even... Trying to recover a catastrophe."
213 
214  files_list=make_files_pairs(files_list)
215 
216  # Try to couple them according to their sample
217  ref_filenames=[]
218  test_filenames=[]
219  #files_list.sort(key=name2version)
220  #files_list.sort(key=name2sample)
221  #files_list.sort(key=name2run)
222  for iname in xrange(len(files_list)):
223  filename=files_list[iname]
224  if iname%2==0:
225  ref_filenames.append(filename)
226  else:
227  test_filenames.append(filename)
228 
229  print "The guess would be the following:"
230  for ref,test in zip(ref_filenames,test_filenames):
231  refbasedir=os.path.dirname(ref)
232  testbasedir=os.path.dirname(test)
233  dir_to_print=refbasedir
234  if refbasedir!=testbasedir:
235  dir_to_print="%s and %s" %(refbasedir,testbasedir)
236  print "* Directory: %s " %dir_to_print
237  refname=os.path.basename(ref)
238  testname=os.path.basename(test)
239  print " o %s" %refname
240  print " o %s" %testname
241 
242  #is_ok=ask_ok("Is that ok?")
243  #if not is_ok:
244  #print "Manual input needed then!"
245  #exit(2)
246 
247 
248  return ref_filenames,test_filenames
249 
250 
251 #-------------------------------------------------------------------------------
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:552
def ValidationMatrix.get_roofiles_in_dir (   directory)

Definition at line 189 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, and Association.map.

Referenced by get_filenames_from_pool().

189 
190 def get_roofiles_in_dir(directory):
191  print directory
192  files_list = filter(lambda s: s.endswith(".root"), os.listdir(directory))
193  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
194 
195  return files_list_path
196 
197 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
Association.map
dictionary map
Definition: Association.py:205
def ValidationMatrix.guess_blacklists (   samples,
  ver1,
  ver2,
  hlt 
) 
Build a blacklist for each sample accordind to a set of rules

Definition at line 145 of file ValidationMatrix.py.

References add_to_blacklist(), and dataformats.search().

Referenced by call_compare_using_files().

145 
146 def guess_blacklists(samples,ver1,ver2,hlt):
147  """Build a blacklist for each sample accordind to a set of rules
148  """
149  blacklists={}
150  for sample in samples:
151  blacklists[sample]="FED@1,AlcaBeamMonitor@1,Physics@1,Info@-1,HLT@1,AlCaReco@1"
152 
153  # HLT
154  if hlt: #HLT
155  blacklists[sample]+=",AlCaEcalPi0@2"
156  if not search("2010+|2011+",ver1):
157  print "We are treating MC files for the HLT"
158  for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
159  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
160 # print 'HLT '+pattern
161 # print 'HLT '+sample
162 # print 'HLT '+blacklists[sample]
163  else:
164  print "We are treating Data files for the HLT"
165  # at the moment it does not make sense since hlt is ran already
166 
167  else: #RECO
168  #Monte Carlo
169  if not search("2010+|2011+",ver1):
170  print "We are treating MC files"
171 
172  for pattern,blist in definitions.mc_pattern_blist_pairs:
173  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
174 # print "MC RECO"
175  #print blacklists[sample]
176 
177  # Data
178  else:
179  print "We are treating Data files:"
180  blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"
181  for pattern,blist in definitions.data_pattern_blist_pairs:
182  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
183 # print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])
184 
185 
186  return blacklists
187 
188 #-------------------------------------------------------------------------------
def ValidationMatrix.guess_params (   ref_filenames,
  test_filenames 
)

Definition at line 66 of file ValidationMatrix.py.

References cmsRelvalreport.exit, list(), Association.map, name2sample(), name2version(), dataformats.search(), and runtimedef.set().

Referenced by do_comparisons_threaded().

66 
67 def guess_params(ref_filenames,test_filenames):
68 
69  if len(ref_filenames)*len(test_filenames)==0:
70  print "Empty reference and test filenames lists!"
71  return [],"",""
72 
73  samples=[]
74  ref_versions=[]
75  test_versions=[]
76 
77  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
78 
79  ref_sample=name2sample(ref)
80  ref_version=name2version(ref)
81  test_sample=name2sample(test)
82  test_version=name2version(test)
83 
84  if ref_sample!=test_sample:
85  print "Files %s and %s do not seem to be relative to the same sample." %(ref, test)
86  exit(2)
87 
88  # Slightly modify for data
89  if search("20[01]",ref_version)!=None:
90  ref_sample+=ref_version.split("_")[-1]
91  samples.append(ref_sample)
92 
93  # append the versions
94  ref_versions.append(ref_version)
95  test_versions.append(test_version)
96 
97  # Check if ref and test versions are always the same.
98  ref_versions=list(set(ref_versions))
99  test_versions=list(set(test_versions))
100 
101  #for versions in ref_versions,test_versions:
102  #if len(versions)!=1:
103  #print "More than one kind of CMSSW version selected (%s)" %versions
104  #exit(2)
105 
106  cmssw_version1=ref_versions[0]
107  cmssw_version2=test_versions[0]
108 
109  return samples,cmssw_version1,cmssw_version2
110 
111 
112 #-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
runtimedef::set
void set(const std::string &name, int value)
set the flag, with a run-time name
Definition: ProfilingTools.cc:67
def ValidationMatrix.name2run (   filename)

Definition at line 52 of file ValidationMatrix.py.

References split.

Referenced by name2runskim().

52 
53 def name2run(filename):
54  namebase=os.path.basename(filename)
55  return namebase.split("__")[0].split("_")[2]
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2runskim (   filename)

Definition at line 56 of file ValidationMatrix.py.

References name2run(), name2version(), and split.

56 
57 def name2runskim(filename):
58  run=name2run(filename)
59  skim=name2version(filename).split("_")[-1]
60  # remove skim version
61  if "-v" in skim:
62  skim = skim[:skim.rfind('-v')]
63  return "%s_%s"%(run,skim)
64 
65 #-------------------------------------------------------------------------------
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2sample (   filename)

Definition at line 44 of file ValidationMatrix.py.

Referenced by guess_params().

44 
45 def name2sample(filename):
46  namebase=os.path.basename(filename)
47  return namebase.split("__")[1]
def ValidationMatrix.name2version (   filename)

Definition at line 48 of file ValidationMatrix.py.

Referenced by call_compare_using_files(), guess_params(), and name2runskim().

48 
49 def name2version(filename):
50  namebase=os.path.basename(filename)
51  return namebase.split("__")[2]

Variable Documentation

string ValidationMatrix.action = "store"

—HASHING—##

–Blacklist File –##

Definition at line 459 of file ValidationMatrix.py.

string ValidationMatrix.all_samples = ""

Definition at line 444 of file ValidationMatrix.py.

ValidationMatrix.argv = sys.argv

Definition at line 26 of file ValidationMatrix.py.

ValidationMatrix.default = ref_samples,

Definition at line 461 of file ValidationMatrix.py.

string ValidationMatrix.dest = "ref_samples"

Definition at line 460 of file ValidationMatrix.py.

string ValidationMatrix.help = "The samples that act as reference (comma separated list)"

Definition at line 462 of file ValidationMatrix.py.

ValidationMatrix.hlt = False

Definition at line 452 of file ValidationMatrix.py.

string ValidationMatrix.in_dir = ""

Definition at line 447 of file ValidationMatrix.py.

int ValidationMatrix.n_processes = 1

Definition at line 445 of file ValidationMatrix.py.

int ValidationMatrix.n_threads = 1

Definition at line 448 of file ValidationMatrix.py.

string ValidationMatrix.out_dir = ""

Definition at line 446 of file ValidationMatrix.py.

Referenced by HtrXmlPattern.HtrXmlPattern().

tuple ValidationMatrix.parser = OptionParser(usage="usage: %prog [options]")

Definition at line 456 of file ValidationMatrix.py.

string ValidationMatrix.ref_samples = ""

Definition at line 442 of file ValidationMatrix.py.

int ValidationMatrix.run = -1

Definition at line 449 of file ValidationMatrix.py.

string ValidationMatrix.stat_test = "Chi2"

Definition at line 450 of file ValidationMatrix.py.

string ValidationMatrix.test_samples = ""

Definition at line 443 of file ValidationMatrix.py.

float ValidationMatrix.test_threshold = 0.00001

Definition at line 451 of file ValidationMatrix.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5