CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
ValidationMatrix Namespace Reference

Functions

def add_to_blacklist
 
def call_compare_using_files
 
def check_root_files
 
def count_alive_processes
 
def do_comparisons_threaded
 
def do_html
 
def do_reports
 
def get_clean_fileanames
 
def get_filenames_from_pool
 
def get_roofiles_in_dir
 
def guess_blacklists
 
def guess_params
 
def name2run
 
def name2runskim
 
def name2sample
 
def name2version
 

Variables

string action = "store"
 —HASHING—## More...
 
string all_samples = ""
 
 argv = sys.argv
 
 default = ref_samples,
 
string dest = "ref_samples"
 
string help = "The samples that act as reference (comma separated list)"
 
 hlt = False
 
string in_dir = ""
 
int n_processes = 1
 
int n_threads = 1
 
string out_dir = ""
 
tuple parser = OptionParser(usage="usage: %prog [options]")
 
string ref_samples = ""
 
int run = -1
 
string stat_test = "Chi2"
 
string test_samples = ""
 
float test_threshold = 0.00001
 

Function Documentation

def ValidationMatrix.add_to_blacklist (   blacklist,
  pattern,
  target,
  blist_piece 
)

Definition at line 124 of file ValidationMatrix.py.

References dataformats.search().

Referenced by guess_blacklists().

124 
125 def add_to_blacklist(blacklist, pattern, target, blist_piece):
126  int_pattern=pattern
127  int_pattern=pattern.strip()
128  flip_condition=False
129  if int_pattern[0]=='!':
130  int_pattern=int_pattern[1:]
131  flip_condition=True
132 
133  condition = search(int_pattern,target)!=None
134  if flip_condition:
135  condition = not condition
136 
137  if condition:
138  #print "Found %s in %s" %(pattern,target)
139  if blacklist!="": # if not the first, add a comma
140  blacklist+=","
141  blacklist+=blist_piece
142  #else:
143  #print " NOT Found %s in %s" %(pattern,target)
144  return blacklist
145 
146 #-------------------------------------------------------------------------------
def ValidationMatrix.call_compare_using_files (   args) 
Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 271 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, guess_blacklists(), and name2version().

271 
272 def call_compare_using_files(args):
273  """Creates shell command to compare two files using compare_using_files.py
274  script and calls it."""
275  sample, ref_filename, test_filename, options = args
276  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
277  command = " compare_using_files.py "
278  command+= "%s %s " %(ref_filename,test_filename)
279  command+= " -C -R "
280  if options.do_pngs:
281  command+= " -p "
282  command+= " -o %s " %sample
283  # Change threshold to an experimental and empirical value of 10^-5
284  command+= " --specify_run "
285  command+= " -t %s " %options.test_threshold
286  command+= " -s %s " %options.stat_test
287 
288  # Inspect the HLT directories
289  if options.hlt:
290  command+=" -d HLT "
291 
292  if options.hash_name:
293  command += " --hash_name "
294 
295  if options.blacklist_file:
296  command += " --use_black_file "
297 
298  if options.standalone:
299  command += " --standalone "
300  if len(blacklists[sample]) >0:
301  command+= '-B %s ' %blacklists[sample]
302  print "\nExecuting -- %s" %command
303 
304  process=call(filter(lambda x: len(x)>0,command.split(" ")))
305  return process
306 
307 
308 #--------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.check_root_files (   names_list)

Definition at line 115 of file ValidationMatrix.py.

Referenced by get_clean_fileanames().

115 
116 def check_root_files(names_list):
117  for name in names_list:
118  if not name.endswith(".root"):
119  print "File %s does not seem to be a rootfile. Please check."
120  return False
121  return True
122 
123 #-------------------------------------------------------------------------------
def ValidationMatrix.count_alive_processes (   p_list)

Definition at line 266 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

266 
267 def count_alive_processes(p_list):
268  return len(filter(lambda p: p.returncode==None,p_list))
269 
270 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.do_comparisons_threaded (   options)

Definition at line 309 of file ValidationMatrix.py.

References get_clean_fileanames(), get_filenames_from_pool(), guess_params(), list(), Association.map, and dataformats.search().

309 
310 def do_comparisons_threaded(options):
311 
312  n_processes= int(options.n_processes)
313 
314  ref_filenames=[]
315  test_filenames=[]
316 
317  if len(options.all_samples)>0:
318  ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)
319  else:
320  ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
321 
322  # make the paths absolute
323  ref_filenames=map(os.path.abspath,ref_filenames)
324  test_filenames=map(os.path.abspath,test_filenames)
325 
326  samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
327 
328  if len(samples)==0:
329  print "No Samples found... Quitting"
330  return 0
331 
332 # blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
333 
334  # Launch the single comparisons
335  original_dir=os.getcwd()
336 
337  outdir=options.out_dir
338  if len(outdir)==0:
339  print "Creating automatic outdir:",
340  outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
341  print outdir
342  if len(options.input_dir)==0:
343  print "Creating automatic indir:",
344  options.input_dir=outdir
345  print options.input_dir
346 
347  if not os.path.exists(outdir):
348  os.mkdir(outdir)
349  os.chdir(outdir)
350 
351  # adjust the number of threads
352  n_comparisons=len(ref_filenames)
353  if n_comparisons < n_processes:
354  print "Less comparisons than possible processes: reducing n processes to",
355  n_processes=n_comparisons
356  #elif n_processes/n_comparisons == 0:
357  #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",
358  #original_nprocesses=n_processes
359  #first=True
360  #n_bunches=0
361  #while first or n_processes > original_nprocesses:
362  #n_processes=n_comparisons/2
363  #if n_comparisons%2 !=0:
364  #n_processes+=1
365  #first=False
366 
367  #print n_processes
368  #print n_processes
369 
370  # Test if we treat data
371  skim_name=""
372  if search("20[01]",cmssw_version1)!=None:
373  skim_name=cmssw_version1.split("_")[-1]
374 
375  running_subprocesses=[]
376  process_counter=0
377  #print ref_filenames
378 
379  ## Compare all pairs of root files
380  pool = Pool(n_processes)
381  args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
382  pool.map(call_compare_using_files, args_iterable)
383  # move the pickles on the top, hack
384  os.system("mv */*pkl .")
385 
386  os.chdir("..")
#-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def ValidationMatrix.do_html (   options,
  hashing_flag,
  standalone 
)

Definition at line 415 of file ValidationMatrix.py.

References do_reports(), and directories2html.make_summary_table().

415 
416 def do_html(options, hashing_flag, standalone):
417 
418  if options.reports:
419  print "Preparing reports for the single files..."
420  do_reports(options.input_dir)
421  # Do the summary page
422  aggregation_rules={}
423  aggregation_rules_twiki={}
424  # check which aggregation rules are to be used
425  if options.hlt:
426  print "Aggregating directories according to HLT rules"
427  aggregation_rules=definitions.aggr_pairs_dict['HLT']
428  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
429  else:
430  aggregation_rules=definitions.aggr_pairs_dict['reco']
431  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
432  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone)
433 
434  # create summary html file
435  ofile = open("RelMonSummary.html","w")
436  ofile.write(table_html)
437  ofile.close()
438 
439 #-------------------------------------------------------------------------------
def ValidationMatrix.do_reports (   indir)

Definition at line 387 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

Referenced by do_html().

387 
388 def do_reports(indir):
389  #print indir
390  os.chdir(indir)
391  pkl_list=filter(lambda x:".pkl" in x, os.listdir("./"))
392  running_subprocesses=[]
393  n_processes=int(options.n_processes)
394  process_counter=0
395  for pklfilename in pkl_list:
396  command = "compare_using_files.py "
397  command+= "-R "
398  if options.do_pngs:
399  command+= " -p "
400  command+= "-P %s " %pklfilename
401  command+= "-o %s " %pklfilename[:-4]
402  print "Executing %s" %command
403  process=call(filter(lambda x: len(x)>0,command.split(" ")))
404  process_counter+=1
405  # add it to the list
406  running_subprocesses.append(process)
407  if process_counter>=n_processes:
408  process_counter=0
409  for p in running_subprocesses:
410  #print "Waiting for %s" %p.name
411  p.wait()
412 
413  os.chdir("..")
414 
#-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.get_clean_fileanames (   ref_samples,
  test_samples 
)

Definition at line 251 of file ValidationMatrix.py.

References check_root_files(), cmsRelvalreport.exit, and Association.map.

Referenced by do_comparisons_threaded().

251 
252 def get_clean_fileanames(ref_samples,test_samples):
253  # Process the samples starting from the names
254  ref_filenames=map(lambda s:s.strip(),ref_samples.split(","))
255  test_filenames=map(lambda s:s.strip(),test_samples.split(","))
256 
257  if len(ref_filenames)!=len(test_filenames):
258  print "The numebr of reference and test files does not seem to be the same. Please check."
259  exit(2)
260 
261  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
262  exit(2)
263  return ref_filenames,test_filenames
264 
265 #-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
def ValidationMatrix.get_filenames_from_pool (   all_samples)

Definition at line 197 of file ValidationMatrix.py.

References get_roofiles_in_dir(), and utils.make_files_pairs().

Referenced by do_comparisons_threaded().

197 
198 def get_filenames_from_pool(all_samples):
199 
200  # get a list of the files
201  files_list=get_roofiles_in_dir(all_samples)
202 
203  if len(files_list)==0:
204  print "Zero files found in directory %s!" %all_samples
205  return [],[]
206 
207  # Are they an even number?
208  for name in files_list:
209  print "* ",name
210  if len(files_list)%2!=0:
211  print "The numbuer of file is not even... Trying to recover a catastrophe."
212 
213  files_list=make_files_pairs(files_list)
214 
215  # Try to couple them according to their sample
216  ref_filenames=[]
217  test_filenames=[]
218  #files_list.sort(key=name2version)
219  #files_list.sort(key=name2sample)
220  #files_list.sort(key=name2run)
221  for iname in xrange(len(files_list)):
222  filename=files_list[iname]
223  if iname%2==0:
224  ref_filenames.append(filename)
225  else:
226  test_filenames.append(filename)
227 
228  print "The guess would be the following:"
229  for ref,test in zip(ref_filenames,test_filenames):
230  refbasedir=os.path.dirname(ref)
231  testbasedir=os.path.dirname(test)
232  dir_to_print=refbasedir
233  if refbasedir!=testbasedir:
234  dir_to_print="%s and %s" %(refbasedir,testbasedir)
235  print "* Directory: %s " %dir_to_print
236  refname=os.path.basename(ref)
237  testname=os.path.basename(test)
238  print " o %s" %refname
239  print " o %s" %testname
240 
241  #is_ok=ask_ok("Is that ok?")
242  #if not is_ok:
243  #print "Manual input needed then!"
244  #exit(2)
245 
246 
247  return ref_filenames,test_filenames
248 
249 
250 #-------------------------------------------------------------------------------
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:552
def ValidationMatrix.get_roofiles_in_dir (   directory)

Definition at line 188 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, and Association.map.

Referenced by get_filenames_from_pool().

188 
189 def get_roofiles_in_dir(directory):
190  print directory
191  files_list = filter(lambda s: s.endswith(".root"), os.listdir(directory))
192  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
193 
194  return files_list_path
195 
196 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
Association.map
dictionary map
Definition: Association.py:205
def ValidationMatrix.guess_blacklists (   samples,
  ver1,
  ver2,
  hlt 
) 
Build a blacklist for each sample accordind to a set of rules

Definition at line 147 of file ValidationMatrix.py.

References add_to_blacklist(), and dataformats.search().

Referenced by call_compare_using_files().

147 
148 def guess_blacklists(samples,ver1,ver2,hlt):
149  """Build a blacklist for each sample accordind to a set of rules
150  """
151  blacklists={}
152  for sample in samples:
153  blacklists[sample]="FED@1,AlcaBeamMonitor@1,HLT@1,AlCaReco@1"
154 
155  # HLT
156  if hlt: #HLT
157  blacklists[sample]+=",AlCaEcalPi0@2"
158  if not search("2010+|2011+",ver1):
159  print "We are treating MC files for the HLT"
160  for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
161  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
162  else:
163  print "We are treating Data files for the HLT"
164  # at the moment it does not make sense since hlt is ran already
165 
166  else: #RECO
167  #Monte Carlo
168  if not search("2010+|2011+",ver1):
169  print "We are treating MC files"
170 
171  for pattern,blist in definitions.mc_pattern_blist_pairs:
172  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
173 # print "MC RECO"
174  #print blacklists[sample]
175 
176  # Data
177  else:
178  print "We are treating Data files:"
179  blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"
180  for pattern,blist in definitions.data_pattern_blist_pairs:
181  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
182 # print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])
183 
184 
185  return blacklists
186 
187 #-------------------------------------------------------------------------------
def ValidationMatrix.guess_params (   ref_filenames,
  test_filenames 
)

Definition at line 66 of file ValidationMatrix.py.

References list(), Association.map, name2sample(), name2version(), and dataformats.search().

Referenced by do_comparisons_threaded().

66 
67 def guess_params(ref_filenames,test_filenames):
68 
69  if len(ref_filenames)*len(test_filenames)==0:
70  print "Empty reference and test filenames lists!"
71  return [],"",""
72 
73  samples=[]
74  ref_versions=[]
75  test_versions=[]
76 
77  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
78 
79  ref_sample=name2sample(ref)
80  ref_version=name2version(ref)
81  test_sample=name2sample(test)
82  test_version=name2version(test)
83 
84  print " ## sample 1: %s vs sample 2: %s"%(ref_sample, test_sample)
85 
86  if ref_sample!=test_sample:
87  print "Files %s and %s do not seem to be relative to the same sample." %(ref, test)
88  # exit(2)
89 
90  # Slightly modify for data
91  if search("20[01]",ref_version)!=None:
92  ref_sample+=ref_version.split("_")[-1]
93  samples.append(ref_sample)
94 
95  # append the versions
96  ref_versions.append(ref_version)
97  test_versions.append(test_version)
98 
99  # Check if ref and test versions are always the same.
100  ref_versions=list(set(ref_versions))
101  test_versions=list(set(test_versions))
102 
103  #for versions in ref_versions,test_versions:
104  #if len(versions)!=1:
105  #print "More than one kind of CMSSW version selected (%s)" %versions
106  #exit(2)
107 
108  cmssw_version1=ref_versions[0]
109  cmssw_version2=test_versions[0]
110 
111  return samples,cmssw_version1,cmssw_version2
112 
113 
114 #-------------------------------------------------------------------------------
Association.map
dictionary map
Definition: Association.py:205
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def ValidationMatrix.name2run (   filename)

Definition at line 52 of file ValidationMatrix.py.

References split.

Referenced by name2runskim().

52 
53 def name2run(filename):
54  namebase=os.path.basename(filename)
55  return namebase.split("__")[0].split("_")[2]
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2runskim (   filename)

Definition at line 56 of file ValidationMatrix.py.

References name2run(), name2version(), and split.

56 
57 def name2runskim(filename):
58  run=name2run(filename)
59  skim=name2version(filename).split("_")[-1]
60  # remove skim version
61  if "-v" in skim:
62  skim = skim[:skim.rfind('-v')]
63  return "%s_%s"%(run,skim)
64 
65 #-------------------------------------------------------------------------------
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2sample (   filename)

Definition at line 44 of file ValidationMatrix.py.

Referenced by guess_params().

44 
45 def name2sample(filename):
46  namebase=os.path.basename(filename)
47  return namebase.split("__")[1]
def ValidationMatrix.name2version (   filename)

Definition at line 48 of file ValidationMatrix.py.

Referenced by call_compare_using_files(), guess_params(), and name2runskim().

48 
49 def name2version(filename):
50  namebase=os.path.basename(filename)
51  return namebase.split("__")[2]

Variable Documentation

string ValidationMatrix.action = "store"

—HASHING—##

– USE CSS files in web access, for stand-alone usage –##

–Blacklist File –##

Definition at line 460 of file ValidationMatrix.py.

string ValidationMatrix.all_samples = ""

Definition at line 445 of file ValidationMatrix.py.

ValidationMatrix.argv = sys.argv

Definition at line 26 of file ValidationMatrix.py.

ValidationMatrix.default = ref_samples,

Definition at line 462 of file ValidationMatrix.py.

string ValidationMatrix.dest = "ref_samples"

Definition at line 461 of file ValidationMatrix.py.

string ValidationMatrix.help = "The samples that act as reference (comma separated list)"

Definition at line 463 of file ValidationMatrix.py.

ValidationMatrix.hlt = False

Definition at line 453 of file ValidationMatrix.py.

string ValidationMatrix.in_dir = ""

Definition at line 448 of file ValidationMatrix.py.

int ValidationMatrix.n_processes = 1

Definition at line 446 of file ValidationMatrix.py.

int ValidationMatrix.n_threads = 1

Definition at line 449 of file ValidationMatrix.py.

string ValidationMatrix.out_dir = ""

Definition at line 447 of file ValidationMatrix.py.

Referenced by HtrXmlPattern.HtrXmlPattern().

tuple ValidationMatrix.parser = OptionParser(usage="usage: %prog [options]")

Definition at line 457 of file ValidationMatrix.py.

string ValidationMatrix.ref_samples = ""

Definition at line 443 of file ValidationMatrix.py.

int ValidationMatrix.run = -1

Definition at line 450 of file ValidationMatrix.py.

string ValidationMatrix.stat_test = "Chi2"

Definition at line 451 of file ValidationMatrix.py.

string ValidationMatrix.test_samples = ""

Definition at line 444 of file ValidationMatrix.py.

float ValidationMatrix.test_threshold = 0.00001

Definition at line 452 of file ValidationMatrix.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5