CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
ValidationMatrix Namespace Reference

Functions

def add_to_blacklist
 
def call_compare_using_files
 
def check_root_files
 
def count_alive_processes
 
def do_comparisons_threaded
 
def do_html
 
def do_reports
 
def get_clean_fileanames
 
def get_filenames_from_pool
 
def get_roofiles_in_dir
 
def guess_blacklists
 
def guess_params
 
def name2run
 
def name2runskim
 
def name2sample
 
def name2version
 

Variables

string action = "store"
 —HASHING—## More...
 
string all_samples = ""
 
 argv = sys.argv
 
 default = ref_samples,
 
string dest = "ref_samples"
 
string help = "The samples that act as reference (comma separated list)"
 
 hlt = False
 
string in_dir = ""
 
int n_processes = 1
 
int n_threads = 1
 
string out_dir = ""
 
tuple parser = OptionParser(usage="usage: %prog [options]")
 
string ref_samples = ""
 
int run = -1
 
string stat_test = "Chi2"
 
string test_samples = ""
 
float test_threshold = 0.00001
 

Function Documentation

def ValidationMatrix.add_to_blacklist (   blacklist,
  pattern,
  target,
  blist_piece 
)

Definition at line 121 of file ValidationMatrix.py.

References cond::persistency.search().

Referenced by guess_blacklists().

121 
122 def add_to_blacklist(blacklist, pattern, target, blist_piece):
123  int_pattern=pattern
124  int_pattern=pattern.strip()
125  flip_condition=False
126  if int_pattern[0]=='!':
127  int_pattern=int_pattern[1:]
128  flip_condition=True
129 
130  condition = search(int_pattern,target)!=None
131  if flip_condition:
132  condition = not condition
133 
134  if condition:
135  #print "Found %s in %s" %(pattern,target)
136  if blacklist!="": # if not the first, add a comma
137  blacklist+=","
138  blacklist+=blist_piece
139  #else:
140  #print " NOT Found %s in %s" %(pattern,target)
141  return blacklist
142 
143 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:224
def ValidationMatrix.call_compare_using_files (   args) 
Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 268 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, guess_blacklists(), and name2version().

268 
269 def call_compare_using_files(args):
270  """Creates shell command to compare two files using compare_using_files.py
271  script and calls it."""
272  sample, ref_filename, test_filename, options = args
273  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
274  command = " compare_using_files.py "
275  command+= "%s %s " %(ref_filename,test_filename)
276  command+= " -C -R "
277  if options.do_pngs:
278  command+= " -p "
279  command+= " -o %s " %sample
280  # Change threshold to an experimental and empirical value of 10^-5
281  command+= " --specify_run "
282  command+= " -t %s " %options.test_threshold
283  command+= " -s %s " %options.stat_test
284 
285  # Inspect the HLT directories
286  if options.hlt:
287  command+=" -d HLT "
288 
289  if options.hash_name:
290  command += " --hash_name "
291 
292  if options.blacklist_file:
293  command += " --use_black_file "
294 
295  if options.standalone:
296  command += " --standalone "
297  if len(blacklists[sample]) >0:
298  command+= '-B %s ' %blacklists[sample]
299  print "\nExecuting -- %s" %command
300 
301  process=call(filter(lambda x: len(x)>0,command.split(" ")))
302  return process
303 
304 
305 #--------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.check_root_files (   names_list)

Definition at line 112 of file ValidationMatrix.py.

Referenced by get_clean_fileanames().

112 
113 def check_root_files(names_list):
114  for name in names_list:
115  if not name.endswith(".root"):
116  print "File %s does not seem to be a rootfile. Please check."
117  return False
118  return True
119 
120 #-------------------------------------------------------------------------------
def ValidationMatrix.count_alive_processes (   p_list)

Definition at line 263 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

263 
264 def count_alive_processes(p_list):
265  return len(filter(lambda p: p.returncode==None,p_list))
266 
267 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.do_comparisons_threaded (   options)

Definition at line 306 of file ValidationMatrix.py.

References get_clean_fileanames(), get_filenames_from_pool(), guess_params(), list(), python.multivaluedict.map(), and cond::persistency.search().

306 
307 def do_comparisons_threaded(options):
308 
309  n_processes= int(options.n_processes)
310 
311  ref_filenames=[]
312  test_filenames=[]
313 
314  if len(options.all_samples)>0:
315  ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)
316  else:
317  ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
318 
319  # make the paths absolute
320  ref_filenames=map(os.path.abspath,ref_filenames)
321  test_filenames=map(os.path.abspath,test_filenames)
322 
323  samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
324 
325  if len(samples)==0:
326  print "No Samples found... Quitting"
327  return 0
328 
329 # blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
330 
331  # Launch the single comparisons
332  original_dir=os.getcwd()
333 
334  outdir=options.out_dir
335  if len(outdir)==0:
336  print "Creating automatic outdir:",
337  outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
338  print outdir
339  if len(options.input_dir)==0:
340  print "Creating automatic indir:",
341  options.input_dir=outdir
342  print options.input_dir
343 
344  if not os.path.exists(outdir):
345  os.mkdir(outdir)
346  os.chdir(outdir)
347 
348  # adjust the number of threads
349  n_comparisons=len(ref_filenames)
350  if n_comparisons < n_processes:
351  print "Less comparisons than possible processes: reducing n processes to",
352  n_processes=n_comparisons
353  #elif n_processes/n_comparisons == 0:
354  #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",
355  #original_nprocesses=n_processes
356  #first=True
357  #n_bunches=0
358  #while first or n_processes > original_nprocesses:
359  #n_processes=n_comparisons/2
360  #if n_comparisons%2 !=0:
361  #n_processes+=1
362  #first=False
363 
364  #print n_processes
365  #print n_processes
366 
367  # Test if we treat data
368  skim_name=""
369  if search("20[01]",cmssw_version1)!=None:
370  skim_name=cmssw_version1.split("_")[-1]
371 
372  running_subprocesses=[]
373  process_counter=0
374  #print ref_filenames
375 
376  ## Compare all pairs of root files
377  pool = Pool(n_processes)
378  args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
379  pool.map(call_compare_using_files, args_iterable)
380  # move the pickles on the top, hack
381  os.system("mv */*pkl .")
382 
383  os.chdir("..")
#-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:224
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def ValidationMatrix.do_html (   options,
  hashing_flag,
  standalone 
)

Definition at line 412 of file ValidationMatrix.py.

References do_reports(), and directories2html.make_summary_table().

412 
413 def do_html(options, hashing_flag, standalone):
414 
415  if options.reports:
416  print "Preparing reports for the single files..."
417  do_reports(options.input_dir)
418  # Do the summary page
419  aggregation_rules={}
420  aggregation_rules_twiki={}
421  # check which aggregation rules are to be used
422  if options.hlt:
423  print "Aggregating directories according to HLT rules"
424  aggregation_rules=definitions.aggr_pairs_dict['HLT']
425  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
426  else:
427  aggregation_rules=definitions.aggr_pairs_dict['reco']
428  aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
429  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone)
430 
431  # create summary html file
432  ofile = open("RelMonSummary.html","w")
433  ofile.write(table_html)
434  ofile.close()
435 
436 #-------------------------------------------------------------------------------
def ValidationMatrix.do_reports (   indir)

Definition at line 384 of file ValidationMatrix.py.

References alcazmumu_cfi.filter.

Referenced by do_html().

384 
385 def do_reports(indir):
386  #print indir
387  os.chdir(indir)
388  pkl_list=filter(lambda x:".pkl" in x, os.listdir("./"))
389  running_subprocesses=[]
390  n_processes=int(options.n_processes)
391  process_counter=0
392  for pklfilename in pkl_list:
393  command = "compare_using_files.py "
394  command+= "-R "
395  if options.do_pngs:
396  command+= " -p "
397  command+= "-P %s " %pklfilename
398  command+= "-o %s " %pklfilename[:-4]
399  print "Executing %s" %command
400  process=call(filter(lambda x: len(x)>0,command.split(" ")))
401  process_counter+=1
402  # add it to the list
403  running_subprocesses.append(process)
404  if process_counter>=n_processes:
405  process_counter=0
406  for p in running_subprocesses:
407  #print "Waiting for %s" %p.name
408  p.wait()
409 
410  os.chdir("..")
411 
#-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.get_clean_fileanames (   ref_samples,
  test_samples 
)

Definition at line 248 of file ValidationMatrix.py.

References check_root_files(), cmsRelvalreport.exit, and python.multivaluedict.map().

Referenced by do_comparisons_threaded().

248 
249 def get_clean_fileanames(ref_samples,test_samples):
250  # Process the samples starting from the names
251  ref_filenames=map(lambda s:s.strip(),ref_samples.split(","))
252  test_filenames=map(lambda s:s.strip(),test_samples.split(","))
253 
254  if len(ref_filenames)!=len(test_filenames):
255  print "The numebr of reference and test files does not seem to be the same. Please check."
256  exit(2)
257 
258  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
259  exit(2)
260  return ref_filenames,test_filenames
261 
262 #-------------------------------------------------------------------------------
def ValidationMatrix.get_filenames_from_pool (   all_samples)

Definition at line 194 of file ValidationMatrix.py.

References get_roofiles_in_dir(), and utils.make_files_pairs().

Referenced by do_comparisons_threaded().

194 
195 def get_filenames_from_pool(all_samples):
196 
197  # get a list of the files
198  files_list=get_roofiles_in_dir(all_samples)
199 
200  if len(files_list)==0:
201  print "Zero files found in directory %s!" %all_samples
202  return [],[]
203 
204  # Are they an even number?
205  for name in files_list:
206  print "* ",name
207  if len(files_list)%2!=0:
208  print "The numbuer of file is not even... Trying to recover a catastrophe."
209 
210  files_list=make_files_pairs(files_list)
211 
212  # Try to couple them according to their sample
213  ref_filenames=[]
214  test_filenames=[]
215  #files_list.sort(key=name2version)
216  #files_list.sort(key=name2sample)
217  #files_list.sort(key=name2run)
218  for iname in xrange(len(files_list)):
219  filename=files_list[iname]
220  if iname%2==0:
221  ref_filenames.append(filename)
222  else:
223  test_filenames.append(filename)
224 
225  print "The guess would be the following:"
226  for ref,test in zip(ref_filenames,test_filenames):
227  refbasedir=os.path.dirname(ref)
228  testbasedir=os.path.dirname(test)
229  dir_to_print=refbasedir
230  if refbasedir!=testbasedir:
231  dir_to_print="%s and %s" %(refbasedir,testbasedir)
232  print "* Directory: %s " %dir_to_print
233  refname=os.path.basename(ref)
234  testname=os.path.basename(test)
235  print " o %s" %refname
236  print " o %s" %testname
237 
238  #is_ok=ask_ok("Is that ok?")
239  #if not is_ok:
240  #print "Manual input needed then!"
241  #exit(2)
242 
243 
244  return ref_filenames,test_filenames
245 
246 
247 #-------------------------------------------------------------------------------
utils.make_files_pairs
def make_files_pairs
Definition: utils.py:549
def ValidationMatrix.get_roofiles_in_dir (   directory)

Definition at line 185 of file ValidationMatrix.py.

References alcazmumu_cfi.filter, and python.multivaluedict.map().

Referenced by get_filenames_from_pool().

185 
186 def get_roofiles_in_dir(directory):
187  print directory
188  files_list = filter(lambda s: s.endswith(".root"), os.listdir(directory))
189  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
190 
191  return files_list_path
192 
193 #-------------------------------------------------------------------------------
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def ValidationMatrix.guess_blacklists (   samples,
  ver1,
  ver2,
  hlt 
) 
Build a blacklist for each sample accordind to a set of rules

Definition at line 144 of file ValidationMatrix.py.

References add_to_blacklist(), and cond::persistency.search().

Referenced by call_compare_using_files().

144 
145 def guess_blacklists(samples,ver1,ver2,hlt):
146  """Build a blacklist for each sample accordind to a set of rules
147  """
148  blacklists={}
149  for sample in samples:
150  blacklists[sample]="FED@1,AlcaBeamMonitor@1,HLT@1,AlCaReco@1"
151 
152  # HLT
153  if hlt: #HLT
154  blacklists[sample]+=",AlCaEcalPi0@2"
155  if not search("2010+|2011+",ver1):
156  print "We are treating MC files for the HLT"
157  for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
158  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
159  else:
160  print "We are treating Data files for the HLT"
161  # at the moment it does not make sense since hlt is ran already
162 
163  else: #RECO
164  #Monte Carlo
165  if not search("2010+|2011+",ver1):
166  print "We are treating MC files"
167 
168  for pattern,blist in definitions.mc_pattern_blist_pairs:
169  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
170 # print "MC RECO"
171  #print blacklists[sample]
172 
173  # Data
174  else:
175  print "We are treating Data files:"
176  blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"
177  for pattern,blist in definitions.data_pattern_blist_pairs:
178  blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
179 # print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])
180 
181 
182  return blacklists
183 
184 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:224
def ValidationMatrix.guess_params (   ref_filenames,
  test_filenames 
)

Definition at line 63 of file ValidationMatrix.py.

References list(), python.multivaluedict.map(), name2sample(), name2version(), and cond::persistency.search().

Referenced by do_comparisons_threaded().

63 
64 def guess_params(ref_filenames,test_filenames):
65 
66  if len(ref_filenames)*len(test_filenames)==0:
67  print "Empty reference and test filenames lists!"
68  return [],"",""
69 
70  samples=[]
71  ref_versions=[]
72  test_versions=[]
73 
74  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
75 
76  ref_sample=name2sample(ref)
77  ref_version=name2version(ref)
78  test_sample=name2sample(test)
79  test_version=name2version(test)
80 
81  print " ## sample 1: %s vs sample 2: %s"%(ref_sample, test_sample)
82 
83  if ref_sample!=test_sample:
84  print "Files %s and %s do not seem to be relative to the same sample." %(ref, test)
85  # exit(2)
86 
87  # Slightly modify for data
88  if search("20[01]",ref_version)!=None:
89  ref_sample+=ref_version.split("_")[-1]
90  samples.append(ref_sample)
91 
92  # append the versions
93  ref_versions.append(ref_version)
94  test_versions.append(test_version)
95 
96  # Check if ref and test versions are always the same.
97  ref_versions=list(set(ref_versions))
98  test_versions=list(set(test_versions))
99 
100  #for versions in ref_versions,test_versions:
101  #if len(versions)!=1:
102  #print "More than one kind of CMSSW version selected (%s)" %versions
103  #exit(2)
104 
105  cmssw_version1=ref_versions[0]
106  cmssw_version2=test_versions[0]
107 
108  return samples,cmssw_version1,cmssw_version2
109 
110 
111 #-------------------------------------------------------------------------------
cond::persistency::search
std::vector< T >::const_iterator search(const cond::Time_t &val, const std::vector< T > &container)
Definition: IOVProxy.cc:224
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def ValidationMatrix.name2run (   filename)

Definition at line 49 of file ValidationMatrix.py.

References split.

Referenced by name2runskim().

49 
50 def name2run(filename):
51  namebase=os.path.basename(filename)
52  return namebase.split("__")[0].split("_")[2]
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2runskim (   filename)

Definition at line 53 of file ValidationMatrix.py.

References name2run(), name2version(), and split.

53 
54 def name2runskim(filename):
55  run=name2run(filename)
56  skim=name2version(filename).split("_")[-1]
57  # remove skim version
58  if "-v" in skim:
59  skim = skim[:skim.rfind('-v')]
60  return "%s_%s"%(run,skim)
61 
62 #-------------------------------------------------------------------------------
split
double split
Definition: MVATrainer.cc:139
def ValidationMatrix.name2sample (   filename)

Definition at line 41 of file ValidationMatrix.py.

Referenced by guess_params().

41 
42 def name2sample(filename):
43  namebase=os.path.basename(filename)
44  return namebase.split("__")[1]
def ValidationMatrix.name2version (   filename)

Definition at line 45 of file ValidationMatrix.py.

Referenced by call_compare_using_files(), guess_params(), and name2runskim().

45 
46 def name2version(filename):
47  namebase=os.path.basename(filename)
48  return namebase.split("__")[2]

Variable Documentation

string ValidationMatrix.action = "store"

—HASHING—##

– USE CSS files in web access, for stand-alone usage –##

–Blacklist File –##

Definition at line 457 of file ValidationMatrix.py.

string ValidationMatrix.all_samples = ""

Definition at line 442 of file ValidationMatrix.py.

ValidationMatrix.argv = sys.argv

Definition at line 23 of file ValidationMatrix.py.

ValidationMatrix.default = ref_samples,

Definition at line 459 of file ValidationMatrix.py.

string ValidationMatrix.dest = "ref_samples"

Definition at line 458 of file ValidationMatrix.py.

string ValidationMatrix.help = "The samples that act as reference (comma separated list)"

Definition at line 460 of file ValidationMatrix.py.

ValidationMatrix.hlt = False

Definition at line 450 of file ValidationMatrix.py.

string ValidationMatrix.in_dir = ""

Definition at line 445 of file ValidationMatrix.py.

int ValidationMatrix.n_processes = 1

Definition at line 443 of file ValidationMatrix.py.

int ValidationMatrix.n_threads = 1

Definition at line 446 of file ValidationMatrix.py.

string ValidationMatrix.out_dir = ""

Definition at line 444 of file ValidationMatrix.py.

Referenced by HtrXmlPattern.HtrXmlPattern().

tuple ValidationMatrix.parser = OptionParser(usage="usage: %prog [options]")

Definition at line 454 of file ValidationMatrix.py.

string ValidationMatrix.ref_samples = ""

Definition at line 440 of file ValidationMatrix.py.

int ValidationMatrix.run = -1

Definition at line 447 of file ValidationMatrix.py.

string ValidationMatrix.stat_test = "Chi2"

Definition at line 448 of file ValidationMatrix.py.

string ValidationMatrix.test_samples = ""

Definition at line 441 of file ValidationMatrix.py.

float ValidationMatrix.test_threshold = 0.00001

Definition at line 449 of file ValidationMatrix.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5