ValidationMatrix Namespace Reference

Functions
def	add_to_blacklist
def	call_compare_using_files
def	check_root_files
def	count_alive_processes
def	do_comparisons_threaded
def	do_html
def	do_reports
def	get_clean_fileanames
def	get_filenames_from_pool
def	get_roofiles_in_dir
def	guess_blacklists
def	guess_params
def	name2run
def	name2runskim
def	name2sample
def	name2version
Variables
string	action = "store"
string	all_samples = ""
	argv = sys.argv
	default = ref_samples,
string	dest = "ref_samples"
string	help = "The samples that act as reference (comma separated list)"
	hlt = False
string	in_dir = ""
int	n_processes = 1
int	n_threads = 1
string	out_dir = ""
tuple	parser = OptionParser(usage="usage: %prog [options]")
string	ref_samples = ""
int	run = 1
string	stat_test = "Chi2"
string	test_samples = ""
float	test_threshold = 0.00001

---

Function Documentation

def ValidationMatrix::add_to_blacklist	(		blacklist,
			pattern,
			target,
			blist_piece
	)

Definition at line 122 of file ValidationMatrix.py.

                                                             :
  int_pattern=pattern
  int_pattern=pattern.strip()  
  flip_condition=False
  if int_pattern[0]=='!':
    int_pattern=int_pattern[1:]
    flip_condition=True

  condition = search(int_pattern,target)!=None
  if flip_condition:
    condition = not condition

  if condition:
    #print "Found %s in %s" %(pattern,target)
    if blacklist!="": # if not the first, add a comma
      blacklist+=","
    blacklist+=blist_piece
  #else:
    #print "  NOT Found %s in %s" %(pattern,target)
  return blacklist

#-------------------------------------------------------------------------------

def ValidationMatrix::call_compare_using_files

(

args

)

Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 272 of file ValidationMatrix.py.

                                  :
  """Creates shell command to compare two files using compare_using_files.py
  script and calls it."""
  sample, ref_filename, test_filename, options = args
  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
  command = " compare_using_files.py "
  command+= "%s %s " %(ref_filename,test_filename)
  command+= " -C -R "
  if options.do_pngs:
    command+= " -p "
  command+= " -o %s " %sample
  # Change threshold to an experimental and empirical value of 10^-5
  command+= " --specify_run "
  command+= " -t %s " %options.test_threshold
  command+= " -s %s " %options.stat_test

  # Inspect the HLT directories
  if options.hlt:
    command+=" -d HLT "

  if len(blacklists[sample]) >0:
    command+= '-B %s ' %blacklists[sample]
  print "\nExecuting --  %s" %command

  process=Popen(filter(lambda x: len(x)>0,command.split(" ")))
  process.name=sample

#--------------------------------------------------------------------------------

def ValidationMatrix::check_root_files

(

names_list

)

Definition at line 113 of file ValidationMatrix.py.

                                :
  for name in names_list:
    if not name.endswith(".root"):
      print "File %s does not seem to be a rootfile. Please check."
      return False
  return True

#-------------------------------------------------------------------------------

def ValidationMatrix::count_alive_processes

(

p_list

)

Definition at line 267 of file ValidationMatrix.py.

                                 :
  return len(filter(lambda p: p.returncode==None,p_list))

#-------------------------------------------------------------------------------

def ValidationMatrix::do_comparisons_threaded

(

options

)

Definition at line 301 of file ValidationMatrix.py.

00302                                     :
00303 
00304   n_processes= int(options.n_processes)
00305 
00306   ref_filenames=[]
00307   test_filenames=[]
00308   
00309   if len(options.all_samples)>0:
00310     ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)  
00311   else:
00312     ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
00313  
00314   # make the paths absolute
00315   ref_filenames=map(os.path.abspath,ref_filenames)
00316   test_filenames=map(os.path.abspath,test_filenames)
00317   
00318   samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
00319   
00320   if len(samples)==0:
00321     print "No Samples found... Quitting"
00322     return 0
00323   
00324 #  blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)
00325 
00326   # Launch the single comparisons
00327   original_dir=os.getcwd()
00328 
00329   outdir=options.out_dir
00330   if len(outdir)==0:
00331     print "Creating automatic outdir:",
00332     outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
00333     print outdir
00334   if len(options.input_dir)==0:
00335     print "Creating automatic indir:",
00336     options.input_dir=outdir
00337     print options.input_dir
00338   
00339   if not os.path.exists(outdir):
00340     os.mkdir(outdir)
00341   os.chdir(outdir)  
00342   
00343   # adjust the number of threads
00344   n_comparisons=len(ref_filenames)
00345   if n_comparisons < n_processes:
00346     print "Less comparisons than possible processes: reducing n processes to",
00347     n_processes=n_comparisons
00348   #elif n_processes/n_comparisons == 0:
00349     #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",    
00350     #original_nprocesses=n_processes
00351     #first=True
00352     #n_bunches=0
00353     #while first or n_processes > original_nprocesses:
00354       #n_processes=n_comparisons/2
00355       #if n_comparisons%2 !=0:
00356         #n_processes+=1
00357       #first=False
00358       
00359     #print n_processes
00360   print n_processes
00361   
00362   # Test if we treat data
00363   skim_name=""
00364   if search("20[01]",cmssw_version1)!=None:
00365     skim_name=cmssw_version1.split("_")[-1]
00366     
00367   running_subprocesses=[]
00368   process_counter=0
00369   print ref_filenames
00370 
00371   ## Compare all pairs of root files
00372   pool = Pool(n_processes)
00373   args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
00374   pool.map(call_compare_using_files, args_iterable)
00375 
00376   # move the pickles on the top, hack
00377   os.system("mv */*pkl .")
00378   
00379   os.chdir("..")
#-------------------------------------------------------------------------------

def ValidationMatrix::do_html

(

options

)

Definition at line 408 of file ValidationMatrix.py.

                    :

  if options.reports:
    print "Preparing reports for the single files..."
    do_reports(options.input_dir)
  # Do the summary page
  aggregation_rules={}
  aggregation_rules_twiki={}
  # check which aggregation rules are to be used
  if options.hlt:
    print "Aggregating directories according to HLT rules"
    aggregation_rules=definitions.aggr_pairs_dict['HLT']
    aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
  else:
    aggregation_rules=definitions.aggr_pairs_dict['reco']
    aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki)

  # create summary html file
  ofile = open("RelMonSummary.html","w")
  ofile.write(table_html)
  ofile.close()

#-------------------------------------------------------------------------------

def ValidationMatrix::do_reports

(

indir

)

Definition at line 380 of file ValidationMatrix.py.

00381                      :
00382   print indir
00383   os.chdir(indir)
00384   pkl_list=filter(lambda x:".pkl" in x, os.listdir("./"))
00385   running_subprocesses=[]
00386   n_processes=int(options.n_processes)
00387   process_counter=0
00388   for pklfilename in pkl_list:
00389     command = "compare_using_files.py " 
00390     command+= "-R "
00391     if options.do_pngs:
00392       command+= " -p "
00393     command+= "-P %s " %pklfilename
00394     command+= "-o %s " %pklfilename[:-4]
00395     print "Executing %s" %command
00396     process=Popen(filter(lambda x: len(x)>0,command.split(" ")))
00397     process_counter+=1
00398     # add it to the list
00399     running_subprocesses.append(process)   
00400     if process_counter>=n_processes:
00401       process_counter=0
00402       for p in running_subprocesses:
00403         #print "Waiting for %s" %p.name
00404         p.wait()
00405         
00406   os.chdir("..")
00407   
#-------------------------------------------------------------------------------

def ValidationMatrix::get_clean_fileanames	(		ref_samples,
			test_samples
	)

Definition at line 252 of file ValidationMatrix.py.

                                                  :
  # Process the samples starting from the names
  ref_filenames=map(lambda s:s.strip(),ref_samples.split(","))
  test_filenames=map(lambda s:s.strip(),test_samples.split(","))

  if len(ref_filenames)!=len(test_filenames):
    print "The numebr of reference and test files does not seem to be the same. Please check."
    exit(2)

  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
    exit(2)
  return ref_filenames,test_filenames

#-------------------------------------------------------------------------------

def ValidationMatrix::get_filenames_from_pool

(

all_samples

)

Definition at line 198 of file ValidationMatrix.py.

                                        :
  
  # get a list of the files
  files_list=get_roofiles_in_dir(all_samples)
  
  if len(files_list)==0:
    print "Zero files found in directory %s!" %all_samples
    return [],[]
  
  # Are they an even number?
  for name in files_list:
    print "* ",name  
  if len(files_list)%2!=0:
    print "The numbuer of file is not even... Trying to recover a catastrophe."
    
  files_list=make_files_pairs(files_list)
  
  # Try to couple them according to their sample
  ref_filenames=[]
  test_filenames=[]
  #files_list.sort(key=name2version)
  #files_list.sort(key=name2sample) 
  #files_list.sort(key=name2run)
  for iname in xrange(len(files_list)):
    filename=files_list[iname]
    if iname%2==0:
      ref_filenames.append(filename)
    else:
      test_filenames.append(filename)
      
  print "The guess would be the following:"
  for ref,test in zip(ref_filenames,test_filenames):
    refbasedir=os.path.dirname(ref)
    testbasedir=os.path.dirname(test)
    dir_to_print=refbasedir
    if refbasedir!=testbasedir:
      dir_to_print="%s and %s" %(refbasedir,testbasedir)
    print "* Directory: %s " %dir_to_print
    refname=os.path.basename(ref)
    testname=os.path.basename(test)
    print "  o %s" %refname
    print "  o %s" %testname
  
  #is_ok=ask_ok("Is that ok?")
  #if not is_ok:
    #print "Manual input needed then!"
    #exit(2)
      
  
  return ref_filenames,test_filenames
  

#-------------------------------------------------------------------------------

def ValidationMatrix::get_roofiles_in_dir

(

directory

)

Definition at line 189 of file ValidationMatrix.py.

                                  :  
  print directory
  files_list = filter(lambda s: s.endswith(".root"), os.listdir(directory))
  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
  
  return files_list_path
  
#-------------------------------------------------------------------------------  

def ValidationMatrix::guess_blacklists	(		samples,
			ver1,
			ver2,
			hlt
	)

Build a blacklist for each sample accordind to a set of rules

Definition at line 145 of file ValidationMatrix.py.

                                           :
  """Build a blacklist for each sample accordind to a set of rules
  """
  blacklists={}
  for sample in samples:
    blacklists[sample]="FED@1,AlcaBeamMonitor@1,Physics@1,Info@-1,HLT@1,AlCaReco@1"
    
    # HLT
    if hlt: #HLT
      blacklists[sample]+=",AlCaEcalPi0@2"
      if not search("2010+|2011+",ver1):
        print "We are treating MC files for the HLT"
        for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
#          print 'HLT '+pattern
#          print 'HLT '+sample
#          print 'HLT '+blacklists[sample]   
      else:
        print "We are treating Data files for the HLT"    
        # at the moment it does not make sense since hlt is ran already
    
    else: #RECO
      #Monte Carlo
      if not search("2010+|2011+",ver1):
        print "We are treating MC files"        
        
        for pattern,blist in definitions.mc_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
#          print "MC RECO"
          #print blacklists[sample]
          
      # Data
      else:
        print "We are treating Data files:"      
        blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"                                         
        for pattern,blist in definitions.data_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
#         print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])


  return blacklists

#-------------------------------------------------------------------------------  

def ValidationMatrix::guess_params	(		ref_filenames,
			test_filenames
	)

Definition at line 66 of file ValidationMatrix.py.

                                              :
  
  if len(ref_filenames)*len(test_filenames)==0:
    print "Empty reference and test filenames lists!"
    return [],"",""
  
  samples=[]
  ref_versions=[]
  test_versions=[]
    
  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
    
    ref_sample=name2sample(ref)
    ref_version=name2version(ref)
    test_sample=name2sample(test)
    test_version=name2version(test)
          
    if ref_sample!=test_sample:
      print "Files %s and %s do not seem to be relative to the same sample." %(ref, test)
      exit(2)

    # Slightly modify for data
    if search("20[01]",ref_version)!=None:
      ref_sample+=ref_version.split("_")[-1]
    samples.append(ref_sample)
 
    # append the versions
    ref_versions.append(ref_version)
    test_versions.append(test_version)

  # Check if ref and test versions are always the same.
  ref_versions=list(set(ref_versions))
  test_versions=list(set(test_versions))
  
  #for versions in ref_versions,test_versions:
    #if len(versions)!=1:
      #print "More than one kind of CMSSW version selected (%s)" %versions
      #exit(2)  
  
  cmssw_version1=ref_versions[0]
  cmssw_version2=test_versions[0]
  
  return samples,cmssw_version1,cmssw_version2
  

#-------------------------------------------------------------------------------

def ValidationMatrix::name2run

(

filename

)

Definition at line 52 of file ValidationMatrix.py.

                      :
  namebase=os.path.basename(filename)
  return namebase.split("__")[0].split("_")[2]  

def ValidationMatrix::name2runskim

(

filename

)

Definition at line 56 of file ValidationMatrix.py.

                          :
  run=name2run(filename)
  skim=name2version(filename).split("_")[-1]
  # remove skim version
  if "-v" in skim:
    skim = skim[:skim.rfind('-v')]
  return "%s_%s"%(run,skim)

#-------------------------------------------------------------------------------  

def ValidationMatrix::name2sample

(

filename

)

Definition at line 44 of file ValidationMatrix.py.

                         :
  namebase=os.path.basename(filename)
  return namebase.split("__")[1]

def ValidationMatrix::name2version

(

filename

)

Definition at line 48 of file ValidationMatrix.py.

00049                           :
00050   namebase=os.path.basename(filename)
00051   return namebase.split("__")[2]
  

---

Variable Documentation

string ValidationMatrix::action = "store"

Definition at line 453 of file ValidationMatrix.py.

string ValidationMatrix::all_samples = ""

Definition at line 438 of file ValidationMatrix.py.

ValidationMatrix::argv = sys.argv

Definition at line 26 of file ValidationMatrix.py.

ValidationMatrix::default = ref_samples,

Definition at line 455 of file ValidationMatrix.py.

string ValidationMatrix::dest = "ref_samples"

Definition at line 454 of file ValidationMatrix.py.

string ValidationMatrix::help = "The samples that act as reference (comma separated list)"

Definition at line 456 of file ValidationMatrix.py.

ValidationMatrix::hlt = False

Definition at line 446 of file ValidationMatrix.py.

string ValidationMatrix::in_dir = ""

Definition at line 441 of file ValidationMatrix.py.

int ValidationMatrix::n_processes = 1

Definition at line 439 of file ValidationMatrix.py.

int ValidationMatrix::n_threads = 1

Definition at line 442 of file ValidationMatrix.py.

string ValidationMatrix::out_dir = ""

Definition at line 440 of file ValidationMatrix.py.

Referenced by HtrXmlPattern::HtrXmlPattern().

tuple ValidationMatrix::parser = OptionParser(usage="usage: %prog [options]")

Definition at line 450 of file ValidationMatrix.py.

string ValidationMatrix::ref_samples = ""

Definition at line 436 of file ValidationMatrix.py.

int ValidationMatrix::run = 1

Definition at line 443 of file ValidationMatrix.py.

string ValidationMatrix::stat_test = "Chi2"

Definition at line 444 of file ValidationMatrix.py.

string ValidationMatrix::test_samples = ""

Definition at line 437 of file ValidationMatrix.py.

float ValidationMatrix::test_threshold = 0.00001

Definition at line 445 of file ValidationMatrix.py.