ValidationMatrix Namespace Reference

Functions
def	add_to_blacklist (blacklist, pattern, target, blist_piece)
def	call_compare_using_files (args)
def	check_root_files (names_list)
def	count_alive_processes (p_list)
def	do_comparisons_threaded (options)
def	do_html (options, hashing_flag, standalone)
def	do_reports (indir)
def	get_clean_fileanames (ref_samples, test_samples)
def	get_filenames_from_pool (all_samples)
def	get_roofiles_in_dir (directory)
def	guess_blacklists (samples, ver1, ver2, hlt)
def	guess_params (ref_filenames, test_filenames)
def	name2globaltag (filename)
def	name2run (filename)
def	name2runskim (filename)
def	name2sample (filename)
def	name2version (filename)

Variables
	action
	—HASHING—## More...
	all_samples
	args
	argv
	default
	dest
	help
	hlt
	in_dir
	n_processes
	n_threads
	options
	out_dir
	parser
	ref_samples
	run
	stat_test
	test_samples
	test_threshold
	usage

Function Documentation

add_to_blacklist()

def ValidationMatrix.add_to_blacklist	(		blacklist,
			pattern,
			target,
			blist_piece
	)

Definition at line 127 of file ValidationMatrix.py.

References cond::persistency.search().

Referenced by guess_blacklists().

def add_to_blacklist(blacklist, pattern, target, blist_piece):
  int_pattern=pattern
  int_pattern=pattern.strip()  
  flip_condition=False
  if int_pattern[0]=='!':
    int_pattern=int_pattern[1:]
    flip_condition=True

  condition = search(int_pattern,target)!=None
  if flip_condition:
    condition = not condition

  if condition:
    #print "Found %s in %s" %(pattern,target)
    if blacklist!="": # if not the first, add a comma
      blacklist+=","
    blacklist+=blist_piece
  #else:
    #print "  NOT Found %s in %s" %(pattern,target)
  return blacklist

#-------------------------------------------------------------------------------

call_compare_using_files()

def ValidationMatrix.call_compare_using_files

(

args

)

Creates shell command to compare two files using compare_using_files.py
script and calls it.

Definition at line 274 of file ValidationMatrix.py.

References guess_blacklists(), name2globaltag(), name2version(), and print().

def call_compare_using_files(args):
  """Creates shell command to compare two files using compare_using_files.py
  script and calls it."""
  sample, ref_filename, test_filename, options = args
  gt = name2globaltag(ref_filename)
  blacklists=guess_blacklists([sample],name2version(ref_filename),name2version(test_filename),options.hlt)
  command = " compare_using_files.py "
  command+= "%s %s " %(ref_filename,test_filename)
  command+= " -C -R "
  if options.do_pngs:
    command+= " -p "
  command+= " -o %s_%s " %(sample, gt)
  # Change threshold to an experimental and empirical value of 10^-5
  command+= " --specify_run "
  if options.stat_test in ["Bin2Bin", "BinToBin"]:
    options.test_threshold = 0.9999
  command+= " -t %s " %options.test_threshold
  command+= " -s %s " %options.stat_test

  # Inspect the HLT directories
  if options.hlt:
    command+=" -d HLT "
  
  if options.hash_name:
    command += " --hash_name "

  if options.blacklist_file:
    command += " --use_black_file "

  if options.standalone:
    command += " --standalone "
  if len(blacklists[sample]) >0:
    command+= '-B %s ' %blacklists[sample]
  print("\nExecuting --  %s" %command)

  process=call([x for x in command.split(" ") if len(x)>0])
  return process
  

#--------------------------------------------------------------------------------

check_root_files()

def ValidationMatrix.check_root_files

(

names_list

)

Definition at line 118 of file ValidationMatrix.py.

References print().

Referenced by get_clean_fileanames().

def check_root_files(names_list):
  for name in names_list:
    if not name.endswith(".root"):
      print("File %s does not seem to be a rootfile. Please check.")
      return False
  return True

#-------------------------------------------------------------------------------

count_alive_processes()

def ValidationMatrix.count_alive_processes

(

p_list

)

Definition at line 269 of file ValidationMatrix.py.

def count_alive_processes(p_list):
  return len([p for p in p_list if p.returncode==None])

#-------------------------------------------------------------------------------

do_comparisons_threaded()

def ValidationMatrix.do_comparisons_threaded

(

options

)

Definition at line 315 of file ValidationMatrix.py.

References get_clean_fileanames(), get_filenames_from_pool(), guess_params(), createfilelist.int, genParticles_cff.map, print(), cond::persistency.search(), and reco.zip().

def do_comparisons_threaded(options):

  n_processes= int(options.n_processes)

  ref_filenames=[]
  test_filenames=[]
  
  if len(options.all_samples)>0:
    ref_filenames,test_filenames=get_filenames_from_pool(options.all_samples)  
  else:
    ref_filenames,test_filenames=get_clean_fileanames(options.ref_samples,options.test_samples)
 
  # make the paths absolute
  ref_filenames=list(map(os.path.abspath,ref_filenames))
  test_filenames=list(map(os.path.abspath,test_filenames))
  
  samples,cmssw_version1,cmssw_version2=guess_params(ref_filenames,test_filenames)
  
  if len(samples)==0:
    print("No Samples found... Quitting")
    return 0
  
#  blacklists=guess_blacklists(samples,cmssw_version1,cmssw_version2,options.hlt)

  # Launch the single comparisons
  original_dir=os.getcwd()

  outdir=options.out_dir
  if len(outdir)==0:
    print("Creating automatic outdir:", end=' ')
    outdir="%sVS%s" %(cmssw_version1,cmssw_version2)
    print(outdir)
  if len(options.input_dir)==0:
    print("Creating automatic indir:", end=' ')
    options.input_dir=outdir
    print(options.input_dir)
  
  if not os.path.exists(outdir):
    os.mkdir(outdir)
  os.chdir(outdir)  
  
  # adjust the number of threads
  n_comparisons=len(ref_filenames)
  if n_comparisons < n_processes:
    print("Less comparisons than possible processes: reducing n processes to", end=' ')
    n_processes=n_comparisons
  #elif n_processes/n_comparisons == 0:
    #print "More comparisons than possible processes, can be done in N rounds: reducing n processes to",    
    #original_nprocesses=n_processes
    #first=True
    #n_bunches=0
    #while first or n_processes > original_nprocesses:
      #n_processes=n_comparisons/2
      #if n_comparisons%2 !=0:
        #n_processes+=1
      #first=False
      
    #print n_processes
  #print n_processes
  
  # Test if we treat data
  skim_name=""
  if search("20[01]",cmssw_version1)!=None:
    skim_name=cmssw_version1.split("_")[-1]
    
  running_subprocesses=[]
  process_counter=0
  #print ref_filenames

  
  pool = Pool(n_processes)
  args_iterable = [list(args) + [options] for args in zip(samples, ref_filenames, test_filenames)]
  pool.map(call_compare_using_files, args_iterable) 
  # move the pickles on the top, hack
  os.system("mv */*pkl .")
  
  os.chdir("..")
#-------------------------------------------------------------------------------

do_html()

def ValidationMatrix.do_html	(		options,
			hashing_flag,
			standalone
	)

Definition at line 421 of file ValidationMatrix.py.

References do_reports(), directories2html.make_summary_table(), and print().

def do_html(options, hashing_flag, standalone):

  if options.reports:
    print("Preparing reports for the single files...")
    do_reports(options.input_dir)
  # Do the summary page
  aggregation_rules={}
  aggregation_rules_twiki={}
  # check which aggregation rules are to be used
  if options.hlt:
    print("Aggregating directories according to HLT rules")
    aggregation_rules=definitions.aggr_pairs_dict['HLT']
    aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['HLT']
  else:
    aggregation_rules=definitions.aggr_pairs_dict['reco']
    aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict['reco']
  table_html = make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone)

  # create summary html file
  ofile = open("RelMonSummary.html","w")
  ofile.write(table_html)
  ofile.close()

#-------------------------------------------------------------------------------

do_reports()

def ValidationMatrix.do_reports

(

indir

)

Definition at line 393 of file ValidationMatrix.py.

References createfilelist.int, and print().

Referenced by do_html().

def do_reports(indir):
  #print indir
  os.chdir(indir)
  pkl_list=[x for x in os.listdir("./") if ".pkl" in x]
  running_subprocesses=[]
  n_processes=int(options.n_processes)
  process_counter=0
  for pklfilename in pkl_list:
    command = "compare_using_files.py " 
    command+= "-R "
    if options.do_pngs:
      command+= " -p "
    command+= "-P %s " %pklfilename
    command+= "-o %s " %pklfilename[:-4]
    print("Executing %s" %command)
    process=call([x for x in command.split(" ") if len(x)>0])
    process_counter+=1
    # add it to the list
    running_subprocesses.append(process)   
    if process_counter>=n_processes:
      process_counter=0
      for p in running_subprocesses:
        #print "Waiting for %s" %p.name
        p.wait()
        
  os.chdir("..")
  
#-------------------------------------------------------------------------------

get_clean_fileanames()

def ValidationMatrix.get_clean_fileanames	(		ref_samples,
			test_samples
	)

Definition at line 254 of file ValidationMatrix.py.

References check_root_files(), beamvalidation.exit(), genParticles_cff.map, and print().

Referenced by do_comparisons_threaded().

def get_clean_fileanames(ref_samples,test_samples):
  # Process the samples starting from the names
  ref_filenames=list(map(lambda s:s.strip(),ref_samples.split(",")))
  test_filenames=list(map(lambda s:s.strip(),test_samples.split(",")))

  if len(ref_filenames)!=len(test_filenames):
    print("The numebr of reference and test files does not seem to be the same. Please check.")
    exit(2)

  if not (check_root_files(ref_filenames) and check_root_files(test_filenames)):
    exit(2)
  return ref_filenames,test_filenames

#-------------------------------------------------------------------------------

get_filenames_from_pool()

def ValidationMatrix.get_filenames_from_pool

(

all_samples

)

Definition at line 200 of file ValidationMatrix.py.

References get_roofiles_in_dir(), utils.make_files_pairs(), print(), isotrackApplyRegressor.range, and reco.zip().

Referenced by do_comparisons_threaded().

def get_filenames_from_pool(all_samples):
  
  # get a list of the files
  files_list=get_roofiles_in_dir(all_samples)
  
  if len(files_list)==0:
    print("Zero files found in directory %s!" %all_samples)
    return [],[]
  
  # Are they an even number?
  for name in files_list:
    print("* ",name)  
  if len(files_list)%2!=0:
    print("The numbuer of file is not even... Trying to recover a catastrophe.")
    
  files_list=make_files_pairs(files_list)
  
  # Try to couple them according to their sample
  ref_filenames=[]
  test_filenames=[]
  #files_list.sort(key=name2version)
  #files_list.sort(key=name2sample) 
  #files_list.sort(key=name2run)
  for iname in range(len(files_list)):
    filename=files_list[iname]
    if iname%2==0:
      ref_filenames.append(filename)
    else:
      test_filenames.append(filename)
      
  print("The guess would be the following:")
  for ref,test in zip(ref_filenames,test_filenames):
    refbasedir=os.path.dirname(ref)
    testbasedir=os.path.dirname(test)
    dir_to_print=refbasedir
    if refbasedir!=testbasedir:
      dir_to_print="%s and %s" %(refbasedir,testbasedir)
    print("* Directory: %s " %dir_to_print)
    refname=os.path.basename(ref)
    testname=os.path.basename(test)
    print("  o %s" %refname)
    print("  o %s" %testname)
  
  #is_ok=ask_ok("Is that ok?")
  #if not is_ok:
    #print "Manual input needed then!"
    #exit(2)
      
  
  return ref_filenames,test_filenames
  

#-------------------------------------------------------------------------------

get_roofiles_in_dir()

def ValidationMatrix.get_roofiles_in_dir

(

directory

)

Definition at line 191 of file ValidationMatrix.py.

References genParticles_cff.map, and print().

Referenced by get_filenames_from_pool().

def get_roofiles_in_dir(directory):  
  print(directory)
  files_list = [s for s in os.listdir(directory) if s.endswith(".root")]
  files_list_path=map(lambda s: os.path.join(directory,s), files_list)
  
  return files_list_path
  
#-------------------------------------------------------------------------------  

guess_blacklists()

def ValidationMatrix.guess_blacklists	(		samples,
			ver1,
			ver2,
			hlt
	)

Build a blacklist for each sample accordind to a set of rules

Definition at line 150 of file ValidationMatrix.py.

References add_to_blacklist(), print(), and cond::persistency.search().

Referenced by call_compare_using_files().

def guess_blacklists(samples,ver1,ver2,hlt):
  """Build a blacklist for each sample accordind to a set of rules
  """
  blacklists={}
  for sample in samples:
    blacklists[sample]="FED@1,AlcaBeamMonitor@1,HLT@1,AlCaReco@1"
    
    # HLT
    if hlt: #HLT
      blacklists[sample]+=",AlCaEcalPi0@2"
      if not search("2010+|2011+|2012+|2015+",ver1):
        print("We are treating MC files for the HLT")
        for pattern,blist in definitions.hlt_mc_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
      else:
        print("We are treating Data files for the HLT")    
        # at the moment it does not make sense since hlt is ran already
    
    else: #RECO
      #Monte Carlo
      if not search("2010+|2011+|2012+",ver1):
        print("We are treating MC files")        
        
        for pattern,blist in definitions.mc_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,sample,blist)
#          print "MC RECO"
          #print blacklists[sample]
          
      # Data
      else:
        print("We are treating Data files:")      
        blacklists[sample]+=",By__Lumi__Section@-1,AlCaReco@1"                                         
        for pattern,blist in definitions.data_pattern_blist_pairs:
          blacklists[sample]=add_to_blacklist(blacklists[sample],pattern,ver1,blist)
#         print "DATA RECO: %s %s %s -->%s" %( ver1, pattern, blist, blacklists[sample])


  return blacklists

#-------------------------------------------------------------------------------  

guess_params()

def ValidationMatrix.guess_params	(		ref_filenames,
			test_filenames
	)

Definition at line 69 of file ValidationMatrix.py.

References genParticles_cff.map, name2sample(), name2version(), print(), cond::persistency.search(), and reco.zip().

Referenced by do_comparisons_threaded().

def guess_params(ref_filenames,test_filenames):
  
  if len(ref_filenames)*len(test_filenames)==0:
    print("Empty reference and test filenames lists!")
    return [],"",""
  
  samples=[]
  ref_versions=[]
  test_versions=[]
    
  for ref, test in zip(map(os.path.basename,ref_filenames),map(os.path.basename,test_filenames)):
    
    ref_sample=name2sample(ref)
    ref_version=name2version(ref)
    test_sample=name2sample(test)
    test_version=name2version(test)

    print("  ## sample 1: %s vs sample 2: %s"%(ref_sample, test_sample))
          
    if ref_sample!=test_sample:
      print("Files %s and %s do not seem to be relative to the same sample." %(ref, test))
    #  exit(2)

    # Slightly modify for data
    if search("20[01]",ref_version)!=None:
      ref_sample+=ref_version.split("_")[-1]
    samples.append(ref_sample)
 
    # append the versions
    ref_versions.append(ref_version)
    test_versions.append(test_version)

  # Check if ref and test versions are always the same.
  ref_versions=list(set(ref_versions))
  test_versions=list(set(test_versions))
  
  #for versions in ref_versions,test_versions:
    #if len(versions)!=1:
      #print "More than one kind of CMSSW version selected (%s)" %versions
      #exit(2)  
  
  cmssw_version1=ref_versions[0]
  cmssw_version2=test_versions[0]
  
  return samples,cmssw_version1,cmssw_version2
  

#-------------------------------------------------------------------------------

name2globaltag()

def ValidationMatrix.name2globaltag

(

filename

)

Definition at line 63 of file ValidationMatrix.py.

References submitPVValidationJobs.split().

Referenced by call_compare_using_files().

def name2globaltag(filename):
  namebase = os.path.basename(filename)
  return namebase.split("__")[2].split("-")[1] #returns GT from file basename

#-------------------------------------------------------------------------------  

name2run()

def ValidationMatrix.name2run

(

filename

)

Definition at line 51 of file ValidationMatrix.py.

References submitPVValidationJobs.split().

Referenced by name2runskim().

def name2run(filename):
  namebase=os.path.basename(filename)
  return namebase.split("__")[0].split("_")[2]  

name2runskim()

def ValidationMatrix.name2runskim

(

filename

)

Definition at line 55 of file ValidationMatrix.py.

References name2run(), name2version(), and submitPVValidationJobs.split().

def name2runskim(filename):
  run=name2run(filename)
  skim=name2version(filename).split("_")[-1]
  # remove skim version
  if "-v" in skim:
    skim = skim[:skim.rfind('-v')]
  return "%s_%s"%(run,skim)

name2sample()

def ValidationMatrix.name2sample

(

filename

)

Definition at line 43 of file ValidationMatrix.py.

Referenced by guess_params().

def name2sample(filename):
  namebase=os.path.basename(filename)
  return namebase.split("__")[1]

name2version()

def ValidationMatrix.name2version

(

filename

)

Definition at line 47 of file ValidationMatrix.py.

Referenced by call_compare_using_files(), guess_params(), and name2runskim().

def name2version(filename):
  namebase=os.path.basename(filename)
  return namebase.split("__")[2]
  

Variable Documentation

action

ValidationMatrix.action

—HASHING—##

– USE CSS files in web access, for stand-alone usage –##

–Blacklist File –##

Definition at line 466 of file ValidationMatrix.py.

all_samples

ValidationMatrix.all_samples

Definition at line 451 of file ValidationMatrix.py.

args

ValidationMatrix.args

Definition at line 555 of file ValidationMatrix.py.

argv

ValidationMatrix.argv

Definition at line 25 of file ValidationMatrix.py.

default

ValidationMatrix.default

Definition at line 468 of file ValidationMatrix.py.

dest

ValidationMatrix.dest

Definition at line 467 of file ValidationMatrix.py.

help

ValidationMatrix.help

Definition at line 469 of file ValidationMatrix.py.

hlt

ValidationMatrix.hlt

Definition at line 459 of file ValidationMatrix.py.

Referenced by DQMHcalIsolatedBunchAlCaReco.analyze(), StudyTriggerHLT.analyze(), IsoTrackCalibration.analyze(), HcalIsoTrkSimAnalyzer.analyze(), StudyCaloResponse.analyze(), HcalIsoTrkAnalyzer.analyze(), IsoTrig.analyze(), LumiCalculator.beginRun(), AlCaIsolatedBunchSelector.filter(), AlCaIsoTracksProducerFilter.filter(), AlCaHBHEMuonFilter.filter(), AlCaLowPUHBHEMuonFilter.filter(), AlCaHEMuonFilter.filter(), AlCaIsoTracksFilter.filter(), main(), edm.operator<<(), AlCaHcalHBHEMuonProducer.produce(), AlCaHcalIsotrkProducer.produce(), and AlCaIsoTracksProducer.select().

in_dir

ValidationMatrix.in_dir

Definition at line 454 of file ValidationMatrix.py.

n_processes

ValidationMatrix.n_processes

Definition at line 452 of file ValidationMatrix.py.

n_threads

ValidationMatrix.n_threads

Definition at line 455 of file ValidationMatrix.py.

options

ValidationMatrix.options

Definition at line 555 of file ValidationMatrix.py.

out_dir

ValidationMatrix.out_dir

Definition at line 453 of file ValidationMatrix.py.

parser

ValidationMatrix.parser

Definition at line 463 of file ValidationMatrix.py.

ref_samples

ValidationMatrix.ref_samples

Definition at line 449 of file ValidationMatrix.py.

run

ValidationMatrix.run

Definition at line 456 of file ValidationMatrix.py.

stat_test

ValidationMatrix.stat_test

Definition at line 457 of file ValidationMatrix.py.

test_samples

ValidationMatrix.test_samples

Definition at line 450 of file ValidationMatrix.py.

test_threshold

ValidationMatrix.test_threshold

Definition at line 458 of file ValidationMatrix.py.

usage

ValidationMatrix.usage

Definition at line 463 of file ValidationMatrix.py.