15 from optparse
import OptionParser
21 from subprocess
import call,PIPE
22 from multiprocessing
import Pool
28 if os.environ.has_key(
"RELMON_SA"):
29 import definitions
as definitions
30 from dqm_interfaces
import DirWalkerFile,string2blacklist,DirWalkerFile_thread_wrapper
31 from dirstructure
import Directory
32 from directories2html
import directory2html,make_summary_table
33 from utils
import ask_ok, unpickler, make_files_pairs
36 from Utilities.RelMon.dqm_interfaces
import DirWalkerFile,string2blacklist,DirWalkerFile_thread_wrapper
37 from Utilities.RelMon.dirstructure
import Directory
38 from Utilities.RelMon.directories2html
import directory2html,make_summary_table
39 from Utilities.RelMon.utils
import ask_ok, unpickler, make_files_pairs
45 namebase=os.path.basename(filename)
46 return namebase.split(
"__")[1]
49 namebase=os.path.basename(filename)
50 return namebase.split(
"__")[2]
53 namebase=os.path.basename(filename)
54 return namebase.split(
"__")[0].
split(
"_")[2]
61 skim = skim[:skim.rfind(
'-v')]
62 return "%s_%s"%(run,skim)
68 if len(ref_filenames)*len(test_filenames)==0:
69 print "Empty reference and test filenames lists!"
76 for ref, test
in zip(
map(os.path.basename,ref_filenames),
map(os.path.basename,test_filenames)):
83 if ref_sample!=test_sample:
84 print "Files %s and %s do not seem to be relative to the same sample." %(ref, test)
88 if search(
"20[01]",ref_version)!=
None:
89 ref_sample+=ref_version.split(
"_")[-1]
90 samples.append(ref_sample)
93 ref_versions.append(ref_version)
94 test_versions.append(test_version)
97 ref_versions=
list(
set(ref_versions))
98 test_versions=
list(
set(test_versions))
105 cmssw_version1=ref_versions[0]
106 cmssw_version2=test_versions[0]
108 return samples,cmssw_version1,cmssw_version2
114 for name
in names_list:
115 if not name.endswith(
".root"):
116 print "File %s does not seem to be a rootfile. Please check."
124 int_pattern=pattern.strip()
126 if int_pattern[0]==
'!':
127 int_pattern=int_pattern[1:]
130 condition =
search(int_pattern,target)!=
None
132 condition =
not condition
138 blacklist+=blist_piece
146 """Build a blacklist for each sample accordind to a set of rules
149 for sample
in samples:
150 blacklists[sample]=
"FED@1,AlcaBeamMonitor@1,Physics@1,Info@-1,HLT@1,AlCaReco@1"
154 blacklists[sample]+=
",AlCaEcalPi0@2"
155 if not search(
"2010+|2011+",ver1):
156 print "We are treating MC files for the HLT"
157 for pattern,blist
in definitions.hlt_mc_pattern_blist_pairs:
158 blacklists[sample]=
add_to_blacklist(blacklists[sample],pattern,sample,blist)
163 print "We are treating Data files for the HLT"
168 if not search(
"2010+|2011+",ver1):
169 print "We are treating MC files"
171 for pattern,blist
in definitions.mc_pattern_blist_pairs:
172 blacklists[sample]=
add_to_blacklist(blacklists[sample],pattern,sample,blist)
178 print "We are treating Data files:"
179 blacklists[sample]+=
",By__Lumi__Section@-1,AlCaReco@1"
180 for pattern,blist
in definitions.data_pattern_blist_pairs:
191 files_list =
filter(
lambda s: s.endswith(
".root"), os.listdir(directory))
192 files_list_path=
map(
lambda s: os.path.join(directory,s), files_list)
194 return files_list_path
203 if len(files_list)==0:
204 print "Zero files found in directory %s!" %all_samples
208 for name
in files_list:
210 if len(files_list)%2!=0:
211 print "The numbuer of file is not even... Trying to recover a catastrophe."
221 for iname
in xrange(len(files_list)):
222 filename=files_list[iname]
224 ref_filenames.append(filename)
226 test_filenames.append(filename)
228 print "The guess would be the following:"
229 for ref,test
in zip(ref_filenames,test_filenames):
230 refbasedir=os.path.dirname(ref)
231 testbasedir=os.path.dirname(test)
232 dir_to_print=refbasedir
233 if refbasedir!=testbasedir:
234 dir_to_print=
"%s and %s" %(refbasedir,testbasedir)
235 print "* Directory: %s " %dir_to_print
236 refname=os.path.basename(ref)
237 testname=os.path.basename(test)
238 print " o %s" %refname
239 print " o %s" %testname
247 return ref_filenames,test_filenames
254 ref_filenames=
map(
lambda s:s.strip(),ref_samples.split(
","))
255 test_filenames=
map(
lambda s:s.strip(),test_samples.split(
","))
257 if len(ref_filenames)!=len(test_filenames):
258 print "The numebr of reference and test files does not seem to be the same. Please check."
263 return ref_filenames,test_filenames
268 return len(
filter(
lambda p: p.returncode==
None,p_list))
273 """Creates shell command to compare two files using compare_using_files.py
274 script and calls it."""
275 sample, ref_filename, test_filename, options = args
277 command =
" compare_using_files.py "
278 command+=
"%s %s " %(ref_filename,test_filename)
282 command+=
" -o %s " %sample
284 command+=
" --specify_run "
285 command+=
" -t %s " %options.test_threshold
286 command+=
" -s %s " %options.stat_test
292 if options.hash_name:
293 command +=
" --hash_name "
295 if options.blacklist_file:
296 command +=
" --use_black_file "
298 if len(blacklists[sample]) >0:
299 command+=
'-B %s ' %blacklists[sample]
300 print "\nExecuting -- %s" %command
302 process=call(
filter(
lambda x: len(x)>0,command.split(
" ")))
310 n_processes= int(options.n_processes)
315 if len(options.all_samples)>0:
321 ref_filenames=
map(os.path.abspath,ref_filenames)
322 test_filenames=
map(os.path.abspath,test_filenames)
324 samples,cmssw_version1,cmssw_version2=
guess_params(ref_filenames,test_filenames)
327 print "No Samples found... Quitting"
333 original_dir=os.getcwd()
335 outdir=options.out_dir
337 print "Creating automatic outdir:",
338 outdir=
"%sVS%s" %(cmssw_version1,cmssw_version2)
340 if len(options.input_dir)==0:
341 print "Creating automatic indir:",
342 options.input_dir=outdir
343 print options.input_dir
345 if not os.path.exists(outdir):
350 n_comparisons=len(ref_filenames)
351 if n_comparisons < n_processes:
352 print "Less comparisons than possible processes: reducing n processes to",
353 n_processes=n_comparisons
370 if search(
"20[01]",cmssw_version1)!=
None:
371 skim_name=cmssw_version1.split(
"_")[-1]
373 running_subprocesses=[]
378 pool = Pool(n_processes)
379 args_iterable = [
list(args) + [options]
for args
in zip(samples, ref_filenames, test_filenames)]
380 pool.map(call_compare_using_files, args_iterable)
382 os.system(
"mv */*pkl .")
389 pkl_list=
filter(
lambda x:
".pkl" in x, os.listdir(
"./"))
390 running_subprocesses=[]
391 n_processes=int(options.n_processes)
393 for pklfilename
in pkl_list:
394 command =
"compare_using_files.py "
398 command+=
"-P %s " %pklfilename
399 command+=
"-o %s " %pklfilename[:-4]
400 print "Executing %s" %command
401 process=call(
filter(
lambda x: len(x)>0,command.split(
" ")))
404 running_subprocesses.append(process)
405 if process_counter>=n_processes:
407 for p
in running_subprocesses:
417 print "Preparing reports for the single files..."
421 aggregation_rules_twiki={}
424 print "Aggregating directories according to HLT rules"
425 aggregation_rules=definitions.aggr_pairs_dict[
'HLT']
426 aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict[
'HLT']
428 aggregation_rules=definitions.aggr_pairs_dict[
'reco']
429 aggregation_rules_twiki=definitions.aggr_pairs_twiki_dict[
'reco']
430 table_html =
make_summary_table(options.input_dir,aggregation_rules,aggregation_rules_twiki, hashing_flag)
433 ofile = open(
"RelMonSummary.html",
"w")
434 ofile.write(table_html)
439 if __name__ ==
"__main__":
451 test_threshold=0.00001
456 parser = OptionParser(usage=
"usage: %prog [options]")
458 parser.add_option(
"-R",
"--ref_samples ",
462 help=
"The samples that act as reference (comma separated list)")
464 parser.add_option(
"-T",
"--test_samples",
467 default=test_samples,
468 help=
"The samples to be tested (comma separated list)")
470 parser.add_option(
"-a",
"--all_samples",
474 help=
"EXPERIMENTAL: Try to sort all samples selected (wildacrds) and organise a comparison")
476 parser.add_option(
"-o",
"--out_dir",
480 help=
"The outdir other than <Version1>VS<Version2>")
482 parser.add_option(
"-p",
"--do_pngs",
486 help=
"EXPERIMENTAL!!! Do the pngs of the comparison (takes 50%% of the total running time) \n(default is %s)" %
False)
488 parser.add_option(
"-r",
"--run ",
492 help=
"The run to be checked \n(default is %s)" %run)
494 parser.add_option(
"-t",
"--test_threshold",
496 dest=
"test_threshold",
497 default=test_threshold,
498 help=
"Threshold for the statistical test \n(default is %s)" %test_threshold)
500 parser.add_option(
"-s",
"--stat_test",
504 help=
"Statistical test (KS or Chi2) \n(default is %s)" %stat_test)
506 parser.add_option(
"-N",
"--numberOfProcesses",
510 help=
"Number of parallel processes to be run. Be Polite! \n(default is %s)" %n_processes)
512 parser.add_option(
"--HLT",
516 help=
"Analyse HLT histograms\n(default is %s)" %hlt)
518 parser.add_option(
"-i",
"--input_dir",
522 help=
"Input directory for html creation \n(default is %s)" %in_dir)
524 parser.add_option(
"--reports",
528 help=
"Do the reports for the pickles \n(default is %s)" %in_dir)
530 parser.add_option(
"--hash_name",
534 help=
"Set if you want to minimize & hash the output HTML files.")
536 parser.add_option(
"--use_black_file",
538 dest=
"blacklist_file",
540 help=
"Use a black list file of histograms located @ /RelMon/data")
542 (options, args) = parser.parse_args()
544 if len(options.test_samples)*len(options.ref_samples)+len(options.all_samples)==0
and len(options.input_dir)==0:
545 print "No samples given as input."
549 if len(options.all_samples)>0
or (len(options.ref_samples)*len(options.test_samples)>0):
551 if len(options.input_dir)>0:
552 do_html(options, options.hash_name)
def get_filenames_from_pool
def count_alive_processes
def do_comparisons_threaded
def call_compare_using_files
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
void set(const std::string &name, int value)
set the flag, with a run-time name