Functions | |
def | checkSteps |
def | determineNewProfile |
def | expandHyphens |
def | getFstOccur |
def | getLstIndex |
def | getProfileArray |
def | getSteps |
def | init_vars |
def | main |
def | optionparse |
def | pythonFragment |
def | setInputFile |
def | setOutputFileOption |
def | setupProgramParameters |
def | writeCommands |
def | writeCommandsToReport |
def | writePrerequisteSteps |
def | writeReportFileHeader |
def | writeStepHead |
def | writeUnprofiledSteps |
Variables | |
list | AfterPileUpSteps = [] |
list | AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"] |
string | cmsDriver = 'cmsDriver.py' |
debug = False | |
list | DEF_STEPS = ['GEN,SIM', 'DIGI'] |
tuple | hypreg = re.compile('-') |
dictionary | Profiler |
tuple | THIS_PROG_NAME = os.path.basename(sys.argv[0]) |
Global variables. |
def cmsRelvalreportInput::checkSteps | ( | steps | ) |
Checks user steps <steps> for order against steps defined in AllSteps list. If they are not, prints a warning and exits. No return value.
Definition at line 75 of file cmsRelvalreportInput.py.
00076 : 00077 '''Checks user steps <steps> for order against steps defined in AllSteps list. 00078 If they are not, prints a warning and exits. 00079 No return value.''' 00080 00081 #print steps 00082 #if ":" in steps: 00083 # steps.split( 00084 idx = -2 00085 lstidx = -2 00086 for step in steps: 00087 astep = step 00088 split = [] 00089 #Added special workflow for GEN-FASTSIM, bypass the check of step order: 00090 if step=="GEN,FASTSIM": 00091 pass 00092 else: 00093 if "-" in step: 00094 split = astep.split("-") 00095 astep = split[0].split(":")[0] 00096 # print "This is astep:%s"%astep 00097 #idx =AllSteps.index("GEN,SIM") 00098 #print idx 00099 #print astep 00100 idx = AllSteps.index(astep.split(":")[0]) 00101 if not ( idx == -2 ): 00102 if lstidx > idx: 00103 print "ERROR: Your user defined steps are not in a valid order" 00104 sys.exit() 00105 lstidx = idx 00106 if "-" in step: 00107 lstidx = AllSteps.index(split[1].split(":")[0]) 00108
def cmsRelvalreportInput::determineNewProfile | ( | step, | |
Profile, | |||
SavedProfile | |||
) |
Definition at line 428 of file cmsRelvalreportInput.py.
def cmsRelvalreportInput::expandHyphens | ( | step | ) |
Definition at line 260 of file cmsRelvalreportInput.py.
00261 : 00262 newsteps = [] 00263 # for step in steps: 00264 if "-" in step: 00265 hypsteps = step.split(r"-") 00266 #if not (len(hypsteps) == 2): 00267 # print "ERROR: Could not parse usersteps. You have too many hypens between commas" 00268 # sys.exit() 00269 #el 00270 if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)): 00271 #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps): 00272 print "ERROR: One of the steps you defined is invalid" 00273 sys.exit() 00274 else: 00275 if (hypsteps[0] == hypsteps[1]): 00276 print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring" 00277 newsteps.append(hypsteps[0]) 00278 else: 00279 newsteps.append(hypsteps[0]) 00280 00281 srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1 00282 for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])): 00283 astep = AllSteps[n] 00284 if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))): 00285 #break 00286 astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0] 00287 if astep==hypsteps[-1].split(":")[0]: 00288 break 00289 newsteps.append(astep) 00290 newsteps.append(hypsteps[-1]) 00291 else: 00292 if not (step.split(":")[0] in AllSteps): 00293 print "ERROR: One of the steps you defined is invalid" 00294 sys.exit() 00295 else: 00296 newsteps.append(step) 00297 00298 return newsteps
def cmsRelvalreportInput::getFstOccur | ( | item, | |
list | |||
) |
Definition at line 66 of file cmsRelvalreportInput.py.
def cmsRelvalreportInput::getLstIndex | ( | item, | |
list | |||
) |
Definition at line 69 of file cmsRelvalreportInput.py.
def cmsRelvalreportInput::getProfileArray | ( | ProfileCode | ) |
Definition at line 376 of file cmsRelvalreportInput.py.
00377 : 00378 00379 Profile = [] 00380 00381 # The allowed profiles are: 00382 00383 AllowedProfile = [ 00384 'TimingReport', 00385 'TimeReport', 00386 'SimpleMemReport', 00387 'EdmSize', 00388 'IgProfperf', 00389 'IgProfMemTotal', 00390 'IgProfMemLive', 00391 #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there) 00392 'IgProfMemAnalyse', 00393 'valgrind', 00394 'memcheck_valgrind', 00395 'None', 00396 ] 00397 00398 if _noprof: 00399 Profile.append(AllowedProfile[-1]) 00400 else: 00401 #FIXME: 00402 #Horrible code! 00403 for i in range(10): 00404 if str(i) in ProfileCode: 00405 firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \ 00406 or (i == 1 and str(2) in ProfileCode) 00407 secCase = (i==4 and str(7) in ProfileCode) \ 00408 or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \ 00409 or (i == 6 and str(7) in ProfileCode) 00410 00411 if firstCase or secCase: 00412 Profile.append(AllowedProfile[i] + ' @@@ reuse') 00413 #Here's the hack: 00414 if i==6: #i.e. IgProfMemLive 00415 Profile.append("IgProfMemMax @@@ reuse") 00416 else: 00417 Profile.append(AllowedProfile[i]) 00418 if i==6: #i.e. IgProfMemLive 00419 Profile.append("IgProfMemMax") 00420 00421 return Profile
def cmsRelvalreportInput::getSteps | ( | userSteps | ) |
Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps. Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself). Returns a list of steps <steps>.
Definition at line 109 of file cmsRelvalreportInput.py.
00110 : 00111 '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps. 00112 Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself). 00113 Returns a list of steps <steps>.''' 00114 #Add handling of ":" in steps 00115 #print userSteps 00116 # Then split the user steps into "steps" 00117 gsreg = re.compile('GEN-SIM') 00118 greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT 00119 gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow 00120 StepsTokens = userSteps.split(",") 00121 steps = [] 00122 for astep in StepsTokens: 00123 00124 # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO) 00125 # from using the "-" to using the "," to match cmsDriver.py convention 00126 00127 if gsreg.search(astep): 00128 astep = gsreg.sub(r"GEN,SIM", astep) 00129 elif gfsreg.search(astep): 00130 astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups 00131 elif greg.search(astep): 00132 astep = greg.sub(r"GEN,SIM", astep) 00133 00134 00135 #print astep 00136 # Finally collect all the steps into the steps list: 00137 00138 steps.append(astep) 00139 00140 #steps = expandHypens(steps) 00141 checkSteps(steps) 00142 return steps
def cmsRelvalreportInput::init_vars | ( | ) |
Definition at line 340 of file cmsRelvalreportInput.py.
00341 : 00342 00343 #################### 00344 # Obtain required environment variables 00345 # 00346 00347 try: 00348 CMSSW_BASE = os.environ['CMSSW_BASE'] 00349 CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE'] 00350 CMSSW_VERSION = os.environ['CMSSW_VERSION'] 00351 except KeyError: 00352 print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.' 00353 print ' Please run eval `scramv1 runtime -csh` to set your environment variables' 00354 sys.exit() 00355 00356 return ( CMSSW_BASE, 00357 CMSSW_RELEASE_BASE, 00358 CMSSW_VERSION)
def cmsRelvalreportInput::main | ( | argv = sys.argv | ) |
Definition at line 946 of file cmsRelvalreportInput.py.
00947 : 00948 00949 ##################### 00950 # Options Parser 00951 # 00952 00953 (options, args) = optionparse() 00954 00955 ##################### 00956 # Set up arguments and option handling 00957 # 00958 00959 (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args) 00960 #print setupProgramParameters(options,args) 00961 00962 ###################### 00963 # Initialize a few variables 00964 # 00965 00966 (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars() 00967 00968 ################## 00969 # Ok everything is ready now we need to create the input file for the relvalreport script 00970 # 00971 00972 simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE) 00973 00974 ################## 00975 # Based on the profile code create the array of profiles to run: 00976 # 00977 00978 Profile = getProfileArray(ProfileCode) 00979 00980 00981 ################## 00982 # Write the commands for the report to the file 00983 # 00984 00985 writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile) 00986 00987 print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name) 00988 00989 simcandles.close()
def cmsRelvalreportInput::optionparse | ( | ) |
Definition at line 143 of file cmsRelvalreportInput.py.
00144 : 00145 global _noprof 00146 explanations = map(lambda x: " " + x, Candles) 00147 explanation = "" 00148 for x in range(len(explanations)): 00149 explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x]) 00150 parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption] 00151 00152 Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps. 00153 00154 Arguments: 00155 NUM_EVENTS_PER_CFG - The number of events per config file 00156 00157 CANDLES - The simulation type to perform profiling on 00158 Candles Description 00159 AllCandles Run all of the candles below 00160 %s 00161 00162 PROFILE - the type of profiling to perform (multiple codes can be used): 00163 Code Profile Type 00164 0 TimingReport 00165 1 TimeReport 00166 2 SimpleMemoryCheck 00167 3 EdmSize 00168 4 IgProfPerf 00169 5 IgProfMemTotal 00170 6 IgProfMemLive 00171 7 IgProfAnalyse 00172 8 ValgrindFCE 00173 9 ValgrindMemCheck 00174 00175 Examples (could be obsolete!): 00176 Perform Timing Report profiling for all candles, all steps and 10 events per cfg. 00177 ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\" 00178 Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events. 00179 ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\" 00180 Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events. 00181 ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\" 00182 Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps. 00183 ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI""" 00184 % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME))) 00185 00186 devel = opt.OptionGroup(parser, "Developer Options", 00187 "Caution: use these options at your own risk." 00188 "It is believed that some of them bite.\n") 00189 #parser.set_defaults(debug=False) 00190 00191 parser.add_option( 00192 '-b', 00193 '--bypass-hlt', 00194 action="store_true", 00195 dest='bypasshlt', 00196 default=False, 00197 help='Should we bypass using the HLT root file?' 00198 ) 00199 00200 parser.add_option( 00201 '-u', 00202 '--usersteps', 00203 type='string', 00204 dest='userSteps', 00205 help='Which steps to run', 00206 metavar='<STEPS>', 00207 ) 00208 parser.add_option( 00209 '-c', 00210 '--cmsdriver', 00211 type='string', 00212 dest='cmsDriverOptions', 00213 help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands', 00214 metavar='<CONDITION>', 00215 ) 00216 parser.add_option( 00217 '-i', 00218 '--filein', 00219 type='string', 00220 dest='userInputFile', 00221 help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)', 00222 default='', 00223 metavar='<INPUTFILE>', 00224 ) 00225 devel.add_option( 00226 '-1', 00227 '--no-profile', 00228 action="store_true", 00229 dest='noprof', 00230 help='Do not perform profiling, ever', 00231 #metavar='DEBUG', 00232 ) 00233 00234 devel.add_option( 00235 '-d', 00236 '--debug', 00237 action="store_true", 00238 dest='debug', 00239 help='Show debug output', 00240 #metavar='DEBUG', 00241 ) 00242 00243 parser.set_defaults(debug=False,noprof=False) 00244 parser.add_option_group(devel) 00245 00246 (options, args) = parser.parse_args() 00247 00248 00249 _noprof = options.noprof 00250 debug = options.debug 00251 numofargs = len(args) 00252 # print "runtime cmsinput " + str(numofargs) + " noprof " + str(_noprof) 00253 if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)): 00254 parser.error("There are not enough arguments specified to run this program." 00255 " Please determine the correct arguments from the usage information above." 00256 " Run with --help option for more information.") 00257 sys.exit() 00258 00259 return (options, args)
def cmsRelvalreportInput::pythonFragment | ( | step, | |
cmsdriverOptions | |||
) |
Definition at line 437 of file cmsRelvalreportInput.py.
00438 : 00439 # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py: 00440 #It is now living in cmsPerfCommons.py! 00441 00442 if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")): 00443 return CustomiseFragment['DIGI-PILEUP'] 00444 elif CustomiseFragment.has_key(step): 00445 return CustomiseFragment[step] 00446 else: 00447 #This is a safe default in any case, 00448 #the maintenance of the customise python fragments goes into cmsPerfCommons.py 00449 return CustomiseFragment['DIGI'] 00450
def cmsRelvalreportInput::setInputFile | ( | steps, | |
step, | |||
acandle, | |||
stepIndex, | |||
pileup = False , |
|||
bypasshlt = False |
|||
) |
Definition at line 451 of file cmsRelvalreportInput.py.
00452 : 00453 #print "StepIndex 2:%s"%stepIndex 00454 InputFileOption = "" 00455 if pileup and stepIndex == 0: 00456 InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" ) 00457 else: 00458 InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] ) 00459 00460 if pileup: 00461 pass 00462 else : 00463 if 'GEN,SIM' in step: # there is no input file for GEN,SIM! 00464 InputFileOption = '' 00465 if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM! 00466 InputFileOption = '' 00467 elif 'HLT' in steps[stepIndex - 1] and bypasshlt: 00468 00469 # Special hand skipping of HLT since it is not stable enough, so it will not prevent 00470 # RAW2DIGI,RECO from running 00471 00472 InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 2] ) 00473 00474 if not InputFileOption == "" : 00475 InputFileOption += ".root " 00476 00477 return InputFileOption
def cmsRelvalreportInput::setOutputFileOption | ( | acandle, | |
endstep | |||
) |
Definition at line 538 of file cmsRelvalreportInput.py.
def cmsRelvalreportInput::setupProgramParameters | ( | options, | |
args | |||
) |
Definition at line 299 of file cmsRelvalreportInput.py.
00300 : 00301 steps = [] 00302 cmsDriverOptions = "" 00303 global AfterPileUpSteps 00304 NumberOfEvents = int(args[0]) # first arg 00305 WhichCandles = str(args[1]) # second arg 00306 ProfileCode = "" 00307 if not _noprof: 00308 ProfileCode = str(args[2]) # third arg 00309 00310 if options.cmsDriverOptions: 00311 00312 cmsDriverOptions = options.cmsDriverOptions 00313 print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions 00314 00315 if options.userSteps: 00316 00317 userSteps = options.userSteps 00318 steps = getSteps(userSteps) 00319 00320 if WhichCandles.lower() == 'allcandles': 00321 Candle = Candles 00322 print 'ALL standard simulation candles will be PROCESSED:' 00323 else: 00324 Candle = [WhichCandles] 00325 print 'Candle %s will be PROCESSED' % Candle[0] 00326 00327 # For now the two steps are built in, this can be added as an argument later 00328 # Added the argument option so now this will only be defined if it was not defined already: 00329 00330 if not steps: 00331 print 'The default steps will be run:' 00332 steps = DEF_STEPS 00333 else: 00334 print "You defined your own steps to run:" 00335 00336 for astep in steps: 00337 print astep 00338 00339 return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
def cmsRelvalreportInput::writeCommands | ( | simcandles, | |
Profile, | |||
acandle, | |||
steps, | |||
NumberOfEvents, | |||
cmsDriverOptions, | |||
bypasshlt, | |||
stepIndex = 0 , |
|||
pileup = False , |
|||
userInputFile = "" |
|||
) |
Definition at line 541 of file cmsRelvalreportInput.py.
00551 : 00552 #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile) 00553 OutputStep = "" 00554 00555 stopIndex = len(steps) 00556 start = 0 00557 00558 userSteps = steps 00559 SavedProfile = [] 00560 fstROOTfile = True 00561 fstROOTfileStr = "" 00562 00563 #Handling the case of the first user step not being the first step (GEN,SIM): 00564 #print "Steps passed to writeCommands %s",steps 00565 #Special case for FASTSIM: 00566 if (steps[0] == "GEN,FASTSIM"): 00567 start=-1 00568 stopIndex=1 00569 steps = ["GEN,FASTSIM"] 00570 else: 00571 if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"): 00572 #print "userInputFile: %s"%userInputFile 00573 if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) : 00574 userInputFile = "../INPUT_PILEUP_EVENTS.root" 00575 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0]) 00576 rootFileStr="" 00577 00578 elif userInputFile == "": 00579 #Write the necessary line to run without profiling all the steps before the wanted ones in one shot: 00580 (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt) 00581 else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows 00582 #if "-" in steps[0]: 00583 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0]) 00584 #print "Here!" 00585 #else: 00586 #print "There!" 00587 # stepIndex=AllSteps.index(steps[0]) 00588 rootFileStr="" 00589 #Now take care of setting the indeces and input root file name right for the profiling part... 00590 if fstROOTfile: 00591 fstROOTfileStr = rootFileStr 00592 fstROOTfile = False 00593 start = -1 00594 #if "-" in steps[0]: 00595 start = AllSteps.index(steps[0].split("-")[0].split(":")[0]) 00596 #else: 00597 # start = AllSteps.index(steps[0]) 00598 lst = - 1 00599 if "-" in steps[-1]: 00600 lst = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI 00601 else: 00602 lst = AllSteps.index(steps[-1]) 00603 runSteps = AllSteps[start:lst] 00604 numOfSteps = (lst - start) + 1 00605 stopIndex = start + numOfSteps 00606 #Handling the case in which the first user step is the same as the first step (GEN,SIM) 00607 #elif not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"): 00608 else: 00609 #Handling the case of the last step being a composite one: 00610 if "-" in steps[-1]: 00611 #Set the stop index at the last step of the composite step (WHY???) 00612 stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1 00613 else: 00614 00615 stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1 00616 #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex) 00617 steps = AllSteps 00618 00619 unprofiledSteps = [] 00620 rawreg = re.compile("^RAW2DIGI") 00621 00622 #Horrible structure... to be rewritten sooner or later... 00623 00624 # FOR step in steps 00625 00626 prevPrevOutputFile = "" 00627 #Add if to introduce the possibility to run on user selected input file. 00628 if userInputFile!="": 00629 previousOutputFile=userInputFile 00630 else: 00631 previousOutputFile = "" 00632 00633 for x in range(start,stopIndex,1): 00634 if stepIndex >= stopIndex: 00635 break 00636 step = steps[stepIndex] 00637 00638 # One shot Profiling variables 00639 befStep = step 00640 aftStep = step 00641 # We need this in case we are running one-shot profiling or for DIGI-PILEUP 00642 stepToWrite = step 00643 00644 CustomisePythonFragment = pythonFragment(step,cmsDriverOptions) 00645 oneShotProf = False 00646 hypsteps = [] 00647 00648 #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps 00649 if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)): 00650 00651 if "GEN,FASTSIM" in userSteps: # HLT/L1 menu workaround 00652 stepToWrite = stepToWrite + ",HLT:GRun" 00653 elif "HLT" in userSteps: 00654 stepToWrite = stepToWrite.replace("HLT", "HLT:GRun") 00655 00656 #Checking now if the current step matches the first of a composite step in userSteps 00657 hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps)) 00658 if not len(hypMatch) == 0 : 00659 hypsteps = expandHyphens(hypMatch[0]) 00660 #print hypsteps 00661 #print hypMatch[0] 00662 stepToWrite = ",".join(hypsteps) 00663 if "GEN,SIM-HLT" in userSteps: # HLT/L1 menu workaround 00664 stepToWrite = stepToWrite.replace("HLT","HLT:GRun") 00665 befStep = hypsteps[0] 00666 #Kludge to avoid running HLT in composite steps if the -b option is chosen 00667 if bypasshlt and hypsteps[-1]=='HLT': 00668 aftStep = hypsteps[-2] 00669 else: 00670 aftStep = hypsteps[-1] 00671 oneShotProf = True 00672 if filter(lambda x: stepToWrite==x.split(":")[0],userSteps): 00673 #print "Inside the filter if!" 00674 #print stepToWrite 00675 #Here we are assuming that the steps specified with : are not duplicate in the userSteps command: 00676 #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK 00677 #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!) 00678 stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0] 00679 #print stepToWrite 00680 00681 writeStepHead(simcandles,acandle,stepToWrite) 00682 00683 #Set the output file name for Pile up and for regular case: 00684 if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change to make root file output work (without colons) 00685 stepToWrite = stepToWrite.replace(":","=") 00686 00687 if '--pileup' in cmsDriverOptions: 00688 outfile = stepToWrite + "_PILEUP" 00689 else: 00690 outfile = stepToWrite 00691 00692 #print stepToWrite 00693 00694 if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change it back 00695 stepToWrite = stepToWrite.replace("=",":") 00696 00697 OutputFile = setOutputFileOption(acandle,outfile) 00698 if fstROOTfile: 00699 fstROOTfileStr = OutputFile 00700 fstROOTfile = False 00701 OutputFileOption = "--fileout=" + OutputFile 00702 00703 for prof in Profile: 00704 #First prepare the cmsDriver.py command 00705 00706 #Special case of EventEdmSize profiling 00707 if 'EdmSize' in prof: 00708 EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected. 00709 #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling! 00710 if prof == Profile[0] and not os.path.exists("./" + EdmFile): 00711 # insert command to generate required state ( need to run one more step 00712 # so that EDM can actually check the size of the root file 00713 # If the first step to be profiled is something later on in the steps such 00714 # as HLT then writePrerequisteSteps() should have got you to the step prior to 00715 # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself 00716 00717 InputFileOption = "--filein file:" + previousOutputFile 00718 #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT! 00719 if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile: 00720 InputFileOption = "--filein file:" + prevPrevOutputFile 00721 if previousOutputFile == "": 00722 InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt) 00723 Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" 00724 % (cmsDriver, 00725 KeywordToCfi[acandle], 00726 NumberOfEvents, 00727 stepToWrite, 00728 InputFileOption, 00729 OutputFileOption, 00730 CustomisePythonFragment, 00731 cmsDriverOptions) ) 00732 simcandles.write( "%s @@@ None @@@ None\n" % (Command)) 00733 00734 Command = EdmFile 00735 #all other profiles: 00736 else: 00737 InputFileOption = "--filein file:" + previousOutputFile 00738 if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile: 00739 InputFileOption = "--filein file:" + prevPrevOutputFile 00740 00741 if previousOutputFile == "": 00742 InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt) 00743 00744 #if '--pileup' in cmsDriverOptions: 00745 # stepToWrite = pileupStep 00746 if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'): 00747 Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % ( 00748 cmsDriver, 00749 KeywordToCfi[acandle], 00750 NumberOfEvents, 00751 stepToWrite, 00752 InputFileOption, 00753 OutputFileOption, 00754 CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps 00755 cmsDriverOptions[:cmsDriverOptions.index('--pileup')] 00756 )) 00757 elif '--pileup' in cmsDriverOptions and (stepToWrite=='GEN,FASTSIM' or stepToWrite=='GEN,FASTSIM,HLT:GRun'): #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI'] 00758 #+exception to the HLT:GRun case 00759 Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % ( 00760 cmsDriver, 00761 KeywordToCfi[acandle], 00762 NumberOfEvents, 00763 stepToWrite, 00764 InputFileOption, 00765 OutputFileOption, 00766 CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps 00767 cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')] 00768 )) 00769 00770 else: 00771 Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % ( 00772 cmsDriver, 00773 KeywordToCfi[acandle], 00774 NumberOfEvents, 00775 stepToWrite, 00776 InputFileOption, 00777 OutputFileOption, 00778 CustomisePythonFragment, 00779 cmsDriverOptions 00780 )) 00781 #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file! 00782 if _noprof: 00783 simcandles.write("%s @@@ None @@@ None\n" % Command) 00784 else: 00785 if 'valgrind' in prof: 00786 #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK. 00787 if 'TimeMemoryInfo.py' in Command: 00788 Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '') 00789 #There is actually also the SIM case to take care of: 00790 elif 'TimeMemoryG4Info.py' in Command: 00791 Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '') 00792 stepLabel=stepToWrite 00793 #print "stepLabel is %s"%stepLabel 00794 if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite: 00795 stepLabel = stepToWrite+"_PILEUP" 00796 #Add kludge here to make the IGPROF ANALYSE work: 00797 #The idea is to: 00798 #1-Check if ANALYSE is there everytime we're handling IGPROF profiles 00799 #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata 00800 #[3-Catch the case of ANALYSE by itself and raise an exception] 00801 #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE. 00802 #print "EEEEEEE %s"%prof 00803 #We actually want the analyse to be used also for IgProf perf! 00804 if 'IgProf' in prof: # and 'perf' not in prof: 00805 #New IgProf naming scheme: 00806 #Add a new naming scheme to simplify the handling in igprof-navigator: 00807 #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3) 00808 00809 #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling: 00810 IgProfProfile="IgProf" 00811 if "Mem" in prof: 00812 if "reuse" in prof: 00813 IgProfProfile="IgProfMem @@@ reuse" 00814 else: 00815 IgProfProfile="IgProfMem" 00816 else: 00817 if "reuse" in prof: 00818 IgProfProfile="IgProfPerf @@@ reuse" 00819 else: 00820 IgProfProfile="IgProfPerf" 00821 00822 #Conditions and PileUpOption are extracted from the cmsDriverOptions: 00823 Conditions="UNKNOWN_CONDITIONS" 00824 PileUp="NOPILEUP" 00825 EventContent="UNKNOWN_EVENTCONTENT" 00826 tokens=cmsDriverOptions.split("--") 00827 for token in tokens: 00828 keywords=token.split(" ") 00829 if "=" in keywords[0]: 00830 keywords=keywords[0].split("=") 00831 if "conditions" in keywords[0]: 00832 # check if we are using the autoCond style of flexible conditions 00833 # if so, expand the condition here so that the file names contain the real conditions 00834 if "auto:" in keywords[1]: 00835 from Configuration.AlCa.autoCond import autoCond 00836 fileConditionsOption = autoCond[ keywords[1].split(':')[1] ] 00837 Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0] 00838 else: 00839 # old style, hardcoded, conditions ... 00840 # Complicated expression, just to get rid of FrontierConditions_GlobalTag, 00841 # and ::All at the end: 00842 # Added if to handle new --conditions convention for cmsDriver.py (no more 00843 # FrontierConditions... in front of it) 00844 if "," in keywords[1]: 00845 Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility... 00846 else: 00847 Conditions=keywords[1].split("::")[0] 00848 elif "pileup" in keywords[0]: 00849 PileUp=keywords[1] 00850 elif "eventcontent" in keywords[0]: 00851 EventContent=keywords[1] 00852 00853 #so here's our new MetaName: 00854 MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile 00855 00856 if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile): 00857 00858 simcandles.write("%s @@@ %s @@@ %s\n" % (Command, 00859 Profiler[prof], 00860 MetaName)) 00861 #FileName[acandle], 00862 #stepLabel, 00863 #prof)) 00864 analyse_command = ("%s @@@ %s @@@ %s\n" % (Command, 00865 Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL) 00866 MetaName)) 00867 #FileName[acandle], 00868 #stepLabel, 00869 #prof)) 00870 00871 #for the relval case, GEN-SIM,HLT get rid of the last " @@@ reuse": 00872 if Profiler[prof] == 'IgProf_mem.MEM_MAX' or Profiler[prof] == 'IgProf_perf.PERF_TICKS': 00873 analyse_command = analyse_command.replace("@@@ reuse", "") 00874 00875 simcandles.write(analyse_command) 00876 00877 elif 'Analyse' in prof: 00878 pass 00879 else: 00880 simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command, 00881 Profiler[prof], 00882 FileName[acandle], 00883 stepLabel, 00884 prof)) 00885 00886 if debug: 00887 print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps 00888 print "cmsDriveroptions : " + cmsDriverOption 00889 prevPrevOutputFile = previousOutputFile 00890 previousOutputFile = OutputFile 00891 else: 00892 unprofiledSteps.append(step) 00893 isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false 00894 00895 try: 00896 isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps)) 00897 #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex]) 00898 except IndexError: 00899 # This loop should have terminated if x + 1 is out of bounds! 00900 print "Error: Something is wrong we shouldn't have come this far" 00901 break 00902 00903 if isNextStepForProfile: 00904 #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps... 00905 OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt) 00906 unprofiledSteps = [] 00907 prevPrevOutputFile = previousOutputFile 00908 previousOutputFile = OutputFile 00909 #Dangerous index handling when looping over index x! 00910 if oneShotProf: 00911 stepIndex += len(hypsteps) 00912 else: 00913 stepIndex +=1 00914 return fstROOTfileStr
def cmsRelvalreportInput::writeCommandsToReport | ( | simcandles, | |
Candle, | |||
Profile, | |||
debug, | |||
NumberOfEvents, | |||
cmsDriverOptions, | |||
steps, | |||
bypasshlt, | |||
userInputFile | |||
) |
Definition at line 915 of file cmsRelvalreportInput.py.
00916 : 00917 00918 OutputStep = '' 00919 00920 # This option will not be used for now since the event content names keep changing, it can be 00921 # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone). 00922 # 00923 #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'} 00924 00925 for acandle in Candle: 00926 print '*Candle ' + acandle 00927 00928 ################## 00929 # If the first profiling we run is EdmSize we need to create the root file first 00930 # 00931 00932 #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already: 00933 #print "1-userInputFile: %s"%userInputFile 00934 00935 fstoutfile = writeCommands(simcandles, 00936 Profile, 00937 acandle, 00938 steps, 00939 NumberOfEvents, 00940 cmsDriverOptions, 00941 bypasshlt, 00942 0, 00943 False, 00944 userInputFile) 00945
def cmsRelvalreportInput::writePrerequisteSteps | ( | simcandles, | |
steps, | |||
acandle, | |||
NumberOfEvents, | |||
cmsDriverOptions, | |||
pileup, | |||
bypasshlt | |||
) |
Definition at line 525 of file cmsRelvalreportInput.py.
00526 : 00527 fstIdx = -1 00528 #Hack for GEN-FASTSIM workflow: 00529 #if steps=="GEN,FASTSIM": 00530 #if "-" in steps[0]: 00531 fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0]) 00532 #else: 00533 # fstIdx = AllSteps.index(steps[0]) 00534 CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions) 00535 previousOutputFile="" 00536 OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt) 00537 return (fstIdx, OutputFile)
def cmsRelvalreportInput::writeReportFileHeader | ( | CMSSW_VERSION, | |
CMSSW_RELEASE_BASE, | |||
CMSSW_BASE | |||
) |
Definition at line 359 of file cmsRelvalreportInput.py.
00360 : 00361 00362 SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION 00363 00364 try: 00365 simcandles = open(SimCandlesFile, 'w') 00366 except IOError: 00367 print "Couldn't open %s to save" % SimCandlesFile 00368 00369 simcandles.write('#Candles file automatically generated by %s for %s\n' 00370 % (THIS_PROG_NAME, CMSSW_VERSION)) 00371 simcandles.write("#CMSSW Base : %s\n" % CMSSW_BASE) 00372 simcandles.write("#Release Base: %s\n" % CMSSW_RELEASE_BASE) 00373 simcandles.write("#Version : %s\n\n" % CMSSW_VERSION) 00374 00375 return simcandles
def cmsRelvalreportInput::writeStepHead | ( | simcandles, | |
acandle, | |||
step | |||
) |
Definition at line 422 of file cmsRelvalreportInput.py.
def cmsRelvalreportInput::writeUnprofiledSteps | ( | simcandles, | |
CustomisePythonFragment, | |||
cmsDriverOptions, | |||
unprofiledSteps, | |||
previousOutputFile, | |||
acandle, | |||
NumberOfEvents, | |||
stepIndex, | |||
pileup, | |||
bypasshlt | |||
) |
Definition at line 478 of file cmsRelvalreportInput.py.
00479 : 00480 # reduce(lambda x,y : x + "," + "y",unprofiledSteps) 00481 #stepsStr = ",".join(unprofiledSteps) 00482 00483 00484 #print "unprofiledSteps is %s"%unprofiledSteps 00485 #print "22acandle is %s"%acandle 00486 #Kludge in case -b option to skip HLT is used... 00487 if bypasshlt and unprofiledSteps[-1]=="HLT": 00488 stepsStr = ",".join(unprofiledSteps[:-1]) 00489 OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2]) 00490 else: 00491 stepsStr = ",".join(unprofiledSteps) 00492 OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1]) 00493 simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr)) 00494 OutputFileOption = "--fileout=%s" % OutputFile 00495 #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed 00496 #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there! 00497 #Adding the argument bypasshlt to the calls... 00498 #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1] 00499 #print "StepIndex 1:%s"%stepIndex 00500 #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step... 00501 #stepIndexAdjust=stepIndex - 2 00502 #if stepIndexAdjust < 0: #To avoid issues with negative indeces 00503 # stepIndexAdjust=0 00504 #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt) 00505 #Use an argument to make it easier: 00506 InputFileOption = "--filein file:" + previousOutputFile 00507 if previousOutputFile =="": 00508 InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt) 00509 #Introduce an over-ride of cmsDriverOptions: 00510 #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent 00511 #At the moment the only use case is when running step2 on its own... 00512 for eventcontent in EventContents: 00513 cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions) 00514 Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" 00515 % (cmsDriver, 00516 KeywordToCfi[acandle], 00517 NumberOfEvents, 00518 stepsStr, 00519 InputFileOption, 00520 OutputFileOption, 00521 CustomisePythonFragment, 00522 cmsDriverOptions) ) 00523 simcandles.write( "%s @@@ None @@@ None\n\n" % (Command)) 00524 return OutputFile
list cmsRelvalreportInput::AfterPileUpSteps = [] |
Definition at line 37 of file cmsRelvalreportInput.py.
list cmsRelvalreportInput::AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"] |
Definition at line 36 of file cmsRelvalreportInput.py.
string cmsRelvalreportInput::cmsDriver = 'cmsDriver.py' |
Definition at line 32 of file cmsRelvalreportInput.py.
cmsRelvalreportInput::debug = False |
Definition at line 34 of file cmsRelvalreportInput.py.
list cmsRelvalreportInput::DEF_STEPS = ['GEN,SIM', 'DIGI'] |
Definition at line 35 of file cmsRelvalreportInput.py.
tuple cmsRelvalreportInput::hypreg = re.compile('-') |
Definition at line 33 of file cmsRelvalreportInput.py.
dictionary cmsRelvalreportInput::Profiler |
00001 { 00002 'TimingReport' : 'Timing_Parser', 00003 'TimingReport @@@ reuse' : 'Timing_Parser', 00004 'TimeReport' : 'Timereport_Parser', 00005 'TimeReport @@@ reuse' : 'Timereport_Parser', 00006 'SimpleMemReport' : 'SimpleMem_Parser', 00007 'EdmSize' : 'Edm_Size', 00008 'IgProfperf' : 'IgProf_perf.PERF_TICKS', 00009 'IgProfperf @@@ reuse' : 'IgProf_perf.PERF_TICKS', 00010 'IgProfMemTotal' : 'IgProf_mem.MEM_TOTAL', 00011 'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL', 00012 'IgProfMemLive' : 'IgProf_mem.MEM_LIVE', 00013 'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE', 00014 'IgProfMemMax' : 'IgProf_mem.MEM_MAX', 00015 'IgProfMemMax @@@ reuse' : 'IgProf_mem.MEM_MAX', 00016 'IgProfMemAnalyse' : 'IgProf_mem.ANALYSE', 00017 'valgrind' : 'ValgrindFCE', 00018 'memcheck_valgrind' : 'Memcheck_Valgrind', 00019 'None' : 'None', 00020 }
Definition at line 43 of file cmsRelvalreportInput.py.
tuple cmsRelvalreportInput::THIS_PROG_NAME = os.path.basename(sys.argv[0]) |
Global variables.
Definition at line 31 of file cmsRelvalreportInput.py.