Functions
def	checkSteps
def	determineNewProfile
def	expandHyphens
def	getFstOccur
def	getLstIndex
def	getProfileArray
def	getSteps
def	init_vars
def	main
def	optionparse
def	pythonFragment
def	setInputFile
def	setOutputFileOption
def	setupProgramParameters
def	writeCommands
def	writeCommandsToReport
def	writePrerequisteSteps
def	writeReportFileHeader
def	writeStepHead
def	writeUnprofiledSteps
Variables
list	AfterPileUpSteps = []
list	AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
string	cmsDriver = 'cmsDriver.py'
	debug = False
list	DEF_STEPS = ['GEN,SIM', 'DIGI']
tuple	hypreg = re.compile('-')
dictionary	Profiler
tuple	THIS_PROG_NAME = os.path.basename(sys.argv[0])
	Global variables.

Function Documentation

def cmsRelvalreportInput::checkSteps ( steps )

Checks user steps <steps> for order against steps defined in AllSteps list.
If they are not, prints a warning and exits.
No return value.

Definition at line 75 of file cmsRelvalreportInput.py.

00076                      :
00077     '''Checks user steps <steps> for order against steps defined in AllSteps list.
00078     If they are not, prints a warning and exits.
00079     No return value.'''
00080     
00081     #print steps
00082     #if ":" in steps:
00083     #    steps.split(
00084     idx = -2
00085     lstidx = -2
00086     for step in steps:
00087         astep = step
00088         split = []
00089         #Added special workflow for GEN-FASTSIM, bypass the check of step order:
00090         if step=="GEN,FASTSIM":
00091             pass
00092         else:
00093             if "-" in step:
00094                 split = astep.split("-")
00095                 astep = split[0].split(":")[0]
00096              #   print "This is astep:%s"%astep
00097             #idx =AllSteps.index("GEN,SIM")
00098             #print idx
00099             #print astep
00100             idx = AllSteps.index(astep.split(":")[0])            
00101             if not ( idx == -2 ):
00102                 if lstidx > idx:
00103                     print "ERROR: Your user defined steps are not in a valid order"
00104                     sys.exit()
00105                 lstidx = idx    
00106             if "-" in step:
00107                 lstidx = AllSteps.index(split[1].split(":")[0])
00108

def cmsRelvalreportInput::determineNewProfile	(	step,
		Profile,
		SavedProfile
	)

Definition at line 428 of file cmsRelvalreportInput.py.

00429                                                   :
00430     if 'DIGI2RAW' in step:
00431         SavedProfile = Profile
00432         Profile = [ ]
00433     if 'HLT' in step:
00434         Profile = SavedProfile
00435 
00436     return (Profile, SavedProfile)

def cmsRelvalreportInput::expandHyphens ( step )

Definition at line 260 of file cmsRelvalreportInput.py.

00261                        :
00262     newsteps = []
00263 #    for step in steps:
00264     if "-" in step:
00265         hypsteps = step.split(r"-")
00266         #if not (len(hypsteps) == 2):
00267         #    print "ERROR: Could not parse usersteps. You have too many hypens between commas"
00268         #    sys.exit()
00269         #el
00270         if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
00271         #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
00272             print "ERROR: One of the steps you defined is invalid"
00273             sys.exit()
00274         else:
00275             if (hypsteps[0] == hypsteps[1]):
00276                 print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
00277                 newsteps.append(hypsteps[0])
00278             else:
00279                 newsteps.append(hypsteps[0])
00280                 
00281                 srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
00282                 for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
00283                     astep = AllSteps[n]
00284                     if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
00285                         #break
00286                         astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
00287                         if astep==hypsteps[-1].split(":")[0]:
00288                             break
00289                     newsteps.append(astep)
00290                 newsteps.append(hypsteps[-1])
00291     else:
00292         if not (step.split(":")[0] in AllSteps):
00293             print "ERROR: One of the steps you defined is invalid"
00294             sys.exit()
00295         else:
00296             newsteps.append(step)
00297 
00298     return newsteps

def cmsRelvalreportInput::getFstOccur	(	item,
		list
	)

Definition at line 66 of file cmsRelvalreportInput.py.

00067                            :
00068     return filter(item.__eq__,list)[0]

def cmsRelvalreportInput::getLstIndex	(	item,
		list
	)

Definition at line 69 of file cmsRelvalreportInput.py.

00070                            :
00071     lenlist = len(list)
00072     for x in range(lenlist - 1,0,-1):
00073         if list[x] == item:
00074             return x

def cmsRelvalreportInput::getProfileArray ( ProfileCode )

Definition at line 376 of file cmsRelvalreportInput.py.

00377                                 :
00378 
00379     Profile = []
00380 
00381     # The allowed profiles are:
00382 
00383     AllowedProfile = [
00384         'TimingReport',
00385         'TimeReport',
00386         'SimpleMemReport',
00387         'EdmSize',
00388         'IgProfperf',
00389         'IgProfMemTotal',
00390         'IgProfMemLive',
00391         #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
00392         'IgProfMemAnalyse',
00393         'valgrind',
00394         'memcheck_valgrind',
00395         'None',
00396         ]
00397 
00398     if _noprof:
00399         Profile.append(AllowedProfile[-1])
00400     else:
00401         #FIXME:
00402         #Horrible code!
00403         for i in range(10):
00404             if str(i) in ProfileCode:
00405                 firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
00406                             or (i == 1 and str(2) in ProfileCode)
00407                 secCase   = (i==4 and str(7) in ProfileCode) \
00408                             or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
00409                             or (i == 6 and str(7) in ProfileCode)
00410                 
00411                 if firstCase or secCase:
00412                     Profile.append(AllowedProfile[i] + ' @@@ reuse')
00413                     #Here's the hack:
00414                     if i==6: #i.e. IgProfMemLive
00415                         Profile.append("IgProfMemMax @@@ reuse")
00416                 else:
00417                     Profile.append(AllowedProfile[i])
00418                     if i==6: #i.e. IgProfMemLive
00419                         Profile.append("IgProfMemMax")
00420 
00421     return Profile

def cmsRelvalreportInput::getSteps ( userSteps )

Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
Returns a list of steps <steps>.

Definition at line 109 of file cmsRelvalreportInput.py.

00110                        :
00111     '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
00112     Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
00113     Returns a list of steps <steps>.'''
00114     #Add handling of ":" in steps
00115     #print userSteps
00116     # Then split the user steps into "steps"
00117     gsreg = re.compile('GEN-SIM')
00118     greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
00119     gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
00120     StepsTokens = userSteps.split(",")
00121     steps = [] 
00122     for astep in StepsTokens:
00123 
00124         # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
00125         # from using the "-" to using the "," to match cmsDriver.py convention
00126 
00127         if gsreg.search(astep):
00128             astep = gsreg.sub(r"GEN,SIM", astep)
00129         elif gfsreg.search(astep):
00130             astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
00131         elif greg.search(astep):
00132             astep = greg.sub(r"GEN,SIM", astep)
00133             
00134             
00135         #print astep
00136         # Finally collect all the steps into the steps list:
00137 
00138         steps.append(astep)
00139     
00140     #steps = expandHypens(steps)
00141     checkSteps(steps)
00142     return steps

def cmsRelvalreportInput::init_vars ( )

Definition at line 340 of file cmsRelvalreportInput.py.

00341                :
00342 
00343     ####################
00344     # Obtain required environment variables
00345     #
00346 
00347     try:
00348         CMSSW_BASE         = os.environ['CMSSW_BASE']
00349         CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
00350         CMSSW_VERSION      = os.environ['CMSSW_VERSION']
00351     except KeyError:
00352         print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
00353         print '       Please run eval `scramv1 runtime -csh` to set your environment variables'
00354         sys.exit()
00355 
00356     return ( CMSSW_BASE,
00357              CMSSW_RELEASE_BASE,
00358              CMSSW_VERSION)

def cmsRelvalreportInput::main ( argv = sys.argv )

Definition at line 922 of file cmsRelvalreportInput.py.

00923                        :
00924 
00925     #####################
00926     # Options Parser 
00927     #
00928 
00929     (options, args) = optionparse()
00930 
00931     #####################
00932     # Set up arguments and option handling
00933     #
00934 
00935     (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
00936     #print setupProgramParameters(options,args)
00937 
00938     ######################
00939     # Initialize a few variables
00940     #
00941 
00942     (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
00943 
00944     ##################
00945     # Ok everything is ready now we need to create the input file for the relvalreport script 
00946     #
00947 
00948     simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
00949 
00950     ##################
00951     # Based on the profile code create the array of profiles to run:
00952     #
00953 
00954     Profile = getProfileArray(ProfileCode)
00955 
00956 
00957     ##################
00958     # Write the commands for the report to the file
00959     #
00960 
00961     writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
00962 
00963     print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
00964                 
00965     simcandles.close()

def cmsRelvalreportInput::optionparse ( )

Definition at line 143 of file cmsRelvalreportInput.py.

00144                  :
00145     global _noprof
00146     explanations = map(lambda x: "          " + x, Candles)
00147     explanation  = ""
00148     for x in range(len(explanations)):
00149         explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
00150     parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
00151 
00152     Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
00153 
00154     Arguments:
00155         NUM_EVENTS_PER_CFG - The number of events per config file
00156 
00157         CANDLES - The simulation type to perform profiling on
00158           Candles              Description
00159           AllCandles           Run all of the candles below
00160 %s
00161 
00162         PROFILE - the type of profiling to perform (multiple codes can be used):
00163           Code  Profile Type
00164           0     TimingReport
00165           1     TimeReport
00166           2     SimpleMemoryCheck
00167           3     EdmSize
00168           4     IgProfPerf
00169           5     IgProfMemTotal
00170           6     IgProfMemLive
00171           7     IgProfAnalyse
00172           8     ValgrindFCE
00173           9     ValgrindMemCheck    
00174 
00175     Examples (could be obsolete!):  
00176        Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
00177         ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00178        Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
00179         ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00180        Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
00181         ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
00182        Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
00183         ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
00184       % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
00185     
00186     devel  = opt.OptionGroup(parser, "Developer Options",
00187                                      "Caution: use these options at your own risk."
00188                                      "It is believed that some of them bite.\n")
00189     #parser.set_defaults(debug=False)
00190 
00191     parser.add_option(
00192         '-b',
00193         '--bypass-hlt',
00194         action="store_true",
00195         dest='bypasshlt',
00196         default=False,
00197         help='Should we bypass using the HLT root file?'
00198         )
00199 
00200     parser.add_option(
00201         '-u',
00202         '--usersteps',
00203         type='string',
00204         dest='userSteps',
00205         help='Which steps to run',
00206         metavar='<STEPS>',
00207         )
00208     parser.add_option(
00209         '-c',
00210         '--cmsdriver',
00211         type='string',
00212         dest='cmsDriverOptions',
00213         help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
00214         metavar='<CONDITION>',
00215         )
00216     parser.add_option(
00217         '-i',
00218         '--filein',
00219         type='string',
00220         dest='userInputFile',
00221         help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
00222         default='',
00223         metavar='<INPUTFILE>',
00224         )
00225     devel.add_option(
00226         '-1',
00227         '--no-profile',
00228         action="store_true",
00229         dest='noprof',
00230         help='Do not perform profiling, ever',
00231         #metavar='DEBUG',
00232         )
00233     
00234     devel.add_option(
00235         '-d',
00236         '--debug',
00237         action="store_true",
00238         dest='debug',
00239         help='Show debug output',
00240         #metavar='DEBUG',
00241         )
00242 
00243     parser.set_defaults(debug=False,noprof=False)
00244     parser.add_option_group(devel)
00245 
00246     (options, args) = parser.parse_args()
00247     
00248 
00249     _noprof = options.noprof
00250     debug = options.debug
00251     numofargs = len(args) 
00252 #    print "runtime cmsinput " + str(numofargs) + " noprof "  + str(_noprof)
00253     if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
00254         parser.error("There are not enough arguments specified to run this program."
00255                      " Please determine the correct arguments from the usage information above."
00256                      " Run with --help option for more information.")
00257         sys.exit()
00258 
00259     return (options, args)

def cmsRelvalreportInput::pythonFragment	(	step,
		cmsdriverOptions
	)

Definition at line 437 of file cmsRelvalreportInput.py.

00438                                          :
00439     # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
00440     #It is now living in cmsPerfCommons.py!
00441     
00442     if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
00443         return CustomiseFragment['DIGI-PILEUP']
00444     elif CustomiseFragment.has_key(step):
00445         return CustomiseFragment[step] 
00446     else:
00447         #This is a safe default in any case,
00448         #the maintenance of the customise python fragments goes into cmsPerfCommons.py
00449         return CustomiseFragment['DIGI']
00450

def cmsRelvalreportInput::setInputFile	(	steps,
		step,
		acandle,
		stepIndex,
		pileup = `False`,
		bypasshlt = `False`
	)

Definition at line 451 of file cmsRelvalreportInput.py.

00452                                                                            :
00453     #print "StepIndex 2:%s"%stepIndex
00454     InputFileOption = ""
00455     if pileup and stepIndex == 0:
00456         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
00457     else:
00458         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
00459         
00460     if pileup:
00461         pass
00462     else :
00463         if 'GEN,SIM' in step:  # there is no input file for GEN,SIM!
00464             InputFileOption = ''
00465         if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
00466             InputFileOption = ''
00467         elif   'HLT' in steps[stepIndex - 1] and bypasshlt:
00468 
00469             # Special hand skipping of HLT since it is not stable enough, so it will not prevent
00470             # RAW2DIGI,RECO from running
00471 
00472             InputFileOption = "--filein file:%s_%s"  % ( FileName[acandle],steps[stepIndex - 2] )
00473 
00474     if not InputFileOption == "" :
00475         InputFileOption += ".root "
00476 
00477     return InputFileOption

def cmsRelvalreportInput::setOutputFileOption	(	acandle,
		endstep
	)

Definition at line 538 of file cmsRelvalreportInput.py.

00539                                         :
00540     return "%s_%s.root" % ( FileName[acandle],endstep)

def cmsRelvalreportInput::setupProgramParameters	(	options,
		args
	)

Definition at line 299 of file cmsRelvalreportInput.py.

00300                                         :
00301     steps = []
00302     cmsDriverOptions = ""
00303     global AfterPileUpSteps
00304     NumberOfEvents = int(args[0])  # first arg
00305     WhichCandles   = str(args[1])  # second arg
00306     ProfileCode = ""
00307     if not _noprof:
00308         ProfileCode    = str(args[2])  # third arg
00309 
00310     if options.cmsDriverOptions:
00311 
00312         cmsDriverOptions = options.cmsDriverOptions
00313         print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
00314 
00315     if options.userSteps:
00316 
00317         userSteps = options.userSteps
00318         steps = getSteps(userSteps)
00319 
00320     if WhichCandles.lower() == 'allcandles':
00321         Candle = Candles
00322         print 'ALL standard simulation candles will be PROCESSED:'
00323     else:
00324         Candle = [WhichCandles]
00325         print 'Candle %s will be PROCESSED' % Candle[0]
00326 
00327     # For now the two steps are built in, this can be added as an argument later
00328     # Added the argument option so now this will only be defined if it was not defined already:
00329 
00330     if not steps:
00331         print 'The default steps will be run:'
00332         steps = DEF_STEPS
00333     else:
00334         print "You defined your own steps to run:"
00335 
00336     for astep in steps:
00337         print astep
00338 
00339     return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)

def cmsRelvalreportInput::writeCommands	(	simcandles,
		Profile,
		acandle,
		steps,
		NumberOfEvents,
		cmsDriverOptions,
		bypasshlt,
		stepIndex = `0`,
		pileup = `False`,
		userInputFile = `""`
	)

Definition at line 541 of file cmsRelvalreportInput.py.

00551                                    :
00552     #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
00553     OutputStep = ""
00554 
00555     stopIndex = len(steps)
00556     start = 0
00557 
00558     userSteps = steps
00559     SavedProfile = []
00560     fstROOTfile = True
00561     fstROOTfileStr = ""
00562 
00563     #Handling the case of the first user step not being the first step (GEN,SIM):
00564     #print "Steps passed to writeCommands %s",steps
00565     #Special case for FASTSIM:
00566     if (steps[0] == "GEN,FASTSIM"):
00567         start=-1
00568         stopIndex=1
00569         steps = ["GEN,FASTSIM"]
00570     else:
00571         if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
00572             #print "userInputFile: %s"%userInputFile
00573             if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
00574                 userInputFile = "../INPUT_PILEUP_EVENTS.root"
00575                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00576                 rootFileStr=""
00577                 
00578             elif userInputFile == "":
00579                 #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
00580                 (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
00581             else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
00582                 #if "-" in steps[0]:
00583                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00584                     #print "Here!"
00585                 #else:
00586                     #print "There!"
00587                 #    stepIndex=AllSteps.index(steps[0])
00588                 rootFileStr=""
00589             #Now take care of setting the indeces and input root file name right for the profiling part...
00590             if fstROOTfile:
00591                 fstROOTfileStr = rootFileStr
00592                 fstROOTfile = False
00593             start = -1
00594             #if "-" in steps[0]:
00595             start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00596             #else:
00597             #    start = AllSteps.index(steps[0])
00598             lst = - 1
00599             if "-" in steps[-1]:
00600                 lst   = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
00601             else:
00602                 lst   = AllSteps.index(steps[-1])
00603             runSteps = AllSteps[start:lst]
00604             numOfSteps = (lst - start) + 1
00605             stopIndex = start + numOfSteps
00606             #Handling the case in which the first user step is the same as the first step (GEN,SIM)
00607             #elif  not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
00608         else:
00609             #Handling the case of the last step being a composite one:
00610             if "-" in steps[-1]:
00611                 #Set the stop index at the last step of the composite step (WHY???) 
00612                 stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
00613             else:
00614                 
00615                 stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
00616                 #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
00617         steps = AllSteps
00618             
00619     unprofiledSteps = []
00620     rawreg = re.compile("^RAW2DIGI")
00621 
00622 #Horrible structure... to be rewritten sooner or later...
00623     
00624 #   FOR step in steps
00625 
00626     prevPrevOutputFile = ""
00627     #Add if to introduce the possibility to run on user selected input file.
00628     if userInputFile!="":
00629         previousOutputFile=userInputFile
00630     else:
00631         previousOutputFile = ""
00632 
00633     for x in range(start,stopIndex,1):
00634         if stepIndex >= stopIndex:
00635             break
00636         step = steps[stepIndex]
00637 
00638         # One shot Profiling variables
00639         befStep     = step
00640         aftStep     = step
00641         # We need this in case we are running one-shot profiling or for DIGI-PILEUP
00642         stepToWrite = step 
00643         
00644         CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
00645         oneShotProf = False
00646         hypsteps = []
00647 
00648         #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
00649         if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)): 
00650 
00651             #Checking now if the current step matches the first of a composite step in userSteps
00652             hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
00653             if not len(hypMatch) == 0 :
00654                 hypsteps    = expandHyphens(hypMatch[0])
00655                 #print hypsteps
00656                 #print hypMatch[0]
00657                 stepToWrite = ",".join(hypsteps)
00658                 befStep     = hypsteps[0]
00659                 #Kludge to avoid running HLT in composite steps if the -b option is chosen
00660                 if bypasshlt and hypsteps[-1]=='HLT':
00661                     aftStep     = hypsteps[-2]
00662                 else:
00663                     aftStep     = hypsteps[-1]
00664                 oneShotProf = True
00665             if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
00666                 #print "Inside the filter if!"
00667                 #print stepToWrite
00668                 #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
00669                 #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
00670                 #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!) 
00671                 stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
00672                 #print stepToWrite
00673 
00674             writeStepHead(simcandles,acandle,stepToWrite)
00675 
00676             #Set the output file name for Pile up and for regular case:
00677             if '--pileup' in cmsDriverOptions:
00678                 outfile = stepToWrite + "_PILEUP"
00679             else:
00680                 outfile = stepToWrite
00681                 
00682             OutputFile = setOutputFileOption(acandle,outfile)
00683             if fstROOTfile:
00684                 fstROOTfileStr = OutputFile
00685                 fstROOTfile = False
00686             OutputFileOption = "--fileout=" + OutputFile
00687 
00688             for prof in Profile:
00689                 #First prepare the cmsDriver.py command
00690                 
00691                 #Special case of EventEdmSize profiling 
00692                 if 'EdmSize' in prof:
00693                     EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
00694                     #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
00695                     if prof == Profile[0] and not os.path.exists("./" + EdmFile):
00696                         # insert command to generate required state ( need to run one more step
00697                         # so that EDM can actually check the size of the root file
00698                         # If the first step to be profiled is something later on in the steps such
00699                         # as HLT then writePrerequisteSteps() should have got you to the step prior to
00700                         # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
00701 
00702                         InputFileOption = "--filein file:" + previousOutputFile
00703                         #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
00704                         if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00705                             InputFileOption = "--filein file:" + prevPrevOutputFile
00706                         if previousOutputFile == "":
00707                             InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00708                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00709                                            % (cmsDriver,
00710                                               KeywordToCfi[acandle],
00711                                               NumberOfEvents,
00712                                               stepToWrite,
00713                                               InputFileOption,
00714                                               OutputFileOption,
00715                                               CustomisePythonFragment,
00716                                               cmsDriverOptions) )
00717                         simcandles.write( "%s @@@ None @@@ None\n" % (Command))
00718 
00719                     Command = EdmFile
00720                 #all other profiles:
00721                 else:
00722                     InputFileOption = "--filein file:" + previousOutputFile
00723                     if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00724                         InputFileOption = "--filein file:" + prevPrevOutputFile
00725 
00726                     if previousOutputFile == "":
00727                         InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
00728 
00729                     #if '--pileup' in cmsDriverOptions:
00730                     #    stepToWrite = pileupStep
00731                     if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'): 
00732                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00733                                cmsDriver,
00734                                KeywordToCfi[acandle],
00735                                NumberOfEvents,
00736                                stepToWrite,
00737                                InputFileOption,
00738                                OutputFileOption,
00739                                CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00740                                cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
00741                            ))
00742                     elif '--pileup' in cmsDriverOptions and stepToWrite=='GEN,FASTSIM': #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
00743                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00744                                cmsDriver,
00745                                KeywordToCfi[acandle],
00746                                NumberOfEvents,
00747                                stepToWrite,
00748                                InputFileOption,
00749                                OutputFileOption,
00750                                CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00751                                cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')] 
00752                            ))
00753                     
00754                     else:
00755                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00756                                cmsDriver,
00757                                KeywordToCfi[acandle],
00758                                NumberOfEvents,
00759                                stepToWrite,
00760                                InputFileOption,
00761                                OutputFileOption,
00762                                CustomisePythonFragment,
00763                                cmsDriverOptions
00764                            ))
00765                 #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
00766                 if _noprof:
00767                     simcandles.write("%s @@@ None @@@ None\n" % Command)
00768                 else:
00769                     if 'valgrind' in prof:
00770                         #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
00771                         if 'TimeMemoryInfo.py' in Command:
00772                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
00773                         #There is actually also the SIM case to take care of:
00774                         elif 'TimeMemoryG4Info.py' in Command:
00775                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
00776                     stepLabel=stepToWrite
00777                     #print "stepLabel is %s"%stepLabel
00778                     if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
00779                         stepLabel = stepToWrite+"_PILEUP"
00780                     #Add kludge here to make the IGPROF ANALYSE work:
00781                     #The idea is to:
00782                     #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
00783                     #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
00784                     #[3-Catch the case of ANALYSE by itself and raise an exception]
00785                     #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
00786                     #print "EEEEEEE %s"%prof
00787                     #We actually want the analyse to be used also for IgProf perf!
00788                     if 'IgProf' in prof: # and 'perf' not in prof:
00789                         #New IgProf naming scheme:
00790                         #Add a new naming scheme to simplify the handling in igprof-navigator:
00791                         #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
00792 
00793                         #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
00794                         IgProfProfile="IgProf"
00795                         if "Mem" in prof:
00796                             if "reuse" in prof:
00797                                 IgProfProfile="IgProfMem @@@ reuse"
00798                             else:
00799                                 IgProfProfile="IgProfMem"
00800                         else:
00801                             if "reuse" in prof:
00802                                 IgProfProfile="IgProfPerf @@@ reuse"
00803                             else:
00804                                 IgProfProfile="IgProfPerf"
00805 
00806                         #Conditions and PileUpOption are extracted from the cmsDriverOptions:
00807                         Conditions="UNKNOWN_CONDITIONS"
00808                         PileUp="NOPILEUP"
00809                         EventContent="UNKNOWN_EVENTCONTENT"
00810                         tokens=cmsDriverOptions.split("--")
00811                         for token in tokens:
00812                             keywords=token.split(" ")
00813                             if "=" in keywords[0]:
00814                                 keywords=keywords[0].split("=")
00815                             if "conditions" in keywords[0]:
00816                                 # check if we are using the autoCond style of flexible conditions
00817                                 # if so, expand the condition here so that the file names contain the real conditions
00818                                 if "auto:" in keywords[1]:
00819                                     from Configuration.PyReleaseValidation.autoCond import autoCond
00820                                     fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
00821                                     Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0] 
00822                                 else:
00823                                     # old style, hardcoded, conditions ...
00824                                     # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
00825                                     # and ::All at the end:
00826                                     # Added if to handle new --conditions convention for cmsDriver.py (no more
00827                                     # FrontierConditions... in front of it)
00828                                     if "," in keywords[1]: 
00829                                         Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
00830                                     else:
00831                                         Conditions=keywords[1].split("::")[0] 
00832                             elif "pileup" in keywords[0]:
00833                                 PileUp=keywords[1]
00834                             elif "eventcontent" in keywords[0]:
00835                                 EventContent=keywords[1]
00836                         #so here's our new MetaName:
00837                         MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
00838                         
00839                         if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
00840                                 
00841                             simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
00842                                                                            Profiler[prof],
00843                                                                            MetaName))
00844                                                                            #FileName[acandle],
00845                                                                            #stepLabel,
00846                                                                            #prof))
00847                             simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
00848                                                                            Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
00849                                                                            MetaName))
00850                                                                            #FileName[acandle],
00851                                                                            #stepLabel,
00852                                                                            #prof))
00853                         elif 'Analyse' in prof:
00854                             pass
00855                     else:    
00856                         simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
00857                                                                        Profiler[prof],
00858                                                                        FileName[acandle],
00859                                                                        stepLabel,
00860                                                                        prof))
00861 
00862                 if debug:
00863                     print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
00864                     print "cmsDriveroptions : " + cmsDriverOption
00865             prevPrevOutputFile = previousOutputFile          
00866             previousOutputFile = OutputFile
00867         else:
00868             unprofiledSteps.append(step)
00869             isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
00870             
00871             try:
00872                 isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
00873                 #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
00874             except IndexError:
00875                 # This loop should have terminated if x + 1 is out of bounds!
00876                 print "Error: Something is wrong we shouldn't have come this far"
00877                 break
00878 
00879             if isNextStepForProfile:
00880                 #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
00881                 OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
00882                 unprofiledSteps = []
00883                 prevPrevOutputFile = previousOutputFile          
00884                 previousOutputFile = OutputFile
00885         #Dangerous index handling when looping over index x!        
00886         if oneShotProf:
00887             stepIndex += len(hypsteps)            
00888         else:
00889             stepIndex +=1
00890     return fstROOTfileStr

def cmsRelvalreportInput::writeCommandsToReport	(	simcandles,
		Candle,
		Profile,
		debug,
		NumberOfEvents,
		cmsDriverOptions,
		steps,
		bypasshlt,
		userInputFile
	)

Definition at line 891 of file cmsRelvalreportInput.py.

00892                                                                                                                         :
00893 
00894     OutputStep = ''
00895 
00896     # This option will not be used for now since the event content names keep changing, it can be
00897     # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
00898     #
00899     #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
00900 
00901     for acandle in Candle:
00902         print '*Candle ' + acandle
00903         
00904         ##################
00905         # If the first profiling we run is EdmSize we need to create the root file first
00906         #
00907 
00908         #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
00909         #print "1-userInputFile: %s"%userInputFile
00910         
00911         fstoutfile = writeCommands(simcandles,
00912                                    Profile,
00913                                    acandle,
00914                                    steps,
00915                                    NumberOfEvents,
00916                                    cmsDriverOptions,
00917                                    bypasshlt,
00918                                    0,
00919                                    False,
00920                                    userInputFile)
00921

def cmsRelvalreportInput::writePrerequisteSteps	(	simcandles,
		steps,
		acandle,
		NumberOfEvents,
		cmsDriverOptions,
		pileup,
		bypasshlt
	)

Definition at line 525 of file cmsRelvalreportInput.py.

00526                                                                                                     :
00527     fstIdx = -1
00528     #Hack for GEN-FASTSIM workflow:
00529     #if steps=="GEN,FASTSIM":   
00530     #if "-" in steps[0]:
00531     fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00532     #else:
00533     #    fstIdx = AllSteps.index(steps[0])
00534     CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
00535     previousOutputFile=""
00536     OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt) 
00537     return (fstIdx, OutputFile)

def cmsRelvalreportInput::writeReportFileHeader	(	CMSSW_VERSION,
		CMSSW_RELEASE_BASE,
		CMSSW_BASE
	)

Definition at line 359 of file cmsRelvalreportInput.py.

00360                                                                       :
00361 
00362     SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
00363     
00364     try:
00365         simcandles = open(SimCandlesFile, 'w')
00366     except IOError:
00367         print "Couldn't open %s to save" % SimCandlesFile
00368 
00369     simcandles.write('#Candles file automatically generated by %s for %s\n'
00370                       % (THIS_PROG_NAME, CMSSW_VERSION))
00371     simcandles.write("#CMSSW Base  : %s\n"   % CMSSW_BASE)
00372     simcandles.write("#Release Base: %s\n"   % CMSSW_RELEASE_BASE)
00373     simcandles.write("#Version     : %s\n\n" % CMSSW_VERSION)
00374 
00375     return simcandles

def cmsRelvalreportInput::writeStepHead	(	simcandles,
		acandle,
		step
	)

Definition at line 422 of file cmsRelvalreportInput.py.

00423                                           :
00424     simcandles.write('#%s\n' % FileName[acandle])
00425     simcandles.write('#Step %s\n' % step)
00426     #print step
00427

def cmsRelvalreportInput::writeUnprofiledSteps	(	simcandles,
		CustomisePythonFragment,
		cmsDriverOptions,
		unprofiledSteps,
		previousOutputFile,
		acandle,
		NumberOfEvents,
		stepIndex,
		pileup,
		bypasshlt
	)

Definition at line 478 of file cmsRelvalreportInput.py.

00479                                                                                                                                                                     :
00480     # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
00481     #stepsStr = ",".join(unprofiledSteps)
00482 
00483     
00484     #print "unprofiledSteps is %s"%unprofiledSteps
00485     #print "22acandle is %s"%acandle
00486     #Kludge in case -b option to skip HLT is used...
00487     if bypasshlt and unprofiledSteps[-1]=="HLT":
00488         stepsStr = ",".join(unprofiledSteps[:-1])
00489         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
00490     else:
00491         stepsStr = ",".join(unprofiledSteps)
00492         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
00493     simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
00494     OutputFileOption = "--fileout=%s" % OutputFile
00495     #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
00496     #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
00497     #Adding the argument bypasshlt to the calls...
00498     #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
00499     #print "StepIndex 1:%s"%stepIndex
00500     #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
00501     #stepIndexAdjust=stepIndex - 2
00502     #if stepIndexAdjust < 0: #To avoid issues with negative indeces
00503     #    stepIndexAdjust=0
00504     #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
00505     #Use an argument to make it easier:
00506     InputFileOption = "--filein file:" + previousOutputFile
00507     if previousOutputFile =="":
00508         InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00509     #Introduce an over-ride of cmsDriverOptions:
00510     #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
00511     #At the moment the only use case is when running step2 on its own...
00512     for eventcontent in EventContents:
00513         cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
00514     Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00515                        % (cmsDriver,
00516                           KeywordToCfi[acandle],
00517                           NumberOfEvents,
00518                           stepsStr,
00519                           InputFileOption,
00520                           OutputFileOption,
00521                           CustomisePythonFragment,
00522                           cmsDriverOptions) )
00523     simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
00524     return OutputFile

Variable Documentation

list cmsRelvalreportInput::AfterPileUpSteps = []

Definition at line 37 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput::AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]

Definition at line 36 of file cmsRelvalreportInput.py.

string cmsRelvalreportInput::cmsDriver = 'cmsDriver.py'

Definition at line 32 of file cmsRelvalreportInput.py.

cmsRelvalreportInput::debug = False

Definition at line 34 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput::DEF_STEPS = ['GEN,SIM', 'DIGI']

Definition at line 35 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput::hypreg = re.compile('-')

Definition at line 33 of file cmsRelvalreportInput.py.

dictionary cmsRelvalreportInput::Profiler

Initial value:

00001 {
00002     'TimingReport'            : 'Timing_Parser',
00003     'TimingReport @@@ reuse'  : 'Timing_Parser',
00004     'TimeReport'              : 'Timereport_Parser',
00005     'TimeReport @@@ reuse'    : 'Timereport_Parser',
00006     'SimpleMemReport'         : 'SimpleMem_Parser',
00007     'EdmSize'                 : 'Edm_Size',
00008     'IgProfperf'              : 'IgProf_perf.PERF_TICKS',
00009     'IgProfperf @@@ reuse'    : 'IgProf_perf.PERF_TICKS',
00010     'IgProfMemTotal'          : 'IgProf_mem.MEM_TOTAL',
00011     'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
00012     'IgProfMemLive'           : 'IgProf_mem.MEM_LIVE',
00013     'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
00014     'IgProfMemMax'            : 'IgProf_mem.MEM_MAX',
00015     'IgProfMemMax @@@ reuse'  : 'IgProf_mem.MEM_MAX',
00016     'IgProfMemAnalyse'        : 'IgProf_mem.ANALYSE',
00017     'valgrind'                : 'ValgrindFCE',
00018     'memcheck_valgrind'       : 'Memcheck_Valgrind',
00019     'None'                    : 'None',
00020 }

Definition at line 43 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput::THIS_PROG_NAME = os.path.basename(sys.argv[0])

Global variables.

Definition at line 31 of file cmsRelvalreportInput.py.

cmsRelvalreportInput Namespace Reference

Functions

Variables

Function Documentation

Variable Documentation