CMSSW Reference Manual

00001 #!/usr/bin/env python
00002 # -*- coding: utf-8 -*-
00003 # GBenelli Dec 21, 2007 and JNicolson July 23, 2008
00004 # This script is designed to run on a local directory
00005 # after the user has created a local CMSSW release,
00006 # initialized its environment variables by executing
00007 # in the release /src directory:
00008 # eval `scramv1 runtime -csh`
00009 # project CMSSW
00010 # The script will create a SimulationCandles_$CMSSW_VERSION.txt ASCII
00011 # file, input to cmsRelvalreport.py, to launch the
00012 # Performance Suite.
00013 
00014 # Input arguments are three:
00015 # 1-Number of events to put in the cfg files
00016 # 2-Name of the candle(s) to process (either AllCandles, or NameOfTheCandle)
00017 # 3-Profiles to run (with code below)
00018 # E.g.: ./cmsRelValReportInput.py 50 AllCandles 012
00019 
00020 ###############
00021 # Import modules and define functions
00022 #
00023 
00024 import sys, os, re, operator
00025 import optparse as opt
00026 from cmsPerfCommons import Candles, CandDesc, FileName, KeywordToCfi, CustomiseFragment, CandFname, EventContents
00027 ################
00028 # Global variables
00029 #
00030 
00031 THIS_PROG_NAME = os.path.basename(sys.argv[0])
00032 cmsDriver = 'cmsDriver.py'                    #cmsDriver.py path
00033 hypreg = re.compile('-')
00034 debug = False
00035 DEF_STEPS = ['GEN,SIM', 'DIGI']
00036 AllSteps  = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
00037 AfterPileUpSteps=[]
00038 
00039 # Global variables used by writeCommandsToReport and dependents
00040 
00041 # Hash for the profiler to run
00042 
00043 Profiler = {
00044     'TimingReport'            : 'Timing_Parser',
00045     'TimingReport @@@ reuse'  : 'Timing_Parser',
00046     'TimeReport'              : 'Timereport_Parser',
00047     'TimeReport @@@ reuse'    : 'Timereport_Parser',
00048     'SimpleMemReport'         : 'SimpleMem_Parser',
00049     'EdmSize'                 : 'Edm_Size',
00050     'IgProfperf'              : 'IgProf_perf.PERF_TICKS',
00051     'IgProfMemTotal'          : 'IgProf_mem.MEM_TOTAL',
00052     'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
00053     'IgProfMemLive'           : 'IgProf_mem.MEM_LIVE',
00054     'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
00055     'IgProfMemAnalyse'        : 'IgProf_mem.ANALYSE',
00056     'valgrind'                : 'ValgrindFCE',
00057     'memcheck_valgrind'       : 'Memcheck_Valgrind',
00058     'None'                    : 'None',
00059 }
00060 
00061 
00062 
00063 def getFstOccur(item, list):
00064     return filter(item.__eq__,list)[0]
00065 
00066 def getLstIndex(item, list):
00067     lenlist = len(list)
00068     for x in range(lenlist - 1,0,-1):
00069         if list[x] == item:
00070             return x
00071 
00072 def checkSteps(steps):
00073     idx = -2
00074     lstidx = -2
00075     for step in steps:
00076         astep = step
00077         split = []
00078         if "-" in step:
00079             split = astep.split("-")
00080             astep = split[0]
00081         idx = AllSteps.index(astep)
00082         if not ( idx == -2 ):
00083             if lstidx > idx:
00084                 print "ERROR: Your user defined steps are not in a valid order"
00085                 sys.exit()
00086         lstidx = idx    
00087         if "-" in step:
00088             lstidx = AllSteps.index(split[1])
00089 
00090 
00091 def getSteps(userSteps):
00092 
00093     # Then split the user steps into "steps"
00094     gsreg = re.compile('GEN-SIM')
00095     greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
00096     StepsTokens = userSteps.split(",")
00097     steps = [] 
00098     for astep in StepsTokens:
00099 
00100         # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
00101         # from using the "-" to using the "," to match cmsDriver.py convention
00102 
00103         if gsreg.search(astep):
00104             astep = gsreg.sub(r"GEN,SIM", astep)
00105         elif greg.search(astep):
00106             astep = greg.sub(r"GEN,SIM", astep)
00107             
00108         print astep
00109         # Finally collect all the steps into the @Steps array:
00110 
00111         steps.append(astep)
00112     
00113     #steps = expandHypens(steps)
00114     checkSteps(steps)
00115     return steps
00116         
00117 def optionparse():
00118     global _noprof
00119     explanations = map(lambda x: "          " + x, Candles)
00120     explanation  = ""
00121     for x in range(len(explanations)):
00122         explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
00123     parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
00124 
00125     Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
00126 
00127     Arguments:
00128         NUM_EVENTS_PER_CFG - The number of events per config file
00129 
00130         CANDLES - The simulation type to perform profiling on
00131           Candles              Description
00132           AllCandles           Run all of the candles below
00133 %s
00134 
00135         PROFILE - the type of profiling to perform (multiple codes can be used):
00136           Code  Profile Type
00137           0     TimingReport
00138           1     TimeReport
00139           2     SimpleMemoryCheck
00140           3     EdmSize
00141           4     IgProfPerf
00142           5     IgProfMemTotal
00143           6     IgProfMemLive
00144           7     IgProfAnalyse
00145           8     ValgrindFCE
00146           9     ValgrindMemCheck    
00147 
00148     Examples:  
00149        Perform Timing Report profiling for all candles and 10 events per cfg.
00150         ./%s 10 AllCandles 1
00151        Perform Timing Report, Time Report and SimpleMemoryCheck profiling for Higgs boson and 50 events.
00152         ./%s 50 \"HZZLLLL\" 012
00153        Perform IgProfPerf and IgProfMemTotal profiling for TTbar and 100 events.
00154         ./%s 100 \"TTBAR\" 45 --cmsdriver=\"--conditions FakeConditions\"
00155        Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on gensim and digi steps.
00156         ./%s 100 \"MINBIAS\" 89 --cmsdriver=\"--conditions FakeConditions\" \"--usersteps=GEN-SIM,DIGI"""
00157       % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
00158     
00159     devel  = opt.OptionGroup(parser, "Developer Options",
00160                                      "Caution: use these options at your own risk."
00161                                      "It is believed that some of them bite.\n")
00162     #parser.set_defaults(debug=False)
00163 
00164     parser.add_option(
00165         '-b',
00166         '--bypass-hlt',
00167         action="store_true",
00168         dest='bypasshlt',
00169         default=False,
00170         help='Should we bypass using the HLT root file?'
00171         )
00172 
00173     parser.add_option(
00174         '-u',
00175         '--usersteps',
00176         type='string',
00177         dest='userSteps',
00178         help='Which steps to run',
00179         metavar='<STEPS>',
00180         )
00181     parser.add_option(
00182         '-c',
00183         '--cmsdriver',
00184         type='string',
00185         dest='cmsDriverOptions',
00186         help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
00187         metavar='<CONDITION>',
00188         )
00189 
00190     devel.add_option(
00191         '-1',
00192         '--no-profile',
00193         action="store_true",
00194         dest='noprof',
00195         help='Do not perform profiling, ever',
00196         #metavar='DEBUG',
00197         )
00198     
00199     devel.add_option(
00200         '-d',
00201         '--debug',
00202         action="store_true",
00203         dest='debug',
00204         help='Show debug output',
00205         #metavar='DEBUG',
00206         )
00207 
00208     parser.set_defaults(debug=False,noprof=False)
00209     parser.add_option_group(devel)
00210 
00211     (options, args) = parser.parse_args()
00212     
00213 
00214     _noprof = options.noprof
00215     debug = options.debug
00216     numofargs = len(args) 
00217 #    print "runtime cmsinput " + str(numofargs) + " noprof "  + str(_noprof)
00218     if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
00219         parser.error("There are not enough arguments specified to run this program."
00220                      " Please determine the correct arguments from the usage information above."
00221                      " Run with --help option for more information.")
00222         sys.exit()
00223 
00224     return (options, args)
00225 
00226 def expandHyphens(step):
00227     newsteps = []
00228 #    for step in steps:
00229     if "-" in step:
00230         hypsteps = step.split(r"-")
00231         if not (len(hypsteps) == 2):
00232             print "ERROR: Could not parse usersteps. You have too many hypens between commas"
00233             sys.exit()
00234         elif not (hypsteps[0] in AllSteps and hypsteps[1] in AllSteps):
00235             print "ERROR: One of the steps you defined is invalid"
00236             sys.exit()
00237         else:
00238             if (hypsteps[0] == hypsteps[1]):
00239                 print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
00240                 newsteps.append(hypsteps[0])
00241             else:
00242                 newsteps.append(hypsteps[0])
00243                 srt = AllSteps.index(hypsteps[0]) + 1
00244                 for n in range(srt,len(AllSteps) - 1,1):
00245                     astep = AllSteps[n]
00246                     if astep == hypsteps[1]:
00247                         break
00248                     else:
00249                         newsteps.append(astep)
00250                 newsteps.append(hypsteps[1])
00251     else:
00252         if not (step in AllSteps):
00253             print "ERROR: One of the steps you defined is invalid"
00254             sys.exit()
00255         else:
00256             newsteps.append(step)
00257 
00258     return newsteps
00259 
00260 def setupProgramParameters(options,args):
00261     steps = []
00262     cmsDriverOptions = ""
00263     global AfterPileUpSteps
00264     NumberOfEvents = int(args[0])  # first arg
00265     WhichCandles   = str(args[1])  # second arg
00266     ProfileCode = ""
00267     if not _noprof:
00268         ProfileCode    = str(args[2])  # third arg
00269 
00270     if options.cmsDriverOptions:
00271 
00272         cmsDriverOptions = options.cmsDriverOptions
00273         print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
00274 
00275     if options.userSteps:
00276 
00277         userSteps = options.userSteps
00278         steps = getSteps(userSteps)
00279 
00280     if WhichCandles.lower() == 'allcandles':
00281         Candle = Candles
00282         print 'ALL standard simulation candles will be PROCESSED:'
00283     else:
00284         Candle = [WhichCandles]
00285         print 'ONLY %s will be PROCESSED' % Candle[0]
00286 
00287     # For now the two steps are built in, this can be added as an argument later
00288     # Added the argument option so now this will only be defined if it was not defined already:
00289 
00290     if not steps:
00291         print 'The default steps will be run:'
00292         steps = DEF_STEPS
00293     else:
00294         print "You defined your own steps to run:"
00295 
00296     for astep in steps:
00297         print astep
00298 
00299     return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt)
00300 
00301 def init_vars():
00302 
00303     ####################
00304     # Obtain required environment variables
00305     #
00306 
00307     try:
00308         CMSSW_BASE         = os.environ['CMSSW_BASE']
00309         CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
00310         CMSSW_VERSION      = os.environ['CMSSW_VERSION']
00311     except KeyError:
00312         print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
00313         print '       Please run eval `scramv1 runtime -csh` to set your environment variables'
00314         sys.exit()
00315 
00316     return ( CMSSW_BASE,
00317              CMSSW_RELEASE_BASE,
00318              CMSSW_VERSION)
00319 
00320 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
00321 
00322     SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
00323     
00324     try:
00325         simcandles = open(SimCandlesFile, 'w')
00326     except IOError:
00327         print "Couldn't open %s to save" % SimCandlesFile
00328 
00329     simcandles.write('#Candles file automatically generated by %s for %s\n'
00330                       % (THIS_PROG_NAME, CMSSW_VERSION))
00331     simcandles.write("#CMSSW Base  : %s\n"   % CMSSW_BASE)
00332     simcandles.write("#Release Base: %s\n"   % CMSSW_RELEASE_BASE)
00333     simcandles.write("#Version     : %s\n\n" % CMSSW_VERSION)
00334 
00335     return simcandles
00336 
00337 def getProfileArray(ProfileCode):
00338 
00339     Profile = []
00340 
00341     # The allowed profiles are:
00342 
00343     AllowedProfile = [
00344         'TimingReport',
00345         'TimeReport',
00346         'SimpleMemReport',
00347         'EdmSize',
00348         'IgProfperf',
00349         'IgProfMemTotal',
00350         'IgProfMemLive',
00351         'IgProfMemAnalyse',
00352         'valgrind',
00353         'memcheck_valgrind',
00354         'None',
00355         ]
00356 
00357     if _noprof:
00358         Profile.append(AllowedProfile[-1])
00359     else:
00360         for i in range(10):
00361             if str(i) in ProfileCode:
00362                 firstCase = i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode) or i == 1 and str(2) in ProfileCode
00363                 secCase   = i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode) or i == 6 and str(7) in ProfileCode
00364                 
00365                 if firstCase or secCase:
00366                     Profile.append(AllowedProfile[i] + ' @@@ reuse')
00367                 else:
00368                     Profile.append(AllowedProfile[i])
00369                 
00370     return Profile
00371 
00372 def writeStepHead(simcandles,acandle,step):
00373     simcandles.write('#%s\n' % FileName[acandle])
00374     simcandles.write('#Step %s\n' % step)
00375     print step
00376 
00377 
00378 def determineNewProfile(step,Profile,SavedProfile):
00379     if 'DIGI2RAW' in step:
00380         SavedProfile = Profile
00381         Profile = [ ]
00382     if 'HLT' in step:
00383         Profile = SavedProfile
00384 
00385     return (Profile, SavedProfile)
00386 
00387 def pythonFragment(step,cmsdriverOptions):
00388     # Convenient dictionary to map the correct customise Python fragment for cmsDriver.py:
00389     #It is now living in cmsPerfCommons.py!
00390 
00391  #   CustomiseFragment = {
00392  #        'GEN,SIM': 'Validation/Performance/TimeMemoryG4Info.py',
00393  #        'DIGI': 'Validation/Performance/TimeMemoryInfo.py',
00394  #        'DIGI-PILEUP':'Validation/Performance/MixingModule.py'}
00395     
00396     if "--pileup" in cmsdriverOptions:
00397         return CustomiseFragment['DIGI-PILEUP']
00398     elif CustomiseFragment.has_key(step):
00399         return CustomiseFragment[step] 
00400     else:
00401         #This is a safe default in any case,
00402         #the maintenance of the customise python fragments goes into cmsPerfCommons.py
00403         return CustomiseFragment['DIGI']
00404 
00405 
00406 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
00407     #print "StepIndex 2:%s"%stepIndex
00408     InputFileOption = ""
00409     if pileup and stepIndex == 0:
00410         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
00411     else:
00412         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
00413         
00414     if pileup:
00415         pass
00416     else :
00417         if 'GEN,SIM' in step:  # there is no input file for GEN,SIM!
00418             InputFileOption = ''
00419         elif   'HLT' in steps[stepIndex - 1] and bypasshlt:
00420 
00421             # Special hand skipping of HLT since it is not stable enough, so it will not prevent
00422             # RAW2DIGI,RECO from running
00423 
00424             InputFileOption = "--filein file:%s_%s"  % ( FileName[acandle],steps[stepIndex - 2] )
00425 
00426     if not InputFileOption == "" :
00427         InputFileOption += ".root "
00428 
00429     return InputFileOption
00430 
00431 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
00432     # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
00433     #stepsStr = ",".join(unprofiledSteps)
00434 
00435     
00436     #print "unprofiledSteps is %s"%unprofiledSteps
00437     #print "22acandle is %s"%acandle
00438     #Kludge in case -b option to skip HLT is used...
00439     if bypasshlt and unprofiledSteps[-1]=="HLT":
00440         stepsStr = ",".join(unprofiledSteps[:-1])
00441         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
00442     else:
00443         stepsStr = ",".join(unprofiledSteps)
00444         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
00445     simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
00446     OutputFileOption = "--fileout=%s" % OutputFile
00447     #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
00448     #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
00449     #Adding the argument bypasshlt to the calls...
00450     #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
00451     #print "StepIndex 1:%s"%stepIndex
00452     #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
00453     #stepIndexAdjust=stepIndex - 2
00454     #if stepIndexAdjust < 0: #To avoid issues with negative indeces
00455     #    stepIndexAdjust=0
00456     #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
00457     #Use an argument to make it easier:
00458     InputFileOption = "--filein file:" + previousOutputFile
00459     if previousOutputFile =="":
00460         InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00461     #Introduce an over-ride of cmsDriverOptions:
00462     #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
00463     #At the moment the only use case is when running step2 on its own...
00464     for eventcontent in EventContents:
00465         cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
00466     Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00467                        % (cmsDriver,
00468                           KeywordToCfi[acandle],
00469                           NumberOfEvents,
00470                           stepsStr,
00471                           InputFileOption,
00472                           OutputFileOption,
00473                           CustomisePythonFragment,
00474                           cmsDriverOptions) )
00475     simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
00476     return OutputFile
00477 
00478 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
00479     fstIdx = -1
00480     if "-" in steps[0]:
00481         fstIdx = AllSteps.index(steps[0].split("-")[0])
00482     else:
00483         fstIdx = AllSteps.index(steps[0])
00484     CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
00485     previousOutputFile=""
00486     OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt) 
00487     return (fstIdx, OutputFile)
00488 
00489 def setOutputFileOption(acandle,endstep):
00490     return "%s_%s.root" % ( FileName[acandle],endstep)
00491 
00492 def writeCommands(simcandles,
00493                   Profile,
00494                   acandle,
00495                   steps,
00496                   NumberOfEvents,
00497                   cmsDriverOptions,
00498                   bypasshlt,                  
00499                   stepIndex = 0,
00500                   pileup = False):
00501 
00502     OutputStep = ""
00503 
00504     stopIndex = len(steps)
00505     start = 0
00506 
00507     userSteps = steps
00508     SavedProfile = []
00509     fstROOTfile = True
00510     fstROOTfileStr = ""
00511 
00512     #Handling the case of the first user step not being the first step (GEN,SIM):
00513     print "Steps passed to writeCommands %s",steps
00514     if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
00515         #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
00516         (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
00517         
00518         #Now take care of setting the indeces and input root file name right for the profiling part...
00519         if fstROOTfile:
00520             fstROOTfileStr = rootFileStr
00521             fstROOTfile = False
00522         start = -1
00523         if "-" in steps[0]:
00524             start = AllSteps.index(steps[0].split("-")[0])
00525         else:
00526             start = AllSteps.index(steps[0])
00527         lst = - 1
00528         if "-" in steps[-1]:
00529             lst   = AllSteps.index(steps[-1].split("-")[1]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
00530         else:
00531             lst   = AllSteps.index(steps[-1])
00532         runSteps = AllSteps[start:lst]
00533         numOfSteps = (lst - start) + 1
00534         stopIndex = start + numOfSteps
00535         #Handling the case in which the first user step is the same as the first step (GEN,SIM)
00536         #elif  not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
00537     else:
00538         #Handling the case of the last step being a composite one:
00539         if "-" in steps[-1]:
00540             #Set the stop index at the last step of the composite step (WHY???) 
00541             stopIndex = AllSteps.index(steps[-1].split("-")[1]) + 1
00542         else:
00543             stopIndex = AllSteps.index(steps[-1]) + 1
00544             
00545     steps = AllSteps
00546             
00547     unprofiledSteps = []
00548     rawreg = re.compile("^RAW2DIGI")
00549 
00550 #Beginning of a new design of the loop (abandoned for now for lack of time... simply fixed the issues with unprofiled steps by adding one argument to the writeUnprofiledSteps and fixed a couple of inconsistencies with the -b option:
00551 ##    #CurrentStepIndex = 0
00552 ##    print "UserSteps:"
00553 ##    AllStepsIndex=-1
00554 ##    #Tests that can be done before looping:
00555 ##    #Check the first step to see if there are unprofiled pre-steps needed:
00556 ##   if not (userSteps[0] == AllSteps[0]) and (userSteps[0].split("-")[0] != "GEN,SIM"):
00557 ##           print "Need some pre-steps before starting with userSteps %s"%userSteps
00558 ##           print "Will run steps %s"%AllSteps[0:AllSteps.index(userSteps[0].split("-")[0])]
00559 ##           #implement here the running of unprofiled steps between GEN-SIM and AllSteps.index(steps[0].split("-")[0] -1)
00560 ##           #possibly calling a function WriteUnprofiledSteps(unprofiledSteps) that takes care of all (input, output filename)
00561 ##           
00562 ##   for step in userSteps:
00563 ##       print step
00564 ##       #Before establish the corresponding index in AllSteps:
00565 ##       
00566 ##       #Check if there are hypens (they mean we want to run all the steps between the hypens in one shot profiling them)
00567 ##       if "-" in step:
00568 ##           print "Hyphenated step: %s"%step
00569 ##           #implement profiling of steps a la cmsDriver.py, by translating the hypens:
00570 ##           #GEN-HLT -> GEN,SIM,DIGI,L1,DIGI2RAW,HLT
00571 ##           #Also make sure the output is called like the last step (in this case HLT.root)
00572 ##           print "Expanded hyphenated steps: %s"%expandHyphens(step)
00573 ##           FirstStep=expandHyphens(step)[0]
00574 ##           FirstStepIndex=AllSteps.index(FirstStep)
00575 ##           LastStep=expandHyphens(step)[1]
00576 ##           LastStepIndex=AllSteps.index(LastStep)
00577 ##           print "First step %s and last step %s"%(FirstStep,LastStep)
00578 ##           cmsDriverSteps=",".join(AllSteps[FirstStepIndex:LastStepIndex+1])
00579 ##           print "cmsDriverSteps = %s"%cmsDriverSteps
00580 ##           if '--pileup' in cmsDriverOptions:
00581 ##               outfile = LastStep + "_PILEUP"
00582 ##           else:
00583 ##               outfile = LastStep
00584 ##           OutputFile = setOutputFileOption(acandle,outfile)
00585 ##           print "OutputFile is %s"%OutputFile
00586 ##       if AllStepsIndex != -1:
00587 ##           try:
00588 ##               if AllSteps.index(step) == AllStepsIndex + 1:
00589 ##                   print "Consecutive steps to profile"
00590 ##               else:
00591 ##                   print "Step %s is not the consecutive step of %s, so we will need to run the intermediate ones unprofiled"%(AllSteps.index(step),AllStepsIndex)
00592 ##           except:#To catch the case of hyphenated steps
00593 ##               if AllSteps.index(expandHyphens(step)[0]) == AllStepsIndex + 1:
00594 ##                   print "Consecutive steps to profile"
00595 ##               else:
00596 ##                   print "Step %s is not the consecutive step of %s, so we will need to run the intermediate ones unprofiled"%(AllSteps.index(expandHyphens(step)[0]),AllStepsIndex)
00597 ##       #Finally establish the corresponding index in AllSteps:
00598 ##       try:
00599 ##           AllStepsIndex=AllSteps.index(step)
00600 ##       except: #To catch the case of hyphenated steps
00601 ##           AllStepsIndex=AllSteps.index(expandHyphens(step)[-1])
00602 ##
00603 ##
00604 ##       #CurrentStepIndex = CurrentStepIndex + 1
00605 
00606 #Horrible structure... to be rewritten sooner or later...
00607     
00608 #   FOR step in steps
00609 
00610     prevPrevOutputFile = ""
00611     previousOutputFile = ""
00612 
00613     for x in range(start,stopIndex,1):
00614         if stepIndex >= stopIndex:
00615             break
00616         step = steps[stepIndex]
00617 
00618         # One shot Profiling variables
00619         befStep     = step
00620         aftStep     = step
00621         # We need this in case we are running one-shot profiling or for DIGI-PILEUP
00622         stepToWrite = step 
00623         
00624         CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
00625         oneShotProf = False
00626         hypsteps = []
00627 
00628         #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
00629         if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0],userSteps)): 
00630 
00631             #Checking now if the current step matches the first of a composite step in userSteps
00632             hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
00633             if not len(hypMatch) == 0 :
00634                 hypsteps    = expandHyphens(hypMatch[0])
00635                 stepToWrite = ",".join(hypsteps)
00636                 befStep     = hypsteps[0]
00637                 #Kludge to avoid running HLT in composite steps if the -b option is chosen
00638                 if bypasshlt and hypsteps[-1]=='HLT':
00639                     aftStep     = hypsteps[-2]
00640                 else:
00641                     aftStep     = hypsteps[-1]
00642                 oneShotProf = True
00643 
00644             writeStepHead(simcandles,acandle,stepToWrite)
00645 
00646             #Set the output file name for Pile up and for regular case:
00647             if '--pileup' in cmsDriverOptions:
00648                 outfile = stepToWrite + "_PILEUP"
00649             else:
00650                 outfile = stepToWrite
00651                 
00652             OutputFile = setOutputFileOption(acandle,outfile)
00653             if fstROOTfile:
00654                 fstROOTfileStr = OutputFile
00655                 fstROOTfile = False
00656             OutputFileOption = "--fileout=" + OutputFile
00657 
00658             for prof in Profile:
00659                 #First prepare the cmsDriver.py command
00660                 
00661                 #Special case of EventEdmSize profiling 
00662                 if 'EdmSize' in prof:
00663                     EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
00664                     #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
00665                     if prof == Profile[0] and not os.path.exists("./" + EdmFile):
00666                         # insert command to generate required state ( need to run one more step
00667                         # so that EDM can actually check the size of the root file
00668                         # If the first step to be profiled is something later on in the steps such
00669                         # as HLT then writePrerequisteSteps() should have got you to the step prior to
00670                         # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
00671 
00672                         InputFileOption = "--filein file:" + previousOutputFile
00673                         #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
00674                         if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00675                             InputFileOption = "--filein file:" + prevPrevOutputFile
00676                         if previousOutputFile == "":
00677                             InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00678                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00679                                            % (cmsDriver,
00680                                               KeywordToCfi[acandle],
00681                                               NumberOfEvents,
00682                                               stepToWrite,
00683                                               InputFileOption,
00684                                               OutputFileOption,
00685                                               CustomisePythonFragment,
00686                                               cmsDriverOptions) )
00687                         simcandles.write( "%s @@@ None @@@ None\n" % (Command))
00688 
00689                     Command = EdmFile
00690                 #all other profiles:
00691                 else:
00692                     InputFileOption = "--filein file:" + previousOutputFile
00693                     if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00694                         InputFileOption = "--filein file:" + prevPrevOutputFile
00695 
00696                     if previousOutputFile == "":
00697                         InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
00698 
00699                     #if '--pileup' in cmsDriverOptions:
00700                     #    stepToWrite = pileupStep
00701                     if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'):
00702                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00703                                cmsDriver,
00704                                KeywordToCfi[acandle],
00705                                NumberOfEvents,
00706                                stepToWrite,
00707                                InputFileOption,
00708                                OutputFileOption,
00709                                CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00710                                cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
00711                            ))
00712                     else:
00713                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00714                                cmsDriver,
00715                                KeywordToCfi[acandle],
00716                                NumberOfEvents,
00717                                stepToWrite,
00718                                InputFileOption,
00719                                OutputFileOption,
00720                                CustomisePythonFragment,
00721                                cmsDriverOptions
00722                            ))
00723                 #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
00724                 if _noprof:
00725                     simcandles.write("%s @@@ None @@@ None\n" % Command)
00726                 else:
00727                     stepLabel=stepToWrite
00728                     if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
00729                         stepLabel = stepToWrite+"_PILEUP"
00730                     simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
00731                                                                    Profiler[prof],
00732                                                                    FileName[acandle],
00733                                                                    stepLabel,
00734                                                                    prof))
00735 
00736                 if debug:
00737                     print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
00738                     print "cmsDriveroptions : " + cmsDriverOption
00739             prevPrevOutputFile = previousOutputFile          
00740             previousOutputFile = OutputFile
00741         else:
00742             unprofiledSteps.append(step)
00743             isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
00744             
00745             try:
00746                 isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0],userSteps))
00747             except IndexError:
00748                 # This loop should have terminated if x + 1 is out of bounds!
00749                 print "Error: Something is wrong we shouldn't have come this far"
00750                 break
00751 
00752             if isNextStepForProfile:
00753                 #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
00754                 OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
00755                 unprofiledSteps = []
00756                 prevPrevOutputFile = previousOutputFile          
00757                 previousOutputFile = OutputFile
00758         #Dangerous index handling when looping over index x!        
00759         if oneShotProf:
00760             stepIndex += len(hypsteps)            
00761         else:
00762             stepIndex +=1
00763     return fstROOTfileStr
00764 
00765 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt):
00766 
00767     OutputStep = ''
00768 
00769     # This option will not be used for now since the event content names keep changing, it can be
00770     # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
00771     #
00772     #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
00773 
00774     for acandle in Candle:
00775         print '*Candle ' + acandle
00776         
00777         ##################
00778         # If the first profiling we run is EdmSize we need to create the root file first
00779         #
00780 
00781         #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
00782         fstoutfile = writeCommands(simcandles,
00783                                    Profile,
00784                                    acandle,
00785                                    steps,
00786                                    NumberOfEvents,
00787                                    cmsDriverOptions,
00788                                    bypasshlt)
00789         
00790 
00791 def main(argv=sys.argv):
00792 
00793     #####################
00794     # Options Parser 
00795     #
00796 
00797     (options, args) = optionparse()
00798 
00799     #####################
00800     # Set up arguments and option handling
00801     #
00802 
00803     (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt ) = setupProgramParameters(options,args)
00804 
00805     ######################
00806     # Initialize a few variables
00807     #
00808 
00809     (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
00810 
00811     ##################
00812     # Ok everything is ready now we need to create the input file for the relvalreport script 
00813     #
00814 
00815     simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
00816 
00817     ##################
00818     # Based on the profile code create the array of profiles to run:
00819     #
00820 
00821     Profile = getProfileArray(ProfileCode)
00822 
00823     ##################
00824     # Write the commands for the report to the file
00825     #
00826 
00827     writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt)
00828                 
00829     simcandles.close()
00830 
00831 if __name__ == "__main__":
00832     main()
cmsRelvalreportInput.py
