Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 # -*- coding: utf-8 -*-
00003 # GBenelli Dec 21, 2007 and JNicolson July 23, 2008
00004 # This script is designed to run on a local directory
00005 # after the user has created a local CMSSW release,
00006 # initialized its environment variables by executing
00007 # in the release /src directory:
00008 # eval `scramv1 runtime -csh`
00009 # project CMSSW
00010 # The script will create a SimulationCandles_$CMSSW_VERSION.txt ASCII
00011 # file, input to cmsRelvalreport.py, to launch the
00012 # Performance Suite.
00013 
00014 # Input arguments are three:
00015 # 1-Number of events to put in the cfg files
00016 # 2-Name of the candle(s) to process (either AllCandles, or NameOfTheCandle)
00017 # 3-Profiles to run (with code below)
00018 # E.g.: ./cmsRelValReportInput.py 50 AllCandles 012
00019 
00020 ###############
00021 # Import modules and define functions
00022 #
00023 
00024 import sys, os, re, operator
00025 import optparse as opt
00026 from cmsPerfCommons import Candles, CandDesc, FileName, KeywordToCfi, CustomiseFragment, CandFname, EventContents
00027 ################
00028 # Global variables
00029 #
00030 
00031 THIS_PROG_NAME = os.path.basename(sys.argv[0])
00032 cmsDriver = 'cmsDriver.py'                    #cmsDriver.py path
00033 hypreg = re.compile('-')
00034 debug = False
00035 DEF_STEPS = ['GEN,SIM', 'DIGI']
00036 AllSteps  = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
00037 AfterPileUpSteps=[]
00038 
00039 # Global variables used by writeCommandsToReport and dependents
00040 
00041 # Hash for the profiler to run
00042 
00043 Profiler = {
00044     'TimingReport'            : 'Timing_Parser',
00045     'TimingReport @@@ reuse'  : 'Timing_Parser',
00046     'TimeReport'              : 'Timereport_Parser',
00047     'TimeReport @@@ reuse'    : 'Timereport_Parser',
00048     'SimpleMemReport'         : 'SimpleMem_Parser',
00049     'EdmSize'                 : 'Edm_Size',
00050     'IgProfperf'              : 'IgProf_perf.PERF_TICKS',
00051     'IgProfperf @@@ reuse'    : 'IgProf_perf.PERF_TICKS',
00052     'IgProfMemTotal'          : 'IgProf_mem.MEM_TOTAL',
00053     'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
00054     'IgProfMemLive'           : 'IgProf_mem.MEM_LIVE',
00055     'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
00056     'IgProfMemMax'            : 'IgProf_mem.MEM_MAX',
00057     'IgProfMemMax @@@ reuse'  : 'IgProf_mem.MEM_MAX',
00058     'IgProfMemAnalyse'        : 'IgProf_mem.ANALYSE',
00059     'valgrind'                : 'ValgrindFCE',
00060     'memcheck_valgrind'       : 'Memcheck_Valgrind',
00061     'None'                    : 'None',
00062 }
00063 
00064 
00065 
00066 def getFstOccur(item, list):
00067     return filter(item.__eq__,list)[0]
00068 
00069 def getLstIndex(item, list):
00070     lenlist = len(list)
00071     for x in range(lenlist - 1,0,-1):
00072         if list[x] == item:
00073             return x
00074 
00075 def checkSteps(steps):
00076     '''Checks user steps <steps> for order against steps defined in AllSteps list.
00077     If they are not, prints a warning and exits.
00078     No return value.'''
00079     
00080     #print steps
00081     #if ":" in steps:
00082     #    steps.split(
00083     idx = -2
00084     lstidx = -2
00085     for step in steps:
00086         astep = step
00087         split = []
00088         #Added special workflow for GEN-FASTSIM, bypass the check of step order:
00089         if step=="GEN,FASTSIM":
00090             pass
00091         else:
00092             if "-" in step:
00093                 split = astep.split("-")
00094                 astep = split[0].split(":")[0]
00095              #   print "This is astep:%s"%astep
00096             #idx =AllSteps.index("GEN,SIM")
00097             #print idx
00098             #print astep
00099             idx = AllSteps.index(astep.split(":")[0])            
00100             if not ( idx == -2 ):
00101                 if lstidx > idx:
00102                     print "ERROR: Your user defined steps are not in a valid order"
00103                     sys.exit()
00104                 lstidx = idx    
00105             if "-" in step:
00106                 lstidx = AllSteps.index(split[1].split(":")[0])
00107 
00108 
00109 def getSteps(userSteps):
00110     '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
00111     Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
00112     Returns a list of steps <steps>.'''
00113     #Add handling of ":" in steps
00114     #print userSteps
00115     # Then split the user steps into "steps"
00116     gsreg = re.compile('GEN-SIM')
00117     greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
00118     gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
00119     StepsTokens = userSteps.split(",")
00120     steps = [] 
00121     for astep in StepsTokens:
00122 
00123         # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
00124         # from using the "-" to using the "," to match cmsDriver.py convention
00125 
00126         if gsreg.search(astep):
00127             astep = gsreg.sub(r"GEN,SIM", astep)
00128         elif gfsreg.search(astep):
00129             astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
00130         elif greg.search(astep):
00131             astep = greg.sub(r"GEN,SIM", astep)
00132             
00133             
00134         #print astep
00135         # Finally collect all the steps into the steps list:
00136 
00137         steps.append(astep)
00138     
00139     #steps = expandHypens(steps)
00140     checkSteps(steps)
00141     return steps
00142         
00143 def optionparse():
00144     global _noprof
00145     explanations = map(lambda x: "          " + x, Candles)
00146     explanation  = ""
00147     for x in range(len(explanations)):
00148         explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
00149     parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
00150 
00151     Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
00152 
00153     Arguments:
00154         NUM_EVENTS_PER_CFG - The number of events per config file
00155 
00156         CANDLES - The simulation type to perform profiling on
00157           Candles              Description
00158           AllCandles           Run all of the candles below
00159 %s
00160 
00161         PROFILE - the type of profiling to perform (multiple codes can be used):
00162           Code  Profile Type
00163           0     TimingReport
00164           1     TimeReport
00165           2     SimpleMemoryCheck
00166           3     EdmSize
00167           4     IgProfPerf
00168           5     IgProfMemTotal
00169           6     IgProfMemLive
00170           7     IgProfAnalyse
00171           8     ValgrindFCE
00172           9     ValgrindMemCheck    
00173 
00174     Examples (could be obsolete!):  
00175        Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
00176         ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00177        Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
00178         ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00179        Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
00180         ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
00181        Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
00182         ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
00183       % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
00184     
00185     devel  = opt.OptionGroup(parser, "Developer Options",
00186                                      "Caution: use these options at your own risk."
00187                                      "It is believed that some of them bite.\n")
00188     #parser.set_defaults(debug=False)
00189 
00190     parser.add_option(
00191         '-b',
00192         '--bypass-hlt',
00193         action="store_true",
00194         dest='bypasshlt',
00195         default=False,
00196         help='Should we bypass using the HLT root file?'
00197         )
00198 
00199     parser.add_option(
00200         '-u',
00201         '--usersteps',
00202         type='string',
00203         dest='userSteps',
00204         help='Which steps to run',
00205         metavar='<STEPS>',
00206         )
00207     parser.add_option(
00208         '-c',
00209         '--cmsdriver',
00210         type='string',
00211         dest='cmsDriverOptions',
00212         help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
00213         metavar='<CONDITION>',
00214         )
00215     parser.add_option(
00216         '-i',
00217         '--filein',
00218         type='string',
00219         dest='userInputFile',
00220         help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
00221         default='',
00222         metavar='<INPUTFILE>',
00223         )
00224     devel.add_option(
00225         '-1',
00226         '--no-profile',
00227         action="store_true",
00228         dest='noprof',
00229         help='Do not perform profiling, ever',
00230         #metavar='DEBUG',
00231         )
00232     
00233     devel.add_option(
00234         '-d',
00235         '--debug',
00236         action="store_true",
00237         dest='debug',
00238         help='Show debug output',
00239         #metavar='DEBUG',
00240         )
00241 
00242     parser.set_defaults(debug=False,noprof=False)
00243     parser.add_option_group(devel)
00244 
00245     (options, args) = parser.parse_args()
00246     
00247 
00248     _noprof = options.noprof
00249     debug = options.debug
00250     numofargs = len(args) 
00251 #    print "runtime cmsinput " + str(numofargs) + " noprof "  + str(_noprof)
00252     if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
00253         parser.error("There are not enough arguments specified to run this program."
00254                      " Please determine the correct arguments from the usage information above."
00255                      " Run with --help option for more information.")
00256         sys.exit()
00257 
00258     return (options, args)
00259 
00260 def expandHyphens(step):
00261     newsteps = []
00262 #    for step in steps:
00263     if "-" in step:
00264         hypsteps = step.split(r"-")
00265         #if not (len(hypsteps) == 2):
00266         #    print "ERROR: Could not parse usersteps. You have too many hypens between commas"
00267         #    sys.exit()
00268         #el
00269         if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
00270         #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
00271             print "ERROR: One of the steps you defined is invalid"
00272             sys.exit()
00273         else:
00274             if (hypsteps[0] == hypsteps[1]):
00275                 print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
00276                 newsteps.append(hypsteps[0])
00277             else:
00278                 newsteps.append(hypsteps[0])
00279                 
00280                 srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
00281                 for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
00282                     astep = AllSteps[n]
00283                     if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
00284                         #break
00285                         astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
00286                         if astep==hypsteps[-1].split(":")[0]:
00287                             break
00288                     newsteps.append(astep)
00289                 newsteps.append(hypsteps[-1])
00290     else:
00291         if not (step.split(":")[0] in AllSteps):
00292             print "ERROR: One of the steps you defined is invalid"
00293             sys.exit()
00294         else:
00295             newsteps.append(step)
00296 
00297     return newsteps
00298 
00299 def setupProgramParameters(options,args):
00300     steps = []
00301     cmsDriverOptions = ""
00302     global AfterPileUpSteps
00303     NumberOfEvents = int(args[0])  # first arg
00304     WhichCandles   = str(args[1])  # second arg
00305     ProfileCode = ""
00306     if not _noprof:
00307         ProfileCode    = str(args[2])  # third arg
00308 
00309     if options.cmsDriverOptions:
00310 
00311         cmsDriverOptions = options.cmsDriverOptions
00312         print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
00313 
00314     if options.userSteps:
00315 
00316         userSteps = options.userSteps
00317         steps = getSteps(userSteps)
00318 
00319     if WhichCandles.lower() == 'allcandles':
00320         Candle = Candles
00321         print 'ALL standard simulation candles will be PROCESSED:'
00322     else:
00323         Candle = [WhichCandles]
00324         print 'Candle %s will be PROCESSED' % Candle[0]
00325 
00326     # For now the two steps are built in, this can be added as an argument later
00327     # Added the argument option so now this will only be defined if it was not defined already:
00328 
00329     if not steps:
00330         print 'The default steps will be run:'
00331         steps = DEF_STEPS
00332     else:
00333         print "You defined your own steps to run:"
00334 
00335     for astep in steps:
00336         print astep
00337 
00338     return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
00339 
00340 def init_vars():
00341 
00342     ####################
00343     # Obtain required environment variables
00344     #
00345 
00346     try:
00347         CMSSW_BASE         = os.environ['CMSSW_BASE']
00348         CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
00349         CMSSW_VERSION      = os.environ['CMSSW_VERSION']
00350     except KeyError:
00351         print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
00352         print '       Please run eval `scramv1 runtime -csh` to set your environment variables'
00353         sys.exit()
00354 
00355     return ( CMSSW_BASE,
00356              CMSSW_RELEASE_BASE,
00357              CMSSW_VERSION)
00358 
00359 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
00360 
00361     SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
00362     
00363     try:
00364         simcandles = open(SimCandlesFile, 'w')
00365     except IOError:
00366         print "Couldn't open %s to save" % SimCandlesFile
00367 
00368     simcandles.write('#Candles file automatically generated by %s for %s\n'
00369                       % (THIS_PROG_NAME, CMSSW_VERSION))
00370     simcandles.write("#CMSSW Base  : %s\n"   % CMSSW_BASE)
00371     simcandles.write("#Release Base: %s\n"   % CMSSW_RELEASE_BASE)
00372     simcandles.write("#Version     : %s\n\n" % CMSSW_VERSION)
00373 
00374     return simcandles
00375 
00376 def getProfileArray(ProfileCode):
00377 
00378     Profile = []
00379 
00380     # The allowed profiles are:
00381 
00382     AllowedProfile = [
00383         'TimingReport',
00384         'TimeReport',
00385         'SimpleMemReport',
00386         'EdmSize',
00387         'IgProfperf',
00388         'IgProfMemTotal',
00389         'IgProfMemLive',
00390         #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
00391         'IgProfMemAnalyse',
00392         'valgrind',
00393         'memcheck_valgrind',
00394         'None',
00395         ]
00396 
00397     if _noprof:
00398         Profile.append(AllowedProfile[-1])
00399     else:
00400         #FIXME:
00401         #Horrible code!
00402         for i in range(10):
00403             if str(i) in ProfileCode:
00404                 firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
00405                             or (i == 1 and str(2) in ProfileCode)
00406                 secCase   = (i==4 and str(7) in ProfileCode) \
00407                             or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
00408                             or (i == 6 and str(7) in ProfileCode)
00409                 
00410                 if firstCase or secCase:
00411                     Profile.append(AllowedProfile[i] + ' @@@ reuse')
00412                     #Here's the hack:
00413                     if i==6: #i.e. IgProfMemLive
00414                         Profile.append("IgProfMemMax @@@ reuse")
00415                 else:
00416                     Profile.append(AllowedProfile[i])
00417                     if i==6: #i.e. IgProfMemLive
00418                         Profile.append("IgProfMemMax")
00419 
00420     return Profile
00421 
00422 def writeStepHead(simcandles,acandle,step):
00423     simcandles.write('#%s\n' % FileName[acandle])
00424     simcandles.write('#Step %s\n' % step)
00425     #print step
00426 
00427 
00428 def determineNewProfile(step,Profile,SavedProfile):
00429     if 'DIGI2RAW' in step:
00430         SavedProfile = Profile
00431         Profile = [ ]
00432     if 'HLT' in step:
00433         Profile = SavedProfile
00434 
00435     return (Profile, SavedProfile)
00436 
00437 def pythonFragment(step,cmsdriverOptions):
00438     # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
00439     #It is now living in cmsPerfCommons.py!
00440     
00441     if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
00442         return CustomiseFragment['DIGI-PILEUP']
00443     elif CustomiseFragment.has_key(step):
00444         return CustomiseFragment[step] 
00445     else:
00446         #This is a safe default in any case,
00447         #the maintenance of the customise python fragments goes into cmsPerfCommons.py
00448         return CustomiseFragment['DIGI']
00449 
00450 
00451 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
00452     #print "StepIndex 2:%s"%stepIndex
00453     InputFileOption = ""
00454     if pileup and stepIndex == 0:
00455         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
00456     else:
00457         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
00458         
00459     if pileup:
00460         pass
00461     else :
00462         if 'GEN,SIM' in step:  # there is no input file for GEN,SIM!
00463             InputFileOption = ''
00464         if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
00465             InputFileOption = ''
00466         elif   'HLT' in steps[stepIndex - 1] and bypasshlt:
00467 
00468             # Special hand skipping of HLT since it is not stable enough, so it will not prevent
00469             # RAW2DIGI,RECO from running
00470 
00471             InputFileOption = "--filein file:%s_%s"  % ( FileName[acandle],steps[stepIndex - 2] )
00472 
00473     if not InputFileOption == "" :
00474         InputFileOption += ".root "
00475 
00476     return InputFileOption
00477 
00478 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
00479     # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
00480     #stepsStr = ",".join(unprofiledSteps)
00481 
00482     
00483     #print "unprofiledSteps is %s"%unprofiledSteps
00484     #print "22acandle is %s"%acandle
00485     #Kludge in case -b option to skip HLT is used...
00486     if bypasshlt and unprofiledSteps[-1]=="HLT":
00487         stepsStr = ",".join(unprofiledSteps[:-1])
00488         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
00489     else:
00490         stepsStr = ",".join(unprofiledSteps)
00491         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
00492     simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
00493     OutputFileOption = "--fileout=%s" % OutputFile
00494     #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
00495     #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
00496     #Adding the argument bypasshlt to the calls...
00497     #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
00498     #print "StepIndex 1:%s"%stepIndex
00499     #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
00500     #stepIndexAdjust=stepIndex - 2
00501     #if stepIndexAdjust < 0: #To avoid issues with negative indeces
00502     #    stepIndexAdjust=0
00503     #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
00504     #Use an argument to make it easier:
00505     InputFileOption = "--filein file:" + previousOutputFile
00506     if previousOutputFile =="":
00507         InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00508     #Introduce an over-ride of cmsDriverOptions:
00509     #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
00510     #At the moment the only use case is when running step2 on its own...
00511     for eventcontent in EventContents:
00512         cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
00513     Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00514                        % (cmsDriver,
00515                           KeywordToCfi[acandle],
00516                           NumberOfEvents,
00517                           stepsStr,
00518                           InputFileOption,
00519                           OutputFileOption,
00520                           CustomisePythonFragment,
00521                           cmsDriverOptions) )
00522     simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
00523     return OutputFile
00524 
00525 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
00526     fstIdx = -1
00527     #Hack for GEN-FASTSIM workflow:
00528     #if steps=="GEN,FASTSIM":   
00529     #if "-" in steps[0]:
00530     fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00531     #else:
00532     #    fstIdx = AllSteps.index(steps[0])
00533     CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
00534     previousOutputFile=""
00535     OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt) 
00536     return (fstIdx, OutputFile)
00537 
00538 def setOutputFileOption(acandle,endstep):
00539     return "%s_%s.root" % ( FileName[acandle],endstep)
00540 
00541 def writeCommands(simcandles,
00542                   Profile,
00543                   acandle,
00544                   steps,
00545                   NumberOfEvents,
00546                   cmsDriverOptions,
00547                   bypasshlt,                  
00548                   stepIndex = 0,
00549                   pileup = False,
00550                   userInputFile=""):
00551     #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
00552     OutputStep = ""
00553 
00554     stopIndex = len(steps)
00555     start = 0
00556 
00557     userSteps = steps
00558     SavedProfile = []
00559     fstROOTfile = True
00560     fstROOTfileStr = ""
00561 
00562     #Handling the case of the first user step not being the first step (GEN,SIM):
00563     #print "Steps passed to writeCommands %s",steps
00564     #Special case for FASTSIM:
00565     if (steps[0] == "GEN,FASTSIM"):
00566         start=-1
00567         stopIndex=1
00568         steps = ["GEN,FASTSIM"]
00569     else:
00570         if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
00571             #print "userInputFile: %s"%userInputFile
00572             if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
00573                 userInputFile = "../INPUT_PILEUP_EVENTS.root"
00574                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00575                 rootFileStr=""
00576                 
00577             elif userInputFile == "":
00578                 #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
00579                 (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
00580             else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
00581                 #if "-" in steps[0]:
00582                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00583                     #print "Here!"
00584                 #else:
00585                     #print "There!"
00586                 #    stepIndex=AllSteps.index(steps[0])
00587                 rootFileStr=""
00588             #Now take care of setting the indeces and input root file name right for the profiling part...
00589             if fstROOTfile:
00590                 fstROOTfileStr = rootFileStr
00591                 fstROOTfile = False
00592             start = -1
00593             #if "-" in steps[0]:
00594             start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00595             #else:
00596             #    start = AllSteps.index(steps[0])
00597             lst = - 1
00598             if "-" in steps[-1]:
00599                 lst   = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
00600             else:
00601                 lst   = AllSteps.index(steps[-1])
00602             runSteps = AllSteps[start:lst]
00603             numOfSteps = (lst - start) + 1
00604             stopIndex = start + numOfSteps
00605             #Handling the case in which the first user step is the same as the first step (GEN,SIM)
00606             #elif  not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
00607         else:
00608             #Handling the case of the last step being a composite one:
00609             if "-" in steps[-1]:
00610                 #Set the stop index at the last step of the composite step (WHY???) 
00611                 stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
00612             else:
00613                 
00614                 stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
00615                 #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
00616         steps = AllSteps
00617             
00618     unprofiledSteps = []
00619     rawreg = re.compile("^RAW2DIGI")
00620 
00621 #Horrible structure... to be rewritten sooner or later...
00622     
00623 #   FOR step in steps
00624 
00625     prevPrevOutputFile = ""
00626     #Add if to introduce the possibility to run on user selected input file.
00627     if userInputFile!="":
00628         previousOutputFile=userInputFile
00629     else:
00630         previousOutputFile = ""
00631 
00632     for x in range(start,stopIndex,1):
00633         if stepIndex >= stopIndex:
00634             break
00635         step = steps[stepIndex]
00636 
00637         # One shot Profiling variables
00638         befStep     = step
00639         aftStep     = step
00640         # We need this in case we are running one-shot profiling or for DIGI-PILEUP
00641         stepToWrite = step 
00642         
00643         CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
00644         oneShotProf = False
00645         hypsteps = []
00646 
00647         #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
00648         if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)): 
00649 
00650             if "GEN,FASTSIM" in userSteps: # HLT/L1 menu workaround
00651                 stepToWrite = stepToWrite + ",HLT:GRun"
00652             elif "HLT" in userSteps:
00653                 stepToWrite = stepToWrite.replace("HLT", "HLT:GRun")
00654             
00655             #Checking now if the current step matches the first of a composite step in userSteps
00656             hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
00657             if not len(hypMatch) == 0 :
00658                 hypsteps    = expandHyphens(hypMatch[0])
00659                 #print hypsteps
00660                 #print hypMatch[0]
00661                 stepToWrite = ",".join(hypsteps)
00662                 if "GEN,SIM-HLT" in userSteps: # HLT/L1 menu workaround
00663                     stepToWrite = stepToWrite.replace("HLT","HLT:GRun")
00664                 befStep     = hypsteps[0]
00665                 #Kludge to avoid running HLT in composite steps if the -b option is chosen
00666                 if bypasshlt and hypsteps[-1]=='HLT':
00667                     aftStep     = hypsteps[-2]
00668                 else:
00669                     aftStep     = hypsteps[-1]
00670                 oneShotProf = True
00671             if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
00672                 #print "Inside the filter if!"
00673                 #print stepToWrite
00674                 #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
00675                 #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
00676                 #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!) 
00677                 stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
00678                 #print stepToWrite
00679 
00680             writeStepHead(simcandles,acandle,stepToWrite)
00681 
00682             #Set the output file name for Pile up and for regular case:
00683             if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change to make root file output work (without colons)
00684                 stepToWrite = stepToWrite.replace(":","=")
00685 
00686             if '--pileup' in cmsDriverOptions:
00687                 outfile = stepToWrite + "_PILEUP"
00688             else:
00689                 outfile = stepToWrite
00690 
00691             #print stepToWrite
00692 
00693             if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change it back
00694                 stepToWrite = stepToWrite.replace("=",":")
00695 
00696             OutputFile = setOutputFileOption(acandle,outfile)
00697             if fstROOTfile:
00698                 fstROOTfileStr = OutputFile
00699                 fstROOTfile = False
00700             OutputFileOption = "--fileout=" + OutputFile
00701 
00702             for prof in Profile:
00703                 #First prepare the cmsDriver.py command
00704                 
00705                 #Special case of EventEdmSize profiling 
00706                 if 'EdmSize' in prof:
00707                     EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
00708                     #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
00709                     if prof == Profile[0] and not os.path.exists("./" + EdmFile):
00710                         # insert command to generate required state ( need to run one more step
00711                         # so that EDM can actually check the size of the root file
00712                         # If the first step to be profiled is something later on in the steps such
00713                         # as HLT then writePrerequisteSteps() should have got you to the step prior to
00714                         # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
00715 
00716                         InputFileOption = "--filein file:" + previousOutputFile
00717                         #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
00718                         if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00719                             InputFileOption = "--filein file:" + prevPrevOutputFile
00720                         if previousOutputFile == "":
00721                             InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00722                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00723                                            % (cmsDriver,
00724                                               KeywordToCfi[acandle],
00725                                               NumberOfEvents,
00726                                               stepToWrite,
00727                                               InputFileOption,
00728                                               OutputFileOption,
00729                                               CustomisePythonFragment,
00730                                               cmsDriverOptions) )
00731                         simcandles.write( "%s @@@ None @@@ None\n" % (Command))
00732 
00733                     Command = EdmFile
00734                 #all other profiles:
00735                 else:
00736                     InputFileOption = "--filein file:" + previousOutputFile
00737                     if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00738                         InputFileOption = "--filein file:" + prevPrevOutputFile
00739 
00740                     if previousOutputFile == "":
00741                         InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
00742 
00743                     #if '--pileup' in cmsDriverOptions:
00744                     #    stepToWrite = pileupStep
00745                     if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'): 
00746                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00747                                cmsDriver,
00748                                KeywordToCfi[acandle],
00749                                NumberOfEvents,
00750                                stepToWrite,
00751                                InputFileOption,
00752                                OutputFileOption,
00753                                CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00754                                cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
00755                            ))
00756                     elif '--pileup' in cmsDriverOptions and (stepToWrite=='GEN,FASTSIM' or stepToWrite=='GEN,FASTSIM,HLT:GRun'): #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
00757                                                                                                                                 #+exception to the HLT:GRun case
00758                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00759                                cmsDriver,
00760                                KeywordToCfi[acandle],
00761                                NumberOfEvents,
00762                                stepToWrite,
00763                                InputFileOption,
00764                                OutputFileOption,
00765                                CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00766                                cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')] 
00767                            ))
00768                     
00769                     else:
00770                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00771                                cmsDriver,
00772                                KeywordToCfi[acandle],
00773                                NumberOfEvents,
00774                                stepToWrite,
00775                                InputFileOption,
00776                                OutputFileOption,
00777                                CustomisePythonFragment,
00778                                cmsDriverOptions
00779                            ))
00780                 #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
00781                 if _noprof:
00782                     simcandles.write("%s @@@ None @@@ None\n" % Command)
00783                 else:
00784                     if 'valgrind' in prof:
00785                         #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
00786                         if 'TimeMemoryInfo.py' in Command:
00787                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
00788                         #There is actually also the SIM case to take care of:
00789                         elif 'TimeMemoryG4Info.py' in Command:
00790                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
00791                     stepLabel=stepToWrite
00792                     #print "stepLabel is %s"%stepLabel
00793                     if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
00794                         stepLabel = stepToWrite+"_PILEUP"
00795                     #Add kludge here to make the IGPROF ANALYSE work:
00796                     #The idea is to:
00797                     #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
00798                     #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
00799                     #[3-Catch the case of ANALYSE by itself and raise an exception]
00800                     #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
00801                     #print "EEEEEEE %s"%prof
00802                     #We actually want the analyse to be used also for IgProf perf!
00803                     if 'IgProf' in prof: # and 'perf' not in prof:
00804                         #New IgProf naming scheme:
00805                         #Add a new naming scheme to simplify the handling in igprof-navigator:
00806                         #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
00807 
00808                         #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
00809                         IgProfProfile="IgProf"
00810                         if "Mem" in prof:
00811                             if "reuse" in prof:
00812                                 IgProfProfile="IgProfMem @@@ reuse"
00813                             else:
00814                                 IgProfProfile="IgProfMem"
00815                         else:
00816                             if "reuse" in prof:
00817                                 IgProfProfile="IgProfPerf @@@ reuse"
00818                             else:
00819                                 IgProfProfile="IgProfPerf"
00820 
00821                         #Conditions and PileUpOption are extracted from the cmsDriverOptions:
00822                         Conditions="UNKNOWN_CONDITIONS"
00823                         PileUp="NOPILEUP"
00824                         EventContent="UNKNOWN_EVENTCONTENT"
00825                         tokens=cmsDriverOptions.split("--")
00826                         for token in tokens:
00827                             keywords=token.split(" ")
00828                             if "=" in keywords[0]:
00829                                 keywords=keywords[0].split("=")
00830                             if "conditions" in keywords[0]:
00831                                 # check if we are using the autoCond style of flexible conditions
00832                                 # if so, expand the condition here so that the file names contain the real conditions
00833                                 if "auto:" in keywords[1]:
00834                                     from Configuration.AlCa.autoCond import autoCond
00835                                     fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
00836                                     Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0] 
00837                                 else:
00838                                     # old style, hardcoded, conditions ...
00839                                     # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
00840                                     # and ::All at the end:
00841                                     # Added if to handle new --conditions convention for cmsDriver.py (no more
00842                                     # FrontierConditions... in front of it)
00843                                     if "," in keywords[1]: 
00844                                         Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
00845                                     else:
00846                                         Conditions=keywords[1].split("::")[0] 
00847                             elif "pileup" in keywords[0]:
00848                                 PileUp=keywords[1]
00849                             elif "eventcontent" in keywords[0]:
00850                                 EventContent=keywords[1]
00851 
00852                         #so here's our new MetaName:
00853                         MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
00854                         
00855                         if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
00856                                 
00857                             simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
00858                                                                            Profiler[prof],
00859                                                                            MetaName))
00860                                                                            #FileName[acandle],
00861                                                                            #stepLabel,
00862                                                                            #prof))
00863                             analyse_command = ("%s @@@ %s @@@ %s\n" % (Command,
00864                                                                            Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
00865                                                                            MetaName))
00866                                                                            #FileName[acandle],
00867                                                                            #stepLabel,
00868                                                                            #prof))
00869 
00870                             #for the relval case, GEN-SIM,HLT get rid of the last " @@@ reuse":
00871                             if Profiler[prof] == 'IgProf_mem.MEM_MAX' or Profiler[prof] == 'IgProf_perf.PERF_TICKS':
00872                                 analyse_command = analyse_command.replace("@@@ reuse", "")
00873 
00874                             simcandles.write(analyse_command)
00875 
00876                         elif 'Analyse' in prof:
00877                             pass
00878                     else:    
00879                         simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
00880                                                                        Profiler[prof],
00881                                                                        FileName[acandle],
00882                                                                        stepLabel,
00883                                                                        prof))
00884 
00885                 if debug:
00886                     print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
00887                     print "cmsDriveroptions : " + cmsDriverOption
00888             prevPrevOutputFile = previousOutputFile          
00889             previousOutputFile = OutputFile
00890         else:
00891             unprofiledSteps.append(step)
00892             isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
00893             
00894             try:
00895                 isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
00896                 #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
00897             except IndexError:
00898                 # This loop should have terminated if x + 1 is out of bounds!
00899                 print "Error: Something is wrong we shouldn't have come this far"
00900                 break
00901 
00902             if isNextStepForProfile:
00903                 #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
00904                 OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
00905                 unprofiledSteps = []
00906                 prevPrevOutputFile = previousOutputFile          
00907                 previousOutputFile = OutputFile
00908         #Dangerous index handling when looping over index x!        
00909         if oneShotProf:
00910             stepIndex += len(hypsteps)            
00911         else:
00912             stepIndex +=1
00913     return fstROOTfileStr
00914 
00915 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile):
00916 
00917     OutputStep = ''
00918 
00919     # This option will not be used for now since the event content names keep changing, it can be
00920     # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
00921     #
00922     #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
00923 
00924     for acandle in Candle:
00925         print '*Candle ' + acandle
00926         
00927         ##################
00928         # If the first profiling we run is EdmSize we need to create the root file first
00929         #
00930 
00931         #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
00932         #print "1-userInputFile: %s"%userInputFile
00933         
00934         fstoutfile = writeCommands(simcandles,
00935                                    Profile,
00936                                    acandle,
00937                                    steps,
00938                                    NumberOfEvents,
00939                                    cmsDriverOptions,
00940                                    bypasshlt,
00941                                    0,
00942                                    False,
00943                                    userInputFile)
00944         
00945 
00946 def main(argv=sys.argv):
00947 
00948     #####################
00949     # Options Parser 
00950     #
00951 
00952     (options, args) = optionparse()
00953 
00954     #####################
00955     # Set up arguments and option handling
00956     #
00957 
00958     (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
00959     #print setupProgramParameters(options,args)
00960 
00961     ######################
00962     # Initialize a few variables
00963     #
00964 
00965     (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
00966 
00967     ##################
00968     # Ok everything is ready now we need to create the input file for the relvalreport script 
00969     #
00970 
00971     simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
00972 
00973     ##################
00974     # Based on the profile code create the array of profiles to run:
00975     #
00976 
00977     Profile = getProfileArray(ProfileCode)
00978 
00979 
00980     ##################
00981     # Write the commands for the report to the file
00982     #
00983 
00984     writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
00985 
00986     print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
00987                 
00988     simcandles.close()
00989 
00990 if __name__ == "__main__":
00991     main()
/afs/cern.ch/work/a/aaltunda/public/www/CMSSW_5_3_14/src/Validation/Performance/scripts/cmsRelvalreportInput.py
