Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 # -*- coding: utf-8 -*-
00003 # GBenelli Dec 21, 2007 and JNicolson July 23, 2008
00004 # This script is designed to run on a local directory
00005 # after the user has created a local CMSSW release,
00006 # initialized its environment variables by executing
00007 # in the release /src directory:
00008 # eval `scramv1 runtime -csh`
00009 # project CMSSW
00010 # The script will create a SimulationCandles_$CMSSW_VERSION.txt ASCII
00011 # file, input to cmsRelvalreport.py, to launch the
00012 # Performance Suite.
00013 
00014 # Input arguments are three:
00015 # 1-Number of events to put in the cfg files
00016 # 2-Name of the candle(s) to process (either AllCandles, or NameOfTheCandle)
00017 # 3-Profiles to run (with code below)
00018 # E.g.: ./cmsRelValReportInput.py 50 AllCandles 012
00019 
00020 ###############
00021 # Import modules and define functions
00022 #
00023 
00024 import sys, os, re, operator
00025 import optparse as opt
00026 from cmsPerfCommons import Candles, CandDesc, FileName, KeywordToCfi, CustomiseFragment, CandFname, EventContents
00027 ################
00028 # Global variables
00029 #
00030 
00031 THIS_PROG_NAME = os.path.basename(sys.argv[0])
00032 cmsDriver = 'cmsDriver.py'                    #cmsDriver.py path
00033 hypreg = re.compile('-')
00034 debug = False
00035 DEF_STEPS = ['GEN,SIM', 'DIGI']
00036 AllSteps  = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
00037 AfterPileUpSteps=[]
00038 
00039 # Global variables used by writeCommandsToReport and dependents
00040 
00041 # Hash for the profiler to run
00042 
00043 Profiler = {
00044     'TimingReport'            : 'Timing_Parser',
00045     'TimingReport @@@ reuse'  : 'Timing_Parser',
00046     'TimeReport'              : 'Timereport_Parser',
00047     'TimeReport @@@ reuse'    : 'Timereport_Parser',
00048     'SimpleMemReport'         : 'SimpleMem_Parser',
00049     'EdmSize'                 : 'Edm_Size',
00050     'IgProfperf'              : 'IgProf_perf.PERF_TICKS',
00051     'IgProfperf @@@ reuse'    : 'IgProf_perf.PERF_TICKS',
00052     'IgProfMemTotal'          : 'IgProf_mem.MEM_TOTAL',
00053     'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
00054     'IgProfMemLive'           : 'IgProf_mem.MEM_LIVE',
00055     'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
00056     'IgProfMemMax'            : 'IgProf_mem.MEM_MAX',
00057     'IgProfMemMax @@@ reuse'  : 'IgProf_mem.MEM_MAX',
00058     'IgProfMemAnalyse'        : 'IgProf_mem.ANALYSE',
00059     'valgrind'                : 'ValgrindFCE',
00060     'memcheck_valgrind'       : 'Memcheck_Valgrind',
00061     'None'                    : 'None',
00062 }
00063 
00064 
00065 
00066 def getFstOccur(item, list):
00067     return filter(item.__eq__,list)[0]
00068 
00069 def getLstIndex(item, list):
00070     lenlist = len(list)
00071     for x in range(lenlist - 1,0,-1):
00072         if list[x] == item:
00073             return x
00074 
00075 def checkSteps(steps):
00076     '''Checks user steps <steps> for order against steps defined in AllSteps list.
00077     If they are not, prints a warning and exits.
00078     No return value.'''
00079     
00080     #print steps
00081     #if ":" in steps:
00082     #    steps.split(
00083     idx = -2
00084     lstidx = -2
00085     for step in steps:
00086         astep = step
00087         split = []
00088         #Added special workflow for GEN-FASTSIM, bypass the check of step order:
00089         if step=="GEN,FASTSIM":
00090             pass
00091         else:
00092             if "-" in step:
00093                 split = astep.split("-")
00094                 astep = split[0].split(":")[0]
00095              #   print "This is astep:%s"%astep
00096             #idx =AllSteps.index("GEN,SIM")
00097             #print idx
00098             #print astep
00099             idx = AllSteps.index(astep.split(":")[0])            
00100             if not ( idx == -2 ):
00101                 if lstidx > idx:
00102                     print "ERROR: Your user defined steps are not in a valid order"
00103                     sys.exit()
00104                 lstidx = idx    
00105             if "-" in step:
00106                 lstidx = AllSteps.index(split[1].split(":")[0])
00107 
00108 
00109 def getSteps(userSteps):
00110     '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
00111     Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
00112     Returns a list of steps <steps>.'''
00113     #Add handling of ":" in steps
00114     #print userSteps
00115     # Then split the user steps into "steps"
00116     gsreg = re.compile('GEN-SIM')
00117     greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
00118     gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
00119     StepsTokens = userSteps.split(",")
00120     steps = [] 
00121     for astep in StepsTokens:
00122 
00123         # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
00124         # from using the "-" to using the "," to match cmsDriver.py convention
00125 
00126         if gsreg.search(astep):
00127             astep = gsreg.sub(r"GEN,SIM", astep)
00128         elif gfsreg.search(astep):
00129             astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
00130         elif greg.search(astep):
00131             astep = greg.sub(r"GEN,SIM", astep)
00132             
00133             
00134         #print astep
00135         # Finally collect all the steps into the steps list:
00136 
00137         steps.append(astep)
00138     
00139     #steps = expandHypens(steps)
00140     checkSteps(steps)
00141     return steps
00142         
00143 def optionparse():
00144     global _noprof
00145     explanations = map(lambda x: "          " + x, Candles)
00146     explanation  = ""
00147     for x in range(len(explanations)):
00148         explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
00149     parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
00150 
00151     Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
00152 
00153     Arguments:
00154         NUM_EVENTS_PER_CFG - The number of events per config file
00155 
00156         CANDLES - The simulation type to perform profiling on
00157           Candles              Description
00158           AllCandles           Run all of the candles below
00159 %s
00160 
00161         PROFILE - the type of profiling to perform (multiple codes can be used):
00162           Code  Profile Type
00163           0     TimingReport
00164           1     TimeReport
00165           2     SimpleMemoryCheck
00166           3     EdmSize
00167           4     IgProfPerf
00168           5     IgProfMemTotal
00169           6     IgProfMemLive
00170           7     IgProfAnalyse
00171           8     ValgrindFCE
00172           9     ValgrindMemCheck    
00173 
00174     Examples (could be obsolete!):  
00175        Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
00176         ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00177        Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
00178         ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
00179        Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
00180         ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
00181        Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
00182         ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
00183       % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
00184     
00185     devel  = opt.OptionGroup(parser, "Developer Options",
00186                                      "Caution: use these options at your own risk."
00187                                      "It is believed that some of them bite.\n")
00188     #parser.set_defaults(debug=False)
00189 
00190     parser.add_option(
00191         '-b',
00192         '--bypass-hlt',
00193         action="store_true",
00194         dest='bypasshlt',
00195         default=False,
00196         help='Should we bypass using the HLT root file?'
00197         )
00198 
00199     parser.add_option(
00200         '-u',
00201         '--usersteps',
00202         type='string',
00203         dest='userSteps',
00204         help='Which steps to run',
00205         metavar='<STEPS>',
00206         )
00207     parser.add_option(
00208         '-c',
00209         '--cmsdriver',
00210         type='string',
00211         dest='cmsDriverOptions',
00212         help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
00213         metavar='<CONDITION>',
00214         )
00215     parser.add_option(
00216         '-i',
00217         '--filein',
00218         type='string',
00219         dest='userInputFile',
00220         help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
00221         default='',
00222         metavar='<INPUTFILE>',
00223         )
00224     devel.add_option(
00225         '-1',
00226         '--no-profile',
00227         action="store_true",
00228         dest='noprof',
00229         help='Do not perform profiling, ever',
00230         #metavar='DEBUG',
00231         )
00232     
00233     devel.add_option(
00234         '-d',
00235         '--debug',
00236         action="store_true",
00237         dest='debug',
00238         help='Show debug output',
00239         #metavar='DEBUG',
00240         )
00241 
00242     parser.set_defaults(debug=False,noprof=False)
00243     parser.add_option_group(devel)
00244 
00245     (options, args) = parser.parse_args()
00246     
00247 
00248     _noprof = options.noprof
00249     debug = options.debug
00250     numofargs = len(args) 
00251 #    print "runtime cmsinput " + str(numofargs) + " noprof "  + str(_noprof)
00252     if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
00253         parser.error("There are not enough arguments specified to run this program."
00254                      " Please determine the correct arguments from the usage information above."
00255                      " Run with --help option for more information.")
00256         sys.exit()
00257 
00258     return (options, args)
00259 
00260 def expandHyphens(step):
00261     newsteps = []
00262 #    for step in steps:
00263     if "-" in step:
00264         hypsteps = step.split(r"-")
00265         #if not (len(hypsteps) == 2):
00266         #    print "ERROR: Could not parse usersteps. You have too many hypens between commas"
00267         #    sys.exit()
00268         #el
00269         if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
00270         #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
00271             print "ERROR: One of the steps you defined is invalid"
00272             sys.exit()
00273         else:
00274             if (hypsteps[0] == hypsteps[1]):
00275                 print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
00276                 newsteps.append(hypsteps[0])
00277             else:
00278                 newsteps.append(hypsteps[0])
00279                 
00280                 srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
00281                 for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
00282                     astep = AllSteps[n]
00283                     if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
00284                         #break
00285                         astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
00286                         if astep==hypsteps[-1].split(":")[0]:
00287                             break
00288                     newsteps.append(astep)
00289                 newsteps.append(hypsteps[-1])
00290     else:
00291         if not (step.split(":")[0] in AllSteps):
00292             print "ERROR: One of the steps you defined is invalid"
00293             sys.exit()
00294         else:
00295             newsteps.append(step)
00296 
00297     return newsteps
00298 
00299 def setupProgramParameters(options,args):
00300     steps = []
00301     cmsDriverOptions = ""
00302     global AfterPileUpSteps
00303     NumberOfEvents = int(args[0])  # first arg
00304     WhichCandles   = str(args[1])  # second arg
00305     ProfileCode = ""
00306     if not _noprof:
00307         ProfileCode    = str(args[2])  # third arg
00308 
00309     if options.cmsDriverOptions:
00310 
00311         cmsDriverOptions = options.cmsDriverOptions
00312         print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
00313 
00314     if options.userSteps:
00315 
00316         userSteps = options.userSteps
00317         steps = getSteps(userSteps)
00318 
00319     if WhichCandles.lower() == 'allcandles':
00320         Candle = Candles
00321         print 'ALL standard simulation candles will be PROCESSED:'
00322     else:
00323         Candle = [WhichCandles]
00324         print 'Candle %s will be PROCESSED' % Candle[0]
00325 
00326     # For now the two steps are built in, this can be added as an argument later
00327     # Added the argument option so now this will only be defined if it was not defined already:
00328 
00329     if not steps:
00330         print 'The default steps will be run:'
00331         steps = DEF_STEPS
00332     else:
00333         print "You defined your own steps to run:"
00334 
00335     for astep in steps:
00336         print astep
00337 
00338     return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
00339 
00340 def init_vars():
00341 
00342     ####################
00343     # Obtain required environment variables
00344     #
00345 
00346     try:
00347         CMSSW_BASE         = os.environ['CMSSW_BASE']
00348         CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
00349         CMSSW_VERSION      = os.environ['CMSSW_VERSION']
00350     except KeyError:
00351         print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
00352         print '       Please run eval `scramv1 runtime -csh` to set your environment variables'
00353         sys.exit()
00354 
00355     return ( CMSSW_BASE,
00356              CMSSW_RELEASE_BASE,
00357              CMSSW_VERSION)
00358 
00359 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
00360 
00361     SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
00362     
00363     try:
00364         simcandles = open(SimCandlesFile, 'w')
00365     except IOError:
00366         print "Couldn't open %s to save" % SimCandlesFile
00367 
00368     simcandles.write('#Candles file automatically generated by %s for %s\n'
00369                       % (THIS_PROG_NAME, CMSSW_VERSION))
00370     simcandles.write("#CMSSW Base  : %s\n"   % CMSSW_BASE)
00371     simcandles.write("#Release Base: %s\n"   % CMSSW_RELEASE_BASE)
00372     simcandles.write("#Version     : %s\n\n" % CMSSW_VERSION)
00373 
00374     return simcandles
00375 
00376 def getProfileArray(ProfileCode):
00377 
00378     Profile = []
00379 
00380     # The allowed profiles are:
00381 
00382     AllowedProfile = [
00383         'TimingReport',
00384         'TimeReport',
00385         'SimpleMemReport',
00386         'EdmSize',
00387         'IgProfperf',
00388         'IgProfMemTotal',
00389         'IgProfMemLive',
00390         #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
00391         'IgProfMemAnalyse',
00392         'valgrind',
00393         'memcheck_valgrind',
00394         'None',
00395         ]
00396 
00397     if _noprof:
00398         Profile.append(AllowedProfile[-1])
00399     else:
00400         #FIXME:
00401         #Horrible code!
00402         for i in range(10):
00403             if str(i) in ProfileCode:
00404                 firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
00405                             or (i == 1 and str(2) in ProfileCode)
00406                 secCase   = (i==4 and str(7) in ProfileCode) \
00407                             or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
00408                             or (i == 6 and str(7) in ProfileCode)
00409                 
00410                 if firstCase or secCase:
00411                     Profile.append(AllowedProfile[i] + ' @@@ reuse')
00412                     #Here's the hack:
00413                     if i==6: #i.e. IgProfMemLive
00414                         Profile.append("IgProfMemMax @@@ reuse")
00415                 else:
00416                     Profile.append(AllowedProfile[i])
00417                     if i==6: #i.e. IgProfMemLive
00418                         Profile.append("IgProfMemMax")
00419 
00420     return Profile
00421 
00422 def writeStepHead(simcandles,acandle,step):
00423     simcandles.write('#%s\n' % FileName[acandle])
00424     simcandles.write('#Step %s\n' % step)
00425     #print step
00426 
00427 
00428 def determineNewProfile(step,Profile,SavedProfile):
00429     if 'DIGI2RAW' in step:
00430         SavedProfile = Profile
00431         Profile = [ ]
00432     if 'HLT' in step:
00433         Profile = SavedProfile
00434 
00435     return (Profile, SavedProfile)
00436 
00437 def pythonFragment(step,cmsdriverOptions):
00438     # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
00439     #It is now living in cmsPerfCommons.py!
00440     
00441     if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
00442         return CustomiseFragment['DIGI-PILEUP']
00443     elif CustomiseFragment.has_key(step):
00444         return CustomiseFragment[step] 
00445     else:
00446         #This is a safe default in any case,
00447         #the maintenance of the customise python fragments goes into cmsPerfCommons.py
00448         return CustomiseFragment['DIGI']
00449 
00450 
00451 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
00452     #print "StepIndex 2:%s"%stepIndex
00453     InputFileOption = ""
00454     if pileup and stepIndex == 0:
00455         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
00456     else:
00457         InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
00458         
00459     if pileup:
00460         pass
00461     else :
00462         if 'GEN,SIM' in step:  # there is no input file for GEN,SIM!
00463             InputFileOption = ''
00464         if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
00465             InputFileOption = ''
00466         elif   'HLT' in steps[stepIndex - 1] and bypasshlt:
00467 
00468             # Special hand skipping of HLT since it is not stable enough, so it will not prevent
00469             # RAW2DIGI,RECO from running
00470 
00471             InputFileOption = "--filein file:%s_%s"  % ( FileName[acandle],steps[stepIndex - 2] )
00472 
00473     if not InputFileOption == "" :
00474         InputFileOption += ".root "
00475 
00476     return InputFileOption
00477 
00478 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
00479     # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
00480     #stepsStr = ",".join(unprofiledSteps)
00481 
00482     
00483     #print "unprofiledSteps is %s"%unprofiledSteps
00484     #print "22acandle is %s"%acandle
00485     #Kludge in case -b option to skip HLT is used...
00486     if bypasshlt and unprofiledSteps[-1]=="HLT":
00487         stepsStr = ",".join(unprofiledSteps[:-1])
00488         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
00489     else:
00490         stepsStr = ",".join(unprofiledSteps)
00491         OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
00492     simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
00493     OutputFileOption = "--fileout=%s" % OutputFile
00494     #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
00495     #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
00496     #Adding the argument bypasshlt to the calls...
00497     #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
00498     #print "StepIndex 1:%s"%stepIndex
00499     #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
00500     #stepIndexAdjust=stepIndex - 2
00501     #if stepIndexAdjust < 0: #To avoid issues with negative indeces
00502     #    stepIndexAdjust=0
00503     #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
00504     #Use an argument to make it easier:
00505     InputFileOption = "--filein file:" + previousOutputFile
00506     if previousOutputFile =="":
00507         InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00508     #Introduce an over-ride of cmsDriverOptions:
00509     #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
00510     #At the moment the only use case is when running step2 on its own...
00511     for eventcontent in EventContents:
00512         cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
00513     Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00514                        % (cmsDriver,
00515                           KeywordToCfi[acandle],
00516                           NumberOfEvents,
00517                           stepsStr,
00518                           InputFileOption,
00519                           OutputFileOption,
00520                           CustomisePythonFragment,
00521                           cmsDriverOptions) )
00522     simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
00523     return OutputFile
00524 
00525 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
00526     fstIdx = -1
00527     #Hack for GEN-FASTSIM workflow:
00528     #if steps=="GEN,FASTSIM":   
00529     #if "-" in steps[0]:
00530     fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00531     #else:
00532     #    fstIdx = AllSteps.index(steps[0])
00533     CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
00534     previousOutputFile=""
00535     OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt) 
00536     return (fstIdx, OutputFile)
00537 
00538 def setOutputFileOption(acandle,endstep):
00539     return "%s_%s.root" % ( FileName[acandle],endstep)
00540 
00541 def writeCommands(simcandles,
00542                   Profile,
00543                   acandle,
00544                   steps,
00545                   NumberOfEvents,
00546                   cmsDriverOptions,
00547                   bypasshlt,                  
00548                   stepIndex = 0,
00549                   pileup = False,
00550                   userInputFile=""):
00551     #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
00552     OutputStep = ""
00553 
00554     stopIndex = len(steps)
00555     start = 0
00556 
00557     userSteps = steps
00558     SavedProfile = []
00559     fstROOTfile = True
00560     fstROOTfileStr = ""
00561 
00562     #Handling the case of the first user step not being the first step (GEN,SIM):
00563     #print "Steps passed to writeCommands %s",steps
00564     #Special case for FASTSIM:
00565     if (steps[0] == "GEN,FASTSIM"):
00566         start=-1
00567         stopIndex=1
00568         steps = ["GEN,FASTSIM"]
00569     else:
00570         if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
00571             #print "userInputFile: %s"%userInputFile
00572             if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
00573                 userInputFile = "../INPUT_PILEUP_EVENTS.root"
00574                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00575                 rootFileStr=""
00576                 
00577             elif userInputFile == "":
00578                 #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
00579                 (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
00580             else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
00581                 #if "-" in steps[0]:
00582                 stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
00583                     #print "Here!"
00584                 #else:
00585                     #print "There!"
00586                 #    stepIndex=AllSteps.index(steps[0])
00587                 rootFileStr=""
00588             #Now take care of setting the indeces and input root file name right for the profiling part...
00589             if fstROOTfile:
00590                 fstROOTfileStr = rootFileStr
00591                 fstROOTfile = False
00592             start = -1
00593             #if "-" in steps[0]:
00594             start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
00595             #else:
00596             #    start = AllSteps.index(steps[0])
00597             lst = - 1
00598             if "-" in steps[-1]:
00599                 lst   = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
00600             else:
00601                 lst   = AllSteps.index(steps[-1])
00602             runSteps = AllSteps[start:lst]
00603             numOfSteps = (lst - start) + 1
00604             stopIndex = start + numOfSteps
00605             #Handling the case in which the first user step is the same as the first step (GEN,SIM)
00606             #elif  not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
00607         else:
00608             #Handling the case of the last step being a composite one:
00609             if "-" in steps[-1]:
00610                 #Set the stop index at the last step of the composite step (WHY???) 
00611                 stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
00612             else:
00613                 
00614                 stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
00615                 #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
00616         steps = AllSteps
00617             
00618     unprofiledSteps = []
00619     rawreg = re.compile("^RAW2DIGI")
00620 
00621 #Horrible structure... to be rewritten sooner or later...
00622     
00623 #   FOR step in steps
00624 
00625     prevPrevOutputFile = ""
00626     #Add if to introduce the possibility to run on user selected input file.
00627     if userInputFile!="":
00628         previousOutputFile=userInputFile
00629     else:
00630         previousOutputFile = ""
00631 
00632     for x in range(start,stopIndex,1):
00633         if stepIndex >= stopIndex:
00634             break
00635         step = steps[stepIndex]
00636 
00637         # One shot Profiling variables
00638         befStep     = step
00639         aftStep     = step
00640         # We need this in case we are running one-shot profiling or for DIGI-PILEUP
00641         stepToWrite = step 
00642         
00643         CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
00644         oneShotProf = False
00645         hypsteps = []
00646 
00647         #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
00648         if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)): 
00649 
00650             #Checking now if the current step matches the first of a composite step in userSteps
00651             hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
00652             if not len(hypMatch) == 0 :
00653                 hypsteps    = expandHyphens(hypMatch[0])
00654                 #print hypsteps
00655                 #print hypMatch[0]
00656                 stepToWrite = ",".join(hypsteps)
00657                 befStep     = hypsteps[0]
00658                 #Kludge to avoid running HLT in composite steps if the -b option is chosen
00659                 if bypasshlt and hypsteps[-1]=='HLT':
00660                     aftStep     = hypsteps[-2]
00661                 else:
00662                     aftStep     = hypsteps[-1]
00663                 oneShotProf = True
00664             if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
00665                 #print "Inside the filter if!"
00666                 #print stepToWrite
00667                 #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
00668                 #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
00669                 #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!) 
00670                 stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
00671                 #print stepToWrite
00672 
00673             writeStepHead(simcandles,acandle,stepToWrite)
00674 
00675             #Set the output file name for Pile up and for regular case:
00676             if '--pileup' in cmsDriverOptions:
00677                 outfile = stepToWrite + "_PILEUP"
00678             else:
00679                 outfile = stepToWrite
00680                 
00681             OutputFile = setOutputFileOption(acandle,outfile)
00682             if fstROOTfile:
00683                 fstROOTfileStr = OutputFile
00684                 fstROOTfile = False
00685             OutputFileOption = "--fileout=" + OutputFile
00686 
00687             for prof in Profile:
00688                 #First prepare the cmsDriver.py command
00689                 
00690                 #Special case of EventEdmSize profiling 
00691                 if 'EdmSize' in prof:
00692                     EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
00693                     #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
00694                     if prof == Profile[0] and not os.path.exists("./" + EdmFile):
00695                         # insert command to generate required state ( need to run one more step
00696                         # so that EDM can actually check the size of the root file
00697                         # If the first step to be profiled is something later on in the steps such
00698                         # as HLT then writePrerequisteSteps() should have got you to the step prior to
00699                         # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
00700 
00701                         InputFileOption = "--filein file:" + previousOutputFile
00702                         #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
00703                         if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00704                             InputFileOption = "--filein file:" + prevPrevOutputFile
00705                         if previousOutputFile == "":
00706                             InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
00707                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
00708                                            % (cmsDriver,
00709                                               KeywordToCfi[acandle],
00710                                               NumberOfEvents,
00711                                               stepToWrite,
00712                                               InputFileOption,
00713                                               OutputFileOption,
00714                                               CustomisePythonFragment,
00715                                               cmsDriverOptions) )
00716                         simcandles.write( "%s @@@ None @@@ None\n" % (Command))
00717 
00718                     Command = EdmFile
00719                 #all other profiles:
00720                 else:
00721                     InputFileOption = "--filein file:" + previousOutputFile
00722                     if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
00723                         InputFileOption = "--filein file:" + prevPrevOutputFile
00724 
00725                     if previousOutputFile == "":
00726                         InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
00727 
00728                     #if '--pileup' in cmsDriverOptions:
00729                     #    stepToWrite = pileupStep
00730                     if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'): 
00731                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00732                                cmsDriver,
00733                                KeywordToCfi[acandle],
00734                                NumberOfEvents,
00735                                stepToWrite,
00736                                InputFileOption,
00737                                OutputFileOption,
00738                                CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00739                                cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
00740                            ))
00741                     elif '--pileup' in cmsDriverOptions and stepToWrite=='GEN,FASTSIM': #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
00742                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00743                                cmsDriver,
00744                                KeywordToCfi[acandle],
00745                                NumberOfEvents,
00746                                stepToWrite,
00747                                InputFileOption,
00748                                OutputFileOption,
00749                                CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
00750                                cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')] 
00751                            ))
00752                     
00753                     else:
00754                         Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
00755                                cmsDriver,
00756                                KeywordToCfi[acandle],
00757                                NumberOfEvents,
00758                                stepToWrite,
00759                                InputFileOption,
00760                                OutputFileOption,
00761                                CustomisePythonFragment,
00762                                cmsDriverOptions
00763                            ))
00764                 #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
00765                 if _noprof:
00766                     simcandles.write("%s @@@ None @@@ None\n" % Command)
00767                 else:
00768                     if 'valgrind' in prof:
00769                         #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
00770                         if 'TimeMemoryInfo.py' in Command:
00771                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
00772                         #There is actually also the SIM case to take care of:
00773                         elif 'TimeMemoryG4Info.py' in Command:
00774                             Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
00775                     stepLabel=stepToWrite
00776                     #print "stepLabel is %s"%stepLabel
00777                     if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
00778                         stepLabel = stepToWrite+"_PILEUP"
00779                     #Add kludge here to make the IGPROF ANALYSE work:
00780                     #The idea is to:
00781                     #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
00782                     #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
00783                     #[3-Catch the case of ANALYSE by itself and raise an exception]
00784                     #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
00785                     #print "EEEEEEE %s"%prof
00786                     #We actually want the analyse to be used also for IgProf perf!
00787                     if 'IgProf' in prof: # and 'perf' not in prof:
00788                         #New IgProf naming scheme:
00789                         #Add a new naming scheme to simplify the handling in igprof-navigator:
00790                         #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
00791 
00792                         #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
00793                         IgProfProfile="IgProf"
00794                         if "Mem" in prof:
00795                             if "reuse" in prof:
00796                                 IgProfProfile="IgProfMem @@@ reuse"
00797                             else:
00798                                 IgProfProfile="IgProfMem"
00799                         else:
00800                             if "reuse" in prof:
00801                                 IgProfProfile="IgProfPerf @@@ reuse"
00802                             else:
00803                                 IgProfProfile="IgProfPerf"
00804 
00805                         #Conditions and PileUpOption are extracted from the cmsDriverOptions:
00806                         Conditions="UNKNOWN_CONDITIONS"
00807                         PileUp="NOPILEUP"
00808                         EventContent="UNKNOWN_EVENTCONTENT"
00809                         tokens=cmsDriverOptions.split("--")
00810                         for token in tokens:
00811                             keywords=token.split(" ")
00812                             if "=" in keywords[0]:
00813                                 keywords=keywords[0].split("=")
00814                             if "conditions" in keywords[0]:
00815                                 # check if we are using the autoCond style of flexible conditions
00816                                 # if so, expand the condition here so that the file names contain the real conditions
00817                                 if "auto:" in keywords[1]:
00818                                     from Configuration.PyReleaseValidation.autoCond import autoCond
00819                                     fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
00820                                     Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0] 
00821                                 else:
00822                                     # old style, hardcoded, conditions ...
00823                                     # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
00824                                     # and ::All at the end:
00825                                     # Added if to handle new --conditions convention for cmsDriver.py (no more
00826                                     # FrontierConditions... in front of it)
00827                                     if "," in keywords[1]: 
00828                                         Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
00829                                     else:
00830                                         Conditions=keywords[1].split("::")[0] 
00831                             elif "pileup" in keywords[0]:
00832                                 PileUp=keywords[1]
00833                             elif "eventcontent" in keywords[0]:
00834                                 EventContent=keywords[1]
00835                         #so here's our new MetaName:
00836                         MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
00837                         
00838                         if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
00839                                 
00840                             simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
00841                                                                            Profiler[prof],
00842                                                                            MetaName))
00843                                                                            #FileName[acandle],
00844                                                                            #stepLabel,
00845                                                                            #prof))
00846                             simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
00847                                                                            Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
00848                                                                            MetaName))
00849                                                                            #FileName[acandle],
00850                                                                            #stepLabel,
00851                                                                            #prof))
00852                         elif 'Analyse' in prof:
00853                             pass
00854                     else:    
00855                         simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
00856                                                                        Profiler[prof],
00857                                                                        FileName[acandle],
00858                                                                        stepLabel,
00859                                                                        prof))
00860 
00861                 if debug:
00862                     print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
00863                     print "cmsDriveroptions : " + cmsDriverOption
00864             prevPrevOutputFile = previousOutputFile          
00865             previousOutputFile = OutputFile
00866         else:
00867             unprofiledSteps.append(step)
00868             isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
00869             
00870             try:
00871                 isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
00872                 #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
00873             except IndexError:
00874                 # This loop should have terminated if x + 1 is out of bounds!
00875                 print "Error: Something is wrong we shouldn't have come this far"
00876                 break
00877 
00878             if isNextStepForProfile:
00879                 #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
00880                 OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
00881                 unprofiledSteps = []
00882                 prevPrevOutputFile = previousOutputFile          
00883                 previousOutputFile = OutputFile
00884         #Dangerous index handling when looping over index x!        
00885         if oneShotProf:
00886             stepIndex += len(hypsteps)            
00887         else:
00888             stepIndex +=1
00889     return fstROOTfileStr
00890 
00891 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile):
00892 
00893     OutputStep = ''
00894 
00895     # This option will not be used for now since the event content names keep changing, it can be
00896     # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
00897     #
00898     #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
00899 
00900     for acandle in Candle:
00901         print '*Candle ' + acandle
00902         
00903         ##################
00904         # If the first profiling we run is EdmSize we need to create the root file first
00905         #
00906 
00907         #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
00908         #print "1-userInputFile: %s"%userInputFile
00909         
00910         fstoutfile = writeCommands(simcandles,
00911                                    Profile,
00912                                    acandle,
00913                                    steps,
00914                                    NumberOfEvents,
00915                                    cmsDriverOptions,
00916                                    bypasshlt,
00917                                    0,
00918                                    False,
00919                                    userInputFile)
00920         
00921 
00922 def main(argv=sys.argv):
00923 
00924     #####################
00925     # Options Parser 
00926     #
00927 
00928     (options, args) = optionparse()
00929 
00930     #####################
00931     # Set up arguments and option handling
00932     #
00933 
00934     (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
00935     #print setupProgramParameters(options,args)
00936 
00937     ######################
00938     # Initialize a few variables
00939     #
00940 
00941     (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
00942 
00943     ##################
00944     # Ok everything is ready now we need to create the input file for the relvalreport script 
00945     #
00946 
00947     simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
00948 
00949     ##################
00950     # Based on the profile code create the array of profiles to run:
00951     #
00952 
00953     Profile = getProfileArray(ProfileCode)
00954 
00955 
00956     ##################
00957     # Write the commands for the report to the file
00958     #
00959 
00960     writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
00961 
00962     print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
00963                 
00964     simcandles.close()
00965 
00966 if __name__ == "__main__":
00967     main()
/data/doxygen/doxygen-1.7.3/gen/CMSSW_4_2_8/src/Validation/Performance/scripts/cmsRelvalreportInput.py
