CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsRelvalreportInput Namespace Reference

Functions

def checkSteps
 
def determineNewProfile
 
def expandHyphens
 
def getFstOccur
 
def getLstIndex
 
def getProfileArray
 
def getSteps
 
def init_vars
 
def main
 
def optionparse
 
def pythonFragment
 
def setInputFile
 
def setOutputFileOption
 
def setupProgramParameters
 
def writeCommands
 
def writeCommandsToReport
 
def writePrerequisteSteps
 
def writeReportFileHeader
 
def writeStepHead
 
def writeUnprofiledSteps
 

Variables

list AfterPileUpSteps = []
 
list AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
 
string cmsDriver = 'cmsDriver.py'
 
 debug = False
 
list DEF_STEPS = ['GEN,SIM', 'DIGI']
 
tuple hypreg = re.compile('-')
 
dictionary Profiler
 
tuple THIS_PROG_NAME = os.path.basename(sys.argv[0])
 Global variables. More...
 

Function Documentation

def cmsRelvalreportInput.checkSteps (   steps) 
Checks user steps <steps> for order against steps defined in AllSteps list.
If they are not, prints a warning and exits.
No return value.

Definition at line 75 of file cmsRelvalreportInput.py.

References split.

Referenced by getSteps().

75 
76 def checkSteps(steps):
77  '''Checks user steps <steps> for order against steps defined in AllSteps list.
78  If they are not, prints a warning and exits.
79  No return value.'''
80 
81  #print steps
82  #if ":" in steps:
83  # steps.split(
84  idx = -2
85  lstidx = -2
86  for step in steps:
87  astep = step
88  split = []
89  #Added special workflow for GEN-FASTSIM, bypass the check of step order:
90  if step=="GEN,FASTSIM":
91  pass
92  else:
93  if "-" in step:
94  split = astep.split("-")
95  astep = split[0].split(":")[0]
96  # print "This is astep:%s"%astep
97  #idx =AllSteps.index("GEN,SIM")
98  #print idx
99  #print astep
100  idx = AllSteps.index(astep.split(":")[0])
101  if not ( idx == -2 ):
102  if lstidx > idx:
103  print "ERROR: Your user defined steps are not in a valid order"
104  sys.exit()
105  lstidx = idx
106  if "-" in step:
107  lstidx = AllSteps.index(split[1].split(":")[0])
108 
split
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.determineNewProfile (   step,
  Profile,
  SavedProfile 
)

Definition at line 428 of file cmsRelvalreportInput.py.

428 
429 def determineNewProfile(step,Profile,SavedProfile):
430  if 'DIGI2RAW' in step:
431  SavedProfile = Profile
432  Profile = [ ]
433  if 'HLT' in step:
434  Profile = SavedProfile
435 
436  return (Profile, SavedProfile)
def cmsRelvalreportInput.expandHyphens (   step)

Definition at line 260 of file cmsRelvalreportInput.py.

References align_tpl.filter, python.multivaluedict.map(), and split.

Referenced by writeCommands().

260 
261 def expandHyphens(step):
262  newsteps = []
263 # for step in steps:
264  if "-" in step:
265  hypsteps = step.split(r"-")
266  #if not (len(hypsteps) == 2):
267  # print "ERROR: Could not parse usersteps. You have too many hypens between commas"
268  # sys.exit()
269  #el
270  if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
271  #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
272  print "ERROR: One of the steps you defined is invalid"
273  sys.exit()
274  else:
275  if (hypsteps[0] == hypsteps[1]):
276  print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
277  newsteps.append(hypsteps[0])
278  else:
279  newsteps.append(hypsteps[0])
280 
281  srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
282  for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
283  astep = AllSteps[n]
284  if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
285  #break
286  astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
287  if astep==hypsteps[-1].split(":")[0]:
288  break
289  newsteps.append(astep)
290  newsteps.append(hypsteps[-1])
291  else:
292  if not (step.split(":")[0] in AllSteps):
293  print "ERROR: One of the steps you defined is invalid"
294  sys.exit()
295  else:
296  newsteps.append(step)
297 
298  return newsteps
align_tpl.filter
tuple filter
USE THIS FOR SKIMMED TRACKS process.p = cms.Path(process.hltLevel1GTSeed*process.skimming*process.offlineBeamSpot*process.TrackRefitter2) OTHERWISE USE THIS.
Definition: align_tpl.py:86
split
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.getFstOccur (   item,
  list 
)

Definition at line 66 of file cmsRelvalreportInput.py.

References align_tpl.filter.

66 
67 def getFstOccur(item, list):
68  return filter(item.__eq__,list)[0]
align_tpl.filter
tuple filter
USE THIS FOR SKIMMED TRACKS process.p = cms.Path(process.hltLevel1GTSeed*process.skimming*process.offlineBeamSpot*process.TrackRefitter2) OTHERWISE USE THIS.
Definition: align_tpl.py:86
def cmsRelvalreportInput.getLstIndex (   item,
  list 
)

Definition at line 69 of file cmsRelvalreportInput.py.

69 
70 def getLstIndex(item, list):
71  lenlist = len(list)
72  for x in range(lenlist - 1,0,-1):
73  if list[x] == item:
74  return x
def cmsRelvalreportInput.getProfileArray (   ProfileCode)

Definition at line 376 of file cmsRelvalreportInput.py.

Referenced by main().

376 
377 def getProfileArray(ProfileCode):
378 
379  Profile = []
380 
381  # The allowed profiles are:
382 
383  AllowedProfile = [
384  'TimingReport',
385  'TimeReport',
386  'SimpleMemReport',
387  'EdmSize',
388  'IgProfperf',
389  'IgProfMemTotal',
390  'IgProfMemLive',
391  #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
392  'IgProfMemAnalyse',
393  'valgrind',
394  'memcheck_valgrind',
395  'None',
396  ]
397 
398  if _noprof:
399  Profile.append(AllowedProfile[-1])
400  else:
401  #FIXME:
402  #Horrible code!
403  for i in range(10):
404  if str(i) in ProfileCode:
405  firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
406  or (i == 1 and str(2) in ProfileCode)
407  secCase = (i==4 and str(7) in ProfileCode) \
408  or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
409  or (i == 6 and str(7) in ProfileCode)
410 
411  if firstCase or secCase:
412  Profile.append(AllowedProfile[i] + ' @@@ reuse')
413  #Here's the hack:
414  if i==6: #i.e. IgProfMemLive
415  Profile.append("IgProfMemMax @@@ reuse")
416  else:
417  Profile.append(AllowedProfile[i])
418  if i==6: #i.e. IgProfMemLive
419  Profile.append("IgProfMemMax")
420 
421  return Profile
def cmsRelvalreportInput.getSteps (   userSteps) 
Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
Returns a list of steps <steps>.

Definition at line 109 of file cmsRelvalreportInput.py.

References checkSteps().

Referenced by setupProgramParameters().

109 
110 def getSteps(userSteps):
111  '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
112  Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
113  Returns a list of steps <steps>.'''
114  #Add handling of ":" in steps
115  #print userSteps
116  # Then split the user steps into "steps"
117  gsreg = re.compile('GEN-SIM')
118  greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
119  gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
120  StepsTokens = userSteps.split(",")
121  steps = []
122  for astep in StepsTokens:
123 
124  # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
125  # from using the "-" to using the "," to match cmsDriver.py convention
126 
127  if gsreg.search(astep):
128  astep = gsreg.sub(r"GEN,SIM", astep)
129  elif gfsreg.search(astep):
130  astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
131  elif greg.search(astep):
132  astep = greg.sub(r"GEN,SIM", astep)
133 
134 
135  #print astep
136  # Finally collect all the steps into the steps list:
137 
138  steps.append(astep)
139 
140  #steps = expandHypens(steps)
141  checkSteps(steps)
142  return steps
def cmsRelvalreportInput.init_vars ( )

Definition at line 340 of file cmsRelvalreportInput.py.

Referenced by main().

340 
341 def init_vars():
342 
343  ####################
344  # Obtain required environment variables
345  #
346 
347  try:
348  CMSSW_BASE = os.environ['CMSSW_BASE']
349  CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
350  CMSSW_VERSION = os.environ['CMSSW_VERSION']
351  except KeyError:
352  print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
353  print ' Please run eval `scramv1 runtime -csh` to set your environment variables'
354  sys.exit()
355 
356  return ( CMSSW_BASE,
357  CMSSW_RELEASE_BASE,
358  CMSSW_VERSION)
def cmsRelvalreportInput.main (   argv = sys.argv)

Definition at line 922 of file cmsRelvalreportInput.py.

References getProfileArray(), init_vars(), optionparse(), setupProgramParameters(), writeCommandsToReport(), and writeReportFileHeader().

922 
923 def main(argv=sys.argv):
924 
925  #####################
926  # Options Parser
927  #
928 
929  (options, args) = optionparse()
930 
931  #####################
932  # Set up arguments and option handling
933  #
934 
935  (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
936  #print setupProgramParameters(options,args)
937 
938  ######################
939  # Initialize a few variables
940  #
941 
942  (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
943 
944  ##################
945  # Ok everything is ready now we need to create the input file for the relvalreport script
946  #
947 
948  simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
949 
950  ##################
951  # Based on the profile code create the array of profiles to run:
952  #
953 
954  Profile = getProfileArray(ProfileCode)
955 
956 
957  ##################
958  # Write the commands for the report to the file
959  #
960 
961  writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
962 
963  print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
964 
965  simcandles.close()
def cmsRelvalreportInput.optionparse ( )

Definition at line 143 of file cmsRelvalreportInput.py.

References python.multivaluedict.map().

Referenced by main().

143 
144 def optionparse():
145  global _noprof
146  explanations = map(lambda x: " " + x, Candles)
147  explanation = ""
148  for x in range(len(explanations)):
149  explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
150  parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
151 
152  Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
153 
154  Arguments:
155  NUM_EVENTS_PER_CFG - The number of events per config file
156 
157  CANDLES - The simulation type to perform profiling on
158  Candles Description
159  AllCandles Run all of the candles below
160 %s
161 
162  PROFILE - the type of profiling to perform (multiple codes can be used):
163  Code Profile Type
164  0 TimingReport
165  1 TimeReport
166  2 SimpleMemoryCheck
167  3 EdmSize
168  4 IgProfPerf
169  5 IgProfMemTotal
170  6 IgProfMemLive
171  7 IgProfAnalyse
172  8 ValgrindFCE
173  9 ValgrindMemCheck
174 
175  Examples (could be obsolete!):
176  Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
177  ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
178  Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
179  ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
180  Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
181  ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
182  Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
183  ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
184  % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
185 
186  devel = opt.OptionGroup(parser, "Developer Options",
187  "Caution: use these options at your own risk."
188  "It is believed that some of them bite.\n")
189  #parser.set_defaults(debug=False)
190 
191  parser.add_option(
192  '-b',
193  '--bypass-hlt',
194  action="store_true",
195  dest='bypasshlt',
196  default=False,
197  help='Should we bypass using the HLT root file?'
198  )
199 
200  parser.add_option(
201  '-u',
202  '--usersteps',
203  type='string',
204  dest='userSteps',
205  help='Which steps to run',
206  metavar='<STEPS>',
207  )
208  parser.add_option(
209  '-c',
210  '--cmsdriver',
211  type='string',
212  dest='cmsDriverOptions',
213  help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
214  metavar='<CONDITION>',
215  )
216  parser.add_option(
217  '-i',
218  '--filein',
219  type='string',
220  dest='userInputFile',
221  help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
222  default='',
223  metavar='<INPUTFILE>',
224  )
225  devel.add_option(
226  '-1',
227  '--no-profile',
228  action="store_true",
229  dest='noprof',
230  help='Do not perform profiling, ever',
231  #metavar='DEBUG',
232  )
233 
234  devel.add_option(
235  '-d',
236  '--debug',
237  action="store_true",
238  dest='debug',
239  help='Show debug output',
240  #metavar='DEBUG',
241  )
242 
243  parser.set_defaults(debug=False,noprof=False)
244  parser.add_option_group(devel)
245 
246  (options, args) = parser.parse_args()
247 
248 
249  _noprof = options.noprof
250  debug = options.debug
251  numofargs = len(args)
252 # print "runtime cmsinput " + str(numofargs) + " noprof " + str(_noprof)
253  if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
254  parser.error("There are not enough arguments specified to run this program."
255  " Please determine the correct arguments from the usage information above."
256  " Run with --help option for more information.")
257  sys.exit()
258 
259  return (options, args)
def cmsRelvalreportInput.pythonFragment (   step,
  cmsdriverOptions 
)

Definition at line 437 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writePrerequisteSteps().

437 
438 def pythonFragment(step,cmsdriverOptions):
439  # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
440  #It is now living in cmsPerfCommons.py!
441 
442  if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
443  return CustomiseFragment['DIGI-PILEUP']
444  elif CustomiseFragment.has_key(step):
445  return CustomiseFragment[step]
446  else:
447  #This is a safe default in any case,
448  #the maintenance of the customise python fragments goes into cmsPerfCommons.py
449  return CustomiseFragment['DIGI']
450 
def cmsRelvalreportInput.setInputFile (   steps,
  step,
  acandle,
  stepIndex,
  pileup = False,
  bypasshlt = False 
)

Definition at line 451 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writeUnprofiledSteps().

451 
452 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
453  #print "StepIndex 2:%s"%stepIndex
454  InputFileOption = ""
455  if pileup and stepIndex == 0:
456  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
457  else:
458  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
459 
460  if pileup:
461  pass
462  else :
463  if 'GEN,SIM' in step: # there is no input file for GEN,SIM!
464  InputFileOption = ''
465  if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
466  InputFileOption = ''
467  elif 'HLT' in steps[stepIndex - 1] and bypasshlt:
468 
469  # Special hand skipping of HLT since it is not stable enough, so it will not prevent
470  # RAW2DIGI,RECO from running
471 
472  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 2] )
473 
474  if not InputFileOption == "" :
475  InputFileOption += ".root "
476 
477  return InputFileOption
def cmsRelvalreportInput.setOutputFileOption (   acandle,
  endstep 
)

Definition at line 538 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by writeCommands().

538 
539 def setOutputFileOption(acandle,endstep):
540  return "%s_%s.root" % ( FileName[acandle],endstep)
def cmsRelvalreportInput.setupProgramParameters (   options,
  args 
)

Definition at line 299 of file cmsRelvalreportInput.py.

References getSteps().

Referenced by main().

299 
300 def setupProgramParameters(options,args):
301  steps = []
302  cmsDriverOptions = ""
303  global AfterPileUpSteps
304  NumberOfEvents = int(args[0]) # first arg
305  WhichCandles = str(args[1]) # second arg
306  ProfileCode = ""
307  if not _noprof:
308  ProfileCode = str(args[2]) # third arg
309 
310  if options.cmsDriverOptions:
311 
312  cmsDriverOptions = options.cmsDriverOptions
313  print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
314 
315  if options.userSteps:
316 
317  userSteps = options.userSteps
318  steps = getSteps(userSteps)
319 
320  if WhichCandles.lower() == 'allcandles':
321  Candle = Candles
322  print 'ALL standard simulation candles will be PROCESSED:'
323  else:
324  Candle = [WhichCandles]
325  print 'Candle %s will be PROCESSED' % Candle[0]
326 
327  # For now the two steps are built in, this can be added as an argument later
328  # Added the argument option so now this will only be defined if it was not defined already:
329 
330  if not steps:
331  print 'The default steps will be run:'
332  steps = DEF_STEPS
333  else:
334  print "You defined your own steps to run:"
335 
336  for astep in steps:
337  print astep
338 
339  return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
def cmsRelvalreportInput.writeCommands (   simcandles,
  Profile,
  acandle,
  steps,
  NumberOfEvents,
  cmsDriverOptions,
  bypasshlt,
  stepIndex = 0,
  pileup = False,
  userInputFile = "" 
)

Definition at line 550 of file cmsRelvalreportInput.py.

References expandHyphens(), align_tpl.filter, join(), python.multivaluedict.map(), pythonFragment(), setInputFile(), setOutputFileOption(), split, writePrerequisteSteps(), writeStepHead(), and writeUnprofiledSteps().

Referenced by setOutputFileOption(), and writeCommandsToReport().

550 
551  userInputFile=""):
552  #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
553  OutputStep = ""
554 
555  stopIndex = len(steps)
556  start = 0
557 
558  userSteps = steps
559  SavedProfile = []
560  fstROOTfile = True
561  fstROOTfileStr = ""
562 
563  #Handling the case of the first user step not being the first step (GEN,SIM):
564  #print "Steps passed to writeCommands %s",steps
565  #Special case for FASTSIM:
566  if (steps[0] == "GEN,FASTSIM"):
567  start=-1
568  stopIndex=1
569  steps = ["GEN,FASTSIM"]
570  else:
571  if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
572  #print "userInputFile: %s"%userInputFile
573  if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
574  userInputFile = "../INPUT_PILEUP_EVENTS.root"
575  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
576  rootFileStr=""
577 
578  elif userInputFile == "":
579  #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
580  (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
581  else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
582  #if "-" in steps[0]:
583  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
584  #print "Here!"
585  #else:
586  #print "There!"
587  # stepIndex=AllSteps.index(steps[0])
588  rootFileStr=""
589  #Now take care of setting the indeces and input root file name right for the profiling part...
590  if fstROOTfile:
591  fstROOTfileStr = rootFileStr
592  fstROOTfile = False
593  start = -1
594  #if "-" in steps[0]:
595  start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
596  #else:
597  # start = AllSteps.index(steps[0])
598  lst = - 1
599  if "-" in steps[-1]:
600  lst = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
601  else:
602  lst = AllSteps.index(steps[-1])
603  runSteps = AllSteps[start:lst]
604  numOfSteps = (lst - start) + 1
605  stopIndex = start + numOfSteps
606  #Handling the case in which the first user step is the same as the first step (GEN,SIM)
607  #elif not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
608  else:
609  #Handling the case of the last step being a composite one:
610  if "-" in steps[-1]:
611  #Set the stop index at the last step of the composite step (WHY???)
612  stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
613  else:
614 
615  stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
616  #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
617  steps = AllSteps
618 
619  unprofiledSteps = []
620  rawreg = re.compile("^RAW2DIGI")
621 
622 #Horrible structure... to be rewritten sooner or later...
623 
624 # FOR step in steps
625 
626  prevPrevOutputFile = ""
627  #Add if to introduce the possibility to run on user selected input file.
628  if userInputFile!="":
629  previousOutputFile=userInputFile
630  else:
631  previousOutputFile = ""
632 
633  for x in range(start,stopIndex,1):
634  if stepIndex >= stopIndex:
635  break
636  step = steps[stepIndex]
637 
638  # One shot Profiling variables
639  befStep = step
640  aftStep = step
641  # We need this in case we are running one-shot profiling or for DIGI-PILEUP
642  stepToWrite = step
643 
644  CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
645  oneShotProf = False
646  hypsteps = []
647 
648  #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
649  if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)):
650 
651  #Checking now if the current step matches the first of a composite step in userSteps
652  hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
653  if not len(hypMatch) == 0 :
654  hypsteps = expandHyphens(hypMatch[0])
655  #print hypsteps
656  #print hypMatch[0]
657  stepToWrite = ",".join(hypsteps)
658  befStep = hypsteps[0]
659  #Kludge to avoid running HLT in composite steps if the -b option is chosen
660  if bypasshlt and hypsteps[-1]=='HLT':
661  aftStep = hypsteps[-2]
662  else:
663  aftStep = hypsteps[-1]
664  oneShotProf = True
665  if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
666  #print "Inside the filter if!"
667  #print stepToWrite
668  #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
669  #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
670  #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!)
671  stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
672  #print stepToWrite
673 
674  writeStepHead(simcandles,acandle,stepToWrite)
675 
676  #Set the output file name for Pile up and for regular case:
677  if '--pileup' in cmsDriverOptions:
678  outfile = stepToWrite + "_PILEUP"
679  else:
680  outfile = stepToWrite
681 
682  OutputFile = setOutputFileOption(acandle,outfile)
683  if fstROOTfile:
684  fstROOTfileStr = OutputFile
685  fstROOTfile = False
686  OutputFileOption = "--fileout=" + OutputFile
687 
688  for prof in Profile:
689  #First prepare the cmsDriver.py command
690 
691  #Special case of EventEdmSize profiling
692  if 'EdmSize' in prof:
693  EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
694  #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
695  if prof == Profile[0] and not os.path.exists("./" + EdmFile):
696  # insert command to generate required state ( need to run one more step
697  # so that EDM can actually check the size of the root file
698  # If the first step to be profiled is something later on in the steps such
699  # as HLT then writePrerequisteSteps() should have got you to the step prior to
700  # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
701 
702  InputFileOption = "--filein file:" + previousOutputFile
703  #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
704  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
705  InputFileOption = "--filein file:" + prevPrevOutputFile
706  if previousOutputFile == "":
707  InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
708  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
709  % (cmsDriver,
710  KeywordToCfi[acandle],
711  NumberOfEvents,
712  stepToWrite,
713  InputFileOption,
714  OutputFileOption,
715  CustomisePythonFragment,
716  cmsDriverOptions) )
717  simcandles.write( "%s @@@ None @@@ None\n" % (Command))
718 
719  Command = EdmFile
720  #all other profiles:
721  else:
722  InputFileOption = "--filein file:" + previousOutputFile
723  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
724  InputFileOption = "--filein file:" + prevPrevOutputFile
725 
726  if previousOutputFile == "":
727  InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
728 
729  #if '--pileup' in cmsDriverOptions:
730  # stepToWrite = pileupStep
731  if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'):
732  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
733  cmsDriver,
734  KeywordToCfi[acandle],
735  NumberOfEvents,
736  stepToWrite,
737  InputFileOption,
738  OutputFileOption,
739  CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
740  cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
741  ))
742  elif '--pileup' in cmsDriverOptions and stepToWrite=='GEN,FASTSIM': #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
743  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
744  cmsDriver,
745  KeywordToCfi[acandle],
746  NumberOfEvents,
747  stepToWrite,
748  InputFileOption,
749  OutputFileOption,
750  CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
751  cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')]
752  ))
753 
754  else:
755  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
756  cmsDriver,
757  KeywordToCfi[acandle],
758  NumberOfEvents,
759  stepToWrite,
760  InputFileOption,
761  OutputFileOption,
762  CustomisePythonFragment,
763  cmsDriverOptions
764  ))
765  #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
766  if _noprof:
767  simcandles.write("%s @@@ None @@@ None\n" % Command)
768  else:
769  if 'valgrind' in prof:
770  #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
771  if 'TimeMemoryInfo.py' in Command:
772  Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
773  #There is actually also the SIM case to take care of:
774  elif 'TimeMemoryG4Info.py' in Command:
775  Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
776  stepLabel=stepToWrite
777  #print "stepLabel is %s"%stepLabel
778  if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
779  stepLabel = stepToWrite+"_PILEUP"
780  #Add kludge here to make the IGPROF ANALYSE work:
781  #The idea is to:
782  #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
783  #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
784  #[3-Catch the case of ANALYSE by itself and raise an exception]
785  #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
786  #print "EEEEEEE %s"%prof
787  #We actually want the analyse to be used also for IgProf perf!
788  if 'IgProf' in prof: # and 'perf' not in prof:
789  #New IgProf naming scheme:
790  #Add a new naming scheme to simplify the handling in igprof-navigator:
791  #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
792 
793  #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
794  IgProfProfile="IgProf"
795  if "Mem" in prof:
796  if "reuse" in prof:
797  IgProfProfile="IgProfMem @@@ reuse"
798  else:
799  IgProfProfile="IgProfMem"
800  else:
801  if "reuse" in prof:
802  IgProfProfile="IgProfPerf @@@ reuse"
803  else:
804  IgProfProfile="IgProfPerf"
805 
806  #Conditions and PileUpOption are extracted from the cmsDriverOptions:
807  Conditions="UNKNOWN_CONDITIONS"
808  PileUp="NOPILEUP"
809  EventContent="UNKNOWN_EVENTCONTENT"
810  tokens=cmsDriverOptions.split("--")
811  for token in tokens:
812  keywords=token.split(" ")
813  if "=" in keywords[0]:
814  keywords=keywords[0].split("=")
815  if "conditions" in keywords[0]:
816  # check if we are using the autoCond style of flexible conditions
817  # if so, expand the condition here so that the file names contain the real conditions
818  if "auto:" in keywords[1]:
819  from Configuration.PyReleaseValidation.autoCond import autoCond
820  fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
821  Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0]
822  else:
823  # old style, hardcoded, conditions ...
824  # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
825  # and ::All at the end:
826  # Added if to handle new --conditions convention for cmsDriver.py (no more
827  # FrontierConditions... in front of it)
828  if "," in keywords[1]:
829  Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
830  else:
831  Conditions=keywords[1].split("::")[0]
832  elif "pileup" in keywords[0]:
833  PileUp=keywords[1]
834  elif "eventcontent" in keywords[0]:
835  EventContent=keywords[1]
836  #so here's our new MetaName:
837  MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
838 
839  if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
840 
841  simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
842  Profiler[prof],
843  MetaName))
844  #FileName[acandle],
845  #stepLabel,
846  #prof))
847  simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
848  Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
849  MetaName))
850  #FileName[acandle],
851  #stepLabel,
852  #prof))
853  elif 'Analyse' in prof:
854  pass
855  else:
856  simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
857  Profiler[prof],
858  FileName[acandle],
859  stepLabel,
860  prof))
861 
862  if debug:
863  print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
864  print "cmsDriveroptions : " + cmsDriverOption
865  prevPrevOutputFile = previousOutputFile
866  previousOutputFile = OutputFile
867  else:
868  unprofiledSteps.append(step)
869  isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
870 
871  try:
872  isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
873  #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
874  except IndexError:
875  # This loop should have terminated if x + 1 is out of bounds!
876  print "Error: Something is wrong we shouldn't have come this far"
877  break
878 
879  if isNextStepForProfile:
880  #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
881  OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
882  unprofiledSteps = []
883  prevPrevOutputFile = previousOutputFile
884  previousOutputFile = OutputFile
885  #Dangerous index handling when looping over index x!
886  if oneShotProf:
887  stepIndex += len(hypsteps)
888  else:
889  stepIndex +=1
890  return fstROOTfileStr
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
align_tpl.filter
tuple filter
USE THIS FOR SKIMMED TRACKS process.p = cms.Path(process.hltLevel1GTSeed*process.skimming*process.offlineBeamSpot*process.TrackRefitter2) OTHERWISE USE THIS.
Definition: align_tpl.py:86
split
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeCommandsToReport (   simcandles,
  Candle,
  Profile,
  debug,
  NumberOfEvents,
  cmsDriverOptions,
  steps,
  bypasshlt,
  userInputFile 
)

Definition at line 891 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by main().

891 
892 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile):
893 
894  OutputStep = ''
895 
896  # This option will not be used for now since the event content names keep changing, it can be
897  # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
898  #
899  #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
900 
901  for acandle in Candle:
902  print '*Candle ' + acandle
903 
904  ##################
905  # If the first profiling we run is EdmSize we need to create the root file first
906  #
907 
908  #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
909  #print "1-userInputFile: %s"%userInputFile
910 
911  fstoutfile = writeCommands(simcandles,
912  Profile,
913  acandle,
914  steps,
915  NumberOfEvents,
916  cmsDriverOptions,
917  bypasshlt,
918  0,
919  False,
920  userInputFile)
921 
def cmsRelvalreportInput.writePrerequisteSteps (   simcandles,
  steps,
  acandle,
  NumberOfEvents,
  cmsDriverOptions,
  pileup,
  bypasshlt 
)

Definition at line 525 of file cmsRelvalreportInput.py.

References pythonFragment(), split, and writeUnprofiledSteps().

Referenced by writeCommands().

525 
526 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
527  fstIdx = -1
528  #Hack for GEN-FASTSIM workflow:
529  #if steps=="GEN,FASTSIM":
530  #if "-" in steps[0]:
531  fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
532  #else:
533  # fstIdx = AllSteps.index(steps[0])
534  CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
535  previousOutputFile=""
536  OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt)
537  return (fstIdx, OutputFile)
split
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeReportFileHeader (   CMSSW_VERSION,
  CMSSW_RELEASE_BASE,
  CMSSW_BASE 
)

Definition at line 359 of file cmsRelvalreportInput.py.

Referenced by main().

359 
360 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
361 
362  SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
363 
364  try:
365  simcandles = open(SimCandlesFile, 'w')
366  except IOError:
367  print "Couldn't open %s to save" % SimCandlesFile
368 
369  simcandles.write('#Candles file automatically generated by %s for %s\n'
370  % (THIS_PROG_NAME, CMSSW_VERSION))
371  simcandles.write("#CMSSW Base : %s\n" % CMSSW_BASE)
372  simcandles.write("#Release Base: %s\n" % CMSSW_RELEASE_BASE)
373  simcandles.write("#Version : %s\n\n" % CMSSW_VERSION)
374 
375  return simcandles
def cmsRelvalreportInput.writeStepHead (   simcandles,
  acandle,
  step 
)

Definition at line 422 of file cmsRelvalreportInput.py.

Referenced by writeCommands().

422 
423 def writeStepHead(simcandles,acandle,step):
424  simcandles.write('#%s\n' % FileName[acandle])
425  simcandles.write('#Step %s\n' % step)
426  #print step
427 
def cmsRelvalreportInput.writeUnprofiledSteps (   simcandles,
  CustomisePythonFragment,
  cmsDriverOptions,
  unprofiledSteps,
  previousOutputFile,
  acandle,
  NumberOfEvents,
  stepIndex,
  pileup,
  bypasshlt 
)

Definition at line 478 of file cmsRelvalreportInput.py.

References join(), and setInputFile().

Referenced by writeCommands(), and writePrerequisteSteps().

478 
479 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
480  # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
481  #stepsStr = ",".join(unprofiledSteps)
482 
483 
484  #print "unprofiledSteps is %s"%unprofiledSteps
485  #print "22acandle is %s"%acandle
486  #Kludge in case -b option to skip HLT is used...
487  if bypasshlt and unprofiledSteps[-1]=="HLT":
488  stepsStr = ",".join(unprofiledSteps[:-1])
489  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
490  else:
491  stepsStr = ",".join(unprofiledSteps)
492  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
493  simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
494  OutputFileOption = "--fileout=%s" % OutputFile
495  #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
496  #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
497  #Adding the argument bypasshlt to the calls...
498  #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
499  #print "StepIndex 1:%s"%stepIndex
500  #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
501  #stepIndexAdjust=stepIndex - 2
502  #if stepIndexAdjust < 0: #To avoid issues with negative indeces
503  # stepIndexAdjust=0
504  #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
505  #Use an argument to make it easier:
506  InputFileOption = "--filein file:" + previousOutputFile
507  if previousOutputFile =="":
508  InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
509  #Introduce an over-ride of cmsDriverOptions:
510  #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
511  #At the moment the only use case is when running step2 on its own...
512  for eventcontent in EventContents:
513  cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
514  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
515  % (cmsDriver,
516  KeywordToCfi[acandle],
517  NumberOfEvents,
518  stepsStr,
519  InputFileOption,
520  OutputFileOption,
521  CustomisePythonFragment,
522  cmsDriverOptions) )
523  simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
524  return OutputFile
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18

Variable Documentation

list cmsRelvalreportInput.AfterPileUpSteps = []

Definition at line 37 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]

Definition at line 36 of file cmsRelvalreportInput.py.

string cmsRelvalreportInput.cmsDriver = 'cmsDriver.py'

Definition at line 32 of file cmsRelvalreportInput.py.

cmsRelvalreportInput.debug = False

Definition at line 34 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.DEF_STEPS = ['GEN,SIM', 'DIGI']

Definition at line 35 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.hypreg = re.compile('-')

Definition at line 33 of file cmsRelvalreportInput.py.

dictionary cmsRelvalreportInput.Profiler
Initial value:
1 = {
2  'TimingReport' : 'Timing_Parser',
3  'TimingReport @@@ reuse' : 'Timing_Parser',
4  'TimeReport' : 'Timereport_Parser',
5  'TimeReport @@@ reuse' : 'Timereport_Parser',
6  'SimpleMemReport' : 'SimpleMem_Parser',
7  'EdmSize' : 'Edm_Size',
8  'IgProfperf' : 'IgProf_perf.PERF_TICKS',
9  'IgProfperf @@@ reuse' : 'IgProf_perf.PERF_TICKS',
10  'IgProfMemTotal' : 'IgProf_mem.MEM_TOTAL',
11  'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
12  'IgProfMemLive' : 'IgProf_mem.MEM_LIVE',
13  'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
14  'IgProfMemMax' : 'IgProf_mem.MEM_MAX',
15  'IgProfMemMax @@@ reuse' : 'IgProf_mem.MEM_MAX',
16  'IgProfMemAnalyse' : 'IgProf_mem.ANALYSE',
17  'valgrind' : 'ValgrindFCE',
18  'memcheck_valgrind' : 'Memcheck_Valgrind',
19  'None' : 'None',
20 }

Definition at line 43 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.THIS_PROG_NAME = os.path.basename(sys.argv[0])

Global variables.

Definition at line 31 of file cmsRelvalreportInput.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5