test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsRelvalreportInput Namespace Reference

Functions

def checkSteps
 
def determineNewProfile
 
def expandHyphens
 
def getFstOccur
 
def getLstIndex
 
def getProfileArray
 
def getSteps
 
def init_vars
 
def main
 
def optionparse
 
def pythonFragment
 
def setInputFile
 
def setOutputFileOption
 
def setupProgramParameters
 
def writeCommands
 
def writeCommandsToReport
 
def writePrerequisteSteps
 
def writeReportFileHeader
 
def writeStepHead
 
def writeUnprofiledSteps
 

Variables

list AfterPileUpSteps = []
 
list AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
 
string cmsDriver = 'cmsDriver.py'
 
 debug = False
 
list DEF_STEPS = ['GEN,SIM', 'DIGI']
 
tuple hypreg = re.compile('-')
 
dictionary Profiler
 
tuple THIS_PROG_NAME = os.path.basename(sys.argv[0])
 Global variables. More...
 

Function Documentation

def cmsRelvalreportInput.checkSteps (   steps)
Checks user steps <steps> for order against steps defined in AllSteps list.
If they are not, prints a warning and exits.
No return value.

Definition at line 76 of file cmsRelvalreportInput.py.

References split.

Referenced by getSteps().

76 
77 def checkSteps(steps):
78  '''Checks user steps <steps> for order against steps defined in AllSteps list.
79  If they are not, prints a warning and exits.
80  No return value.'''
81 
82  #print steps
83  #if ":" in steps:
84  # steps.split(
85  idx = -2
86  lstidx = -2
87  for step in steps:
88  astep = step
89  split = []
90  #Added special workflow for GEN-FASTSIM, bypass the check of step order:
91  if step=="GEN,FASTSIM":
92  pass
93  else:
94  if "-" in step:
95  split = astep.split("-")
96  astep = split[0].split(":")[0]
97  # print "This is astep:%s"%astep
98  #idx =AllSteps.index("GEN,SIM")
99  #print idx
100  #print astep
101  idx = AllSteps.index(astep.split(":")[0])
102  if not ( idx == -2 ):
103  if lstidx > idx:
104  print "ERROR: Your user defined steps are not in a valid order"
105  sys.exit()
106  lstidx = idx
107  if "-" in step:
108  lstidx = AllSteps.index(split[1].split(":")[0])
109 
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.determineNewProfile (   step,
  Profile,
  SavedProfile 
)

Definition at line 429 of file cmsRelvalreportInput.py.

430 def determineNewProfile(step,Profile,SavedProfile):
431  if 'DIGI2RAW' in step:
432  SavedProfile = Profile
433  Profile = [ ]
434  if 'HLT' in step:
435  Profile = SavedProfile
436 
437  return (Profile, SavedProfile)
def cmsRelvalreportInput.expandHyphens (   step)

Definition at line 261 of file cmsRelvalreportInput.py.

References alcazmumu_cfi.filter, and split.

Referenced by writeCommands().

262 def expandHyphens(step):
263  newsteps = []
264 # for step in steps:
265  if "-" in step:
266  hypsteps = step.split(r"-")
267  #if not (len(hypsteps) == 2):
268  # print "ERROR: Could not parse usersteps. You have too many hypens between commas"
269  # sys.exit()
270  #el
271  if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
272  #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
273  print "ERROR: One of the steps you defined is invalid"
274  sys.exit()
275  else:
276  if (hypsteps[0] == hypsteps[1]):
277  print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
278  newsteps.append(hypsteps[0])
279  else:
280  newsteps.append(hypsteps[0])
281 
282  srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
283  for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
284  astep = AllSteps[n]
285  if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
286  #break
287  astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
288  if astep==hypsteps[-1].split(":")[0]:
289  break
290  newsteps.append(astep)
291  newsteps.append(hypsteps[-1])
292  else:
293  if not (step.split(":")[0] in AllSteps):
294  print "ERROR: One of the steps you defined is invalid"
295  sys.exit()
296  else:
297  newsteps.append(step)
298 
299  return newsteps
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.getFstOccur (   item,
  list 
)

Definition at line 67 of file cmsRelvalreportInput.py.

References alcazmumu_cfi.filter.

67 
68 def getFstOccur(item, list):
69  return filter(item.__eq__,list)[0]
def cmsRelvalreportInput.getLstIndex (   item,
  list 
)

Definition at line 70 of file cmsRelvalreportInput.py.

70 
71 def getLstIndex(item, list):
72  lenlist = len(list)
73  for x in range(lenlist - 1,0,-1):
74  if list[x] == item:
75  return x
def cmsRelvalreportInput.getProfileArray (   ProfileCode)

Definition at line 377 of file cmsRelvalreportInput.py.

Referenced by main().

378 def getProfileArray(ProfileCode):
379 
380  Profile = []
381 
382  # The allowed profiles are:
383 
384  AllowedProfile = [
385  'TimingReport',
386  'TimeReport',
387  'SimpleMemReport',
388  'EdmSize',
389  'IgProfperf',
390  'IgProfMemTotal',
391  'IgProfMemLive',
392  #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
393  'IgProfMemAnalyse',
394  'valgrind',
395  'memcheck_valgrind',
396  'None',
397  ]
398 
399  if _noprof:
400  Profile.append(AllowedProfile[-1])
401  else:
402  #FIXME:
403  #Horrible code!
404  for i in range(10):
405  if str(i) in ProfileCode:
406  firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
407  or (i == 1 and str(2) in ProfileCode)
408  secCase = (i==4 and str(7) in ProfileCode) \
409  or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
410  or (i == 6 and str(7) in ProfileCode)
411 
412  if firstCase or secCase:
413  Profile.append(AllowedProfile[i] + ' @@@ reuse')
414  #Here's the hack:
415  if i==6: #i.e. IgProfMemLive
416  Profile.append("IgProfMemMax @@@ reuse")
417  else:
418  Profile.append(AllowedProfile[i])
419  if i==6: #i.e. IgProfMemLive
420  Profile.append("IgProfMemMax")
421 
422  return Profile
def cmsRelvalreportInput.getSteps (   userSteps)
Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
Returns a list of steps <steps>.

Definition at line 110 of file cmsRelvalreportInput.py.

References checkSteps().

Referenced by setupProgramParameters().

111 def getSteps(userSteps):
112  '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
113  Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
114  Returns a list of steps <steps>.'''
115  #Add handling of ":" in steps
116  #print userSteps
117  # Then split the user steps into "steps"
118  gsreg = re.compile('GEN-SIM')
119  greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
120  gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
121  StepsTokens = userSteps.split(",")
122  steps = []
123  for astep in StepsTokens:
124 
125  # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
126  # from using the "-" to using the "," to match cmsDriver.py convention
127 
128  if gsreg.search(astep):
129  astep = gsreg.sub(r"GEN,SIM", astep)
130  elif gfsreg.search(astep):
131  astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
132  elif greg.search(astep):
133  astep = greg.sub(r"GEN,SIM", astep)
134 
135 
136  #print astep
137  # Finally collect all the steps into the steps list:
138 
139  steps.append(astep)
140 
141  #steps = expandHypens(steps)
142  checkSteps(steps)
143  return steps
def cmsRelvalreportInput.init_vars ( )

Definition at line 341 of file cmsRelvalreportInput.py.

Referenced by main().

342 def init_vars():
343 
344  ####################
345  # Obtain required environment variables
346  #
347 
348  try:
349  CMSSW_BASE = os.environ['CMSSW_BASE']
350  CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
351  CMSSW_VERSION = os.environ['CMSSW_VERSION']
352  except KeyError:
353  print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
354  print ' Please run eval `scramv1 runtime -csh` to set your environment variables'
355  sys.exit()
356 
357  return ( CMSSW_BASE,
358  CMSSW_RELEASE_BASE,
359  CMSSW_VERSION)
def cmsRelvalreportInput.main (   argv = sys.argv)

Definition at line 947 of file cmsRelvalreportInput.py.

References getProfileArray(), init_vars(), optionparse(), setupProgramParameters(), writeCommandsToReport(), and writeReportFileHeader().

948 def main(argv=sys.argv):
949 
950  #####################
951  # Options Parser
952  #
953 
954  (options, args) = optionparse()
955 
956  #####################
957  # Set up arguments and option handling
958  #
959 
960  (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
961  #print setupProgramParameters(options,args)
962 
963  ######################
964  # Initialize a few variables
965  #
966 
967  (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
968 
969  ##################
970  # Ok everything is ready now we need to create the input file for the relvalreport script
971  #
972 
973  simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
974 
975  ##################
976  # Based on the profile code create the array of profiles to run:
977  #
978 
979  Profile = getProfileArray(ProfileCode)
980 
981 
982  ##################
983  # Write the commands for the report to the file
984  #
985 
986  writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
987 
988  print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
989 
990  simcandles.close()
def cmsRelvalreportInput.optionparse ( )

Definition at line 144 of file cmsRelvalreportInput.py.

Referenced by main().

145 def optionparse():
146  global _noprof
147  explanations = map(lambda x: " " + x, Candles)
148  explanation = ""
149  for x in range(len(explanations)):
150  explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
151  parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
152 
153  Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
154 
155  Arguments:
156  NUM_EVENTS_PER_CFG - The number of events per config file
157 
158  CANDLES - The simulation type to perform profiling on
159  Candles Description
160  AllCandles Run all of the candles below
161 %s
162 
163  PROFILE - the type of profiling to perform (multiple codes can be used):
164  Code Profile Type
165  0 TimingReport
166  1 TimeReport
167  2 SimpleMemoryCheck
168  3 EdmSize
169  4 IgProfPerf
170  5 IgProfMemTotal
171  6 IgProfMemLive
172  7 IgProfAnalyse
173  8 ValgrindFCE
174  9 ValgrindMemCheck
175 
176  Examples (could be obsolete!):
177  Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
178  ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
179  Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
180  ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
181  Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
182  ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
183  Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
184  ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
185  % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
186 
187  devel = opt.OptionGroup(parser, "Developer Options",
188  "Caution: use these options at your own risk."
189  "It is believed that some of them bite.\n")
190  #parser.set_defaults(debug=False)
191 
192  parser.add_option(
193  '-b',
194  '--bypass-hlt',
195  action="store_true",
196  dest='bypasshlt',
197  default=False,
198  help='Should we bypass using the HLT root file?'
199  )
200 
201  parser.add_option(
202  '-u',
203  '--usersteps',
204  type='string',
205  dest='userSteps',
206  help='Which steps to run',
207  metavar='<STEPS>',
208  )
209  parser.add_option(
210  '-c',
211  '--cmsdriver',
212  type='string',
213  dest='cmsDriverOptions',
214  help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
215  metavar='<CONDITION>',
216  )
217  parser.add_option(
218  '-i',
219  '--filein',
220  type='string',
221  dest='userInputFile',
222  help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
223  default='',
224  metavar='<INPUTFILE>',
225  )
226  devel.add_option(
227  '-1',
228  '--no-profile',
229  action="store_true",
230  dest='noprof',
231  help='Do not perform profiling, ever',
232  #metavar='DEBUG',
233  )
234 
235  devel.add_option(
236  '-d',
237  '--debug',
238  action="store_true",
239  dest='debug',
240  help='Show debug output',
241  #metavar='DEBUG',
242  )
243 
244  parser.set_defaults(debug=False,noprof=False)
245  parser.add_option_group(devel)
246 
247  (options, args) = parser.parse_args()
248 
249 
250  _noprof = options.noprof
251  debug = options.debug
252  numofargs = len(args)
253 # print "runtime cmsinput " + str(numofargs) + " noprof " + str(_noprof)
254  if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
255  parser.error("There are not enough arguments specified to run this program."
256  " Please determine the correct arguments from the usage information above."
257  " Run with --help option for more information.")
258  sys.exit()
259 
260  return (options, args)
def cmsRelvalreportInput.pythonFragment (   step,
  cmsdriverOptions 
)

Definition at line 438 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writePrerequisteSteps().

439 def pythonFragment(step,cmsdriverOptions):
440  # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
441  #It is now living in cmsPerfCommons.py!
442 
443  if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
444  return CustomiseFragment['DIGI-PILEUP']
445  elif step in CustomiseFragment:
446  return CustomiseFragment[step]
447  else:
448  #This is a safe default in any case,
449  #the maintenance of the customise python fragments goes into cmsPerfCommons.py
450  return CustomiseFragment['DIGI']
451 
def cmsRelvalreportInput.setInputFile (   steps,
  step,
  acandle,
  stepIndex,
  pileup = False,
  bypasshlt = False 
)

Definition at line 452 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writeUnprofiledSteps().

453 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
454  #print "StepIndex 2:%s"%stepIndex
455  InputFileOption = ""
456  if pileup and stepIndex == 0:
457  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
458  else:
459  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
460 
461  if pileup:
462  pass
463  else :
464  if 'GEN,SIM' in step: # there is no input file for GEN,SIM!
465  InputFileOption = ''
466  if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
467  InputFileOption = ''
468  elif 'HLT' in steps[stepIndex - 1] and bypasshlt:
469 
470  # Special hand skipping of HLT since it is not stable enough, so it will not prevent
471  # RAW2DIGI,RECO from running
472 
473  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 2] )
474 
475  if not InputFileOption == "" :
476  InputFileOption += ".root "
477 
478  return InputFileOption
def cmsRelvalreportInput.setOutputFileOption (   acandle,
  endstep 
)

Definition at line 539 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by writeCommands().

540 def setOutputFileOption(acandle,endstep):
541  return "%s_%s.root" % ( FileName[acandle],endstep)
def cmsRelvalreportInput.setupProgramParameters (   options,
  args 
)

Definition at line 300 of file cmsRelvalreportInput.py.

References getSteps().

Referenced by main().

301 def setupProgramParameters(options,args):
302  steps = []
303  cmsDriverOptions = ""
304  global AfterPileUpSteps
305  NumberOfEvents = int(args[0]) # first arg
306  WhichCandles = str(args[1]) # second arg
307  ProfileCode = ""
308  if not _noprof:
309  ProfileCode = str(args[2]) # third arg
310 
311  if options.cmsDriverOptions:
312 
313  cmsDriverOptions = options.cmsDriverOptions
314  print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
315 
316  if options.userSteps:
317 
318  userSteps = options.userSteps
319  steps = getSteps(userSteps)
320 
321  if WhichCandles.lower() == 'allcandles':
322  Candle = Candles
323  print 'ALL standard simulation candles will be PROCESSED:'
324  else:
325  Candle = [WhichCandles]
326  print 'Candle %s will be PROCESSED' % Candle[0]
327 
328  # For now the two steps are built in, this can be added as an argument later
329  # Added the argument option so now this will only be defined if it was not defined already:
330 
331  if not steps:
332  print 'The default steps will be run:'
333  steps = DEF_STEPS
334  else:
335  print "You defined your own steps to run:"
336 
337  for astep in steps:
338  print astep
339 
340  return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
def cmsRelvalreportInput.writeCommands (   simcandles,
  Profile,
  acandle,
  steps,
  NumberOfEvents,
  cmsDriverOptions,
  bypasshlt,
  stepIndex = 0,
  pileup = False,
  userInputFile = "" 
)

Definition at line 551 of file cmsRelvalreportInput.py.

References expandHyphens(), alcazmumu_cfi.filter, join(), pythonFragment(), setInputFile(), setOutputFileOption(), split, writePrerequisteSteps(), writeStepHead(), and writeUnprofiledSteps().

Referenced by setOutputFileOption(), and writeCommandsToReport().

552  userInputFile=""):
553  #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
554  OutputStep = ""
555 
556  stopIndex = len(steps)
557  start = 0
558 
559  userSteps = steps
560  SavedProfile = []
561  fstROOTfile = True
562  fstROOTfileStr = ""
563 
564  #Handling the case of the first user step not being the first step (GEN,SIM):
565  #print "Steps passed to writeCommands %s",steps
566  #Special case for FASTSIM:
567  if (steps[0] == "GEN,FASTSIM"):
568  start=-1
569  stopIndex=1
570  steps = ["GEN,FASTSIM"]
571  else:
572  if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
573  #print "userInputFile: %s"%userInputFile
574  if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
575  userInputFile = "../INPUT_PILEUP_EVENTS.root"
576  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
577  rootFileStr=""
578 
579  elif userInputFile == "":
580  #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
581  (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
582  else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
583  #if "-" in steps[0]:
584  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
585  #print "Here!"
586  #else:
587  #print "There!"
588  # stepIndex=AllSteps.index(steps[0])
589  rootFileStr=""
590  #Now take care of setting the indeces and input root file name right for the profiling part...
591  if fstROOTfile:
592  fstROOTfileStr = rootFileStr
593  fstROOTfile = False
594  start = -1
595  #if "-" in steps[0]:
596  start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
597  #else:
598  # start = AllSteps.index(steps[0])
599  lst = - 1
600  if "-" in steps[-1]:
601  lst = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
602  else:
603  lst = AllSteps.index(steps[-1])
604  runSteps = AllSteps[start:lst]
605  numOfSteps = (lst - start) + 1
606  stopIndex = start + numOfSteps
607  #Handling the case in which the first user step is the same as the first step (GEN,SIM)
608  #elif not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
609  else:
610  #Handling the case of the last step being a composite one:
611  if "-" in steps[-1]:
612  #Set the stop index at the last step of the composite step (WHY???)
613  stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
614  else:
615 
616  stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
617  #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
618  steps = AllSteps
619 
620  unprofiledSteps = []
621  rawreg = re.compile("^RAW2DIGI")
622 
623 #Horrible structure... to be rewritten sooner or later...
624 
625 # FOR step in steps
626 
627  prevPrevOutputFile = ""
628  #Add if to introduce the possibility to run on user selected input file.
629  if userInputFile!="":
630  previousOutputFile=userInputFile
631  else:
632  previousOutputFile = ""
633 
634  for x in range(start,stopIndex,1):
635  if stepIndex >= stopIndex:
636  break
637  step = steps[stepIndex]
638 
639  # One shot Profiling variables
640  befStep = step
641  aftStep = step
642  # We need this in case we are running one-shot profiling or for DIGI-PILEUP
643  stepToWrite = step
644 
645  CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
646  oneShotProf = False
647  hypsteps = []
648 
649  #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
650  if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)):
651 
652  if "GEN,FASTSIM" in userSteps: # HLT/L1 menu workaround
653  stepToWrite = stepToWrite + ",HLT:GRun"
654  elif "HLT" in userSteps:
655  stepToWrite = stepToWrite.replace("HLT", "HLT:GRun")
656 
657  #Checking now if the current step matches the first of a composite step in userSteps
658  hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
659  if not len(hypMatch) == 0 :
660  hypsteps = expandHyphens(hypMatch[0])
661  #print hypsteps
662  #print hypMatch[0]
663  stepToWrite = ",".join(hypsteps)
664  if "GEN,SIM-HLT" in userSteps: # HLT/L1 menu workaround
665  stepToWrite = stepToWrite.replace("HLT","HLT:GRun")
666  befStep = hypsteps[0]
667  #Kludge to avoid running HLT in composite steps if the -b option is chosen
668  if bypasshlt and hypsteps[-1]=='HLT':
669  aftStep = hypsteps[-2]
670  else:
671  aftStep = hypsteps[-1]
672  oneShotProf = True
673  if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
674  #print "Inside the filter if!"
675  #print stepToWrite
676  #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
677  #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
678  #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!)
679  stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
680  #print stepToWrite
681 
682  writeStepHead(simcandles,acandle,stepToWrite)
683 
684  #Set the output file name for Pile up and for regular case:
685  if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change to make root file output work (without colons)
686  stepToWrite = stepToWrite.replace(":","=")
687 
688  if '--pileup' in cmsDriverOptions:
689  outfile = stepToWrite + "_PILEUP"
690  else:
691  outfile = stepToWrite
692 
693  #print stepToWrite
694 
695  if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change it back
696  stepToWrite = stepToWrite.replace("=",":")
697 
698  OutputFile = setOutputFileOption(acandle,outfile)
699  if fstROOTfile:
700  fstROOTfileStr = OutputFile
701  fstROOTfile = False
702  OutputFileOption = "--fileout=" + OutputFile
703 
704  for prof in Profile:
705  #First prepare the cmsDriver.py command
706 
707  #Special case of EventEdmSize profiling
708  if 'EdmSize' in prof:
709  EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
710  #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
711  if prof == Profile[0] and not os.path.exists("./" + EdmFile):
712  # insert command to generate required state ( need to run one more step
713  # so that EDM can actually check the size of the root file
714  # If the first step to be profiled is something later on in the steps such
715  # as HLT then writePrerequisteSteps() should have got you to the step prior to
716  # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
717 
718  InputFileOption = "--filein file:" + previousOutputFile
719  #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
720  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
721  InputFileOption = "--filein file:" + prevPrevOutputFile
722  if previousOutputFile == "":
723  InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
724  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
725  % (cmsDriver,
726  KeywordToCfi[acandle],
727  NumberOfEvents,
728  stepToWrite,
729  InputFileOption,
730  OutputFileOption,
731  CustomisePythonFragment,
732  cmsDriverOptions) )
733  simcandles.write( "%s @@@ None @@@ None\n" % (Command))
734 
735  Command = EdmFile
736  #all other profiles:
737  else:
738  InputFileOption = "--filein file:" + previousOutputFile
739  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
740  InputFileOption = "--filein file:" + prevPrevOutputFile
741 
742  if previousOutputFile == "":
743  InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
744 
745  #if '--pileup' in cmsDriverOptions:
746  # stepToWrite = pileupStep
747  if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'):
748  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
749  cmsDriver,
750  KeywordToCfi[acandle],
751  NumberOfEvents,
752  stepToWrite,
753  InputFileOption,
754  OutputFileOption,
755  CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
756  cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
757  ))
758  elif '--pileup' in cmsDriverOptions and (stepToWrite=='GEN,FASTSIM' or stepToWrite=='GEN,FASTSIM,HLT:GRun'): #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
759  #+exception to the HLT:GRun case
760  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
761  cmsDriver,
762  KeywordToCfi[acandle],
763  NumberOfEvents,
764  stepToWrite,
765  InputFileOption,
766  OutputFileOption,
767  CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
768  cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')]
769  ))
770 
771  else:
772  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
773  cmsDriver,
774  KeywordToCfi[acandle],
775  NumberOfEvents,
776  stepToWrite,
777  InputFileOption,
778  OutputFileOption,
779  CustomisePythonFragment,
780  cmsDriverOptions
781  ))
782  #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
783  if _noprof:
784  simcandles.write("%s @@@ None @@@ None\n" % Command)
785  else:
786  if 'valgrind' in prof:
787  #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
788  if 'TimeMemoryInfo.py' in Command:
789  Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
790  #There is actually also the SIM case to take care of:
791  elif 'TimeMemoryG4Info.py' in Command:
792  Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
793  stepLabel=stepToWrite
794  #print "stepLabel is %s"%stepLabel
795  if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
796  stepLabel = stepToWrite+"_PILEUP"
797  #Add kludge here to make the IGPROF ANALYSE work:
798  #The idea is to:
799  #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
800  #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
801  #[3-Catch the case of ANALYSE by itself and raise an exception]
802  #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
803  #print "EEEEEEE %s"%prof
804  #We actually want the analyse to be used also for IgProf perf!
805  if 'IgProf' in prof: # and 'perf' not in prof:
806  #New IgProf naming scheme:
807  #Add a new naming scheme to simplify the handling in igprof-navigator:
808  #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
809 
810  #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
811  IgProfProfile="IgProf"
812  if "Mem" in prof:
813  if "reuse" in prof:
814  IgProfProfile="IgProfMem @@@ reuse"
815  else:
816  IgProfProfile="IgProfMem"
817  else:
818  if "reuse" in prof:
819  IgProfProfile="IgProfPerf @@@ reuse"
820  else:
821  IgProfProfile="IgProfPerf"
822 
823  #Conditions and PileUpOption are extracted from the cmsDriverOptions:
824  Conditions="UNKNOWN_CONDITIONS"
825  PileUp="NOPILEUP"
826  EventContent="UNKNOWN_EVENTCONTENT"
827  tokens=cmsDriverOptions.split("--")
828  for token in tokens:
829  keywords=token.split(" ")
830  if "=" in keywords[0]:
831  keywords=keywords[0].split("=")
832  if "conditions" in keywords[0]:
833  # check if we are using the autoCond style of flexible conditions
834  # if so, expand the condition here so that the file names contain the real conditions
835  if "auto:" in keywords[1]:
836  from Configuration.AlCa.autoCond import autoCond
837  fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
838  Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0]
839  else:
840  # old style, hardcoded, conditions ...
841  # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
842  # and ::All at the end:
843  # Added if to handle new --conditions convention for cmsDriver.py (no more
844  # FrontierConditions... in front of it)
845  if "," in keywords[1]:
846  Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
847  else:
848  Conditions=keywords[1].split("::")[0]
849  elif "pileup" in keywords[0]:
850  PileUp=keywords[1]
851  elif "eventcontent" in keywords[0]:
852  EventContent=keywords[1]
853 
854  #so here's our new MetaName:
855  MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
856 
857  if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
858 
859  simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
860  Profiler[prof],
861  MetaName))
862  #FileName[acandle],
863  #stepLabel,
864  #prof))
865  analyse_command = ("%s @@@ %s @@@ %s\n" % (Command,
866  Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
867  MetaName))
868  #FileName[acandle],
869  #stepLabel,
870  #prof))
871 
872  #for the relval case, GEN-SIM,HLT get rid of the last " @@@ reuse":
873  if Profiler[prof] == 'IgProf_mem.MEM_MAX' or Profiler[prof] == 'IgProf_perf.PERF_TICKS':
874  analyse_command = analyse_command.replace("@@@ reuse", "")
875 
876  simcandles.write(analyse_command)
877 
878  elif 'Analyse' in prof:
879  pass
880  else:
881  simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
882  Profiler[prof],
883  FileName[acandle],
884  stepLabel,
885  prof))
886 
887  if debug:
888  print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
889  print "cmsDriveroptions : " + cmsDriverOption
890  prevPrevOutputFile = previousOutputFile
891  previousOutputFile = OutputFile
892  else:
893  unprofiledSteps.append(step)
894  isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
895 
896  try:
897  isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
898  #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
899  except IndexError:
900  # This loop should have terminated if x + 1 is out of bounds!
901  print "Error: Something is wrong we shouldn't have come this far"
902  break
903 
904  if isNextStepForProfile:
905  #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
906  OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
907  unprofiledSteps = []
908  prevPrevOutputFile = previousOutputFile
909  previousOutputFile = OutputFile
910  #Dangerous index handling when looping over index x!
911  if oneShotProf:
912  stepIndex += len(hypsteps)
913  else:
914  stepIndex +=1
915  return fstROOTfileStr
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeCommandsToReport (   simcandles,
  Candle,
  Profile,
  debug,
  NumberOfEvents,
  cmsDriverOptions,
  steps,
  bypasshlt,
  userInputFile 
)

Definition at line 916 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by main().

917 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile):
918 
919  OutputStep = ''
920 
921  # This option will not be used for now since the event content names keep changing, it can be
922  # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
923  #
924  #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
925 
926  for acandle in Candle:
927  print '*Candle ' + acandle
928 
929  ##################
930  # If the first profiling we run is EdmSize we need to create the root file first
931  #
932 
933  #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
934  #print "1-userInputFile: %s"%userInputFile
935 
936  fstoutfile = writeCommands(simcandles,
937  Profile,
938  acandle,
939  steps,
940  NumberOfEvents,
941  cmsDriverOptions,
942  bypasshlt,
943  0,
944  False,
945  userInputFile)
946 
def cmsRelvalreportInput.writePrerequisteSteps (   simcandles,
  steps,
  acandle,
  NumberOfEvents,
  cmsDriverOptions,
  pileup,
  bypasshlt 
)

Definition at line 526 of file cmsRelvalreportInput.py.

References pythonFragment(), split, and writeUnprofiledSteps().

Referenced by writeCommands().

527 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
528  fstIdx = -1
529  #Hack for GEN-FASTSIM workflow:
530  #if steps=="GEN,FASTSIM":
531  #if "-" in steps[0]:
532  fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
533  #else:
534  # fstIdx = AllSteps.index(steps[0])
535  CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
536  previousOutputFile=""
537  OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt)
538  return (fstIdx, OutputFile)
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeReportFileHeader (   CMSSW_VERSION,
  CMSSW_RELEASE_BASE,
  CMSSW_BASE 
)

Definition at line 360 of file cmsRelvalreportInput.py.

Referenced by main().

361 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
362 
363  SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
364 
365  try:
366  simcandles = open(SimCandlesFile, 'w')
367  except IOError:
368  print "Couldn't open %s to save" % SimCandlesFile
369 
370  simcandles.write('#Candles file automatically generated by %s for %s\n'
371  % (THIS_PROG_NAME, CMSSW_VERSION))
372  simcandles.write("#CMSSW Base : %s\n" % CMSSW_BASE)
373  simcandles.write("#Release Base: %s\n" % CMSSW_RELEASE_BASE)
374  simcandles.write("#Version : %s\n\n" % CMSSW_VERSION)
375 
376  return simcandles
def cmsRelvalreportInput.writeStepHead (   simcandles,
  acandle,
  step 
)

Definition at line 423 of file cmsRelvalreportInput.py.

Referenced by writeCommands().

424 def writeStepHead(simcandles,acandle,step):
425  simcandles.write('#%s\n' % FileName[acandle])
426  simcandles.write('#Step %s\n' % step)
427  #print step
428 
def cmsRelvalreportInput.writeUnprofiledSteps (   simcandles,
  CustomisePythonFragment,
  cmsDriverOptions,
  unprofiledSteps,
  previousOutputFile,
  acandle,
  NumberOfEvents,
  stepIndex,
  pileup,
  bypasshlt 
)

Definition at line 479 of file cmsRelvalreportInput.py.

References join(), and setInputFile().

Referenced by writeCommands(), and writePrerequisteSteps().

480 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
481  # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
482  #stepsStr = ",".join(unprofiledSteps)
483 
484 
485  #print "unprofiledSteps is %s"%unprofiledSteps
486  #print "22acandle is %s"%acandle
487  #Kludge in case -b option to skip HLT is used...
488  if bypasshlt and unprofiledSteps[-1]=="HLT":
489  stepsStr = ",".join(unprofiledSteps[:-1])
490  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
491  else:
492  stepsStr = ",".join(unprofiledSteps)
493  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
494  simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
495  OutputFileOption = "--fileout=%s" % OutputFile
496  #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
497  #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
498  #Adding the argument bypasshlt to the calls...
499  #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
500  #print "StepIndex 1:%s"%stepIndex
501  #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
502  #stepIndexAdjust=stepIndex - 2
503  #if stepIndexAdjust < 0: #To avoid issues with negative indeces
504  # stepIndexAdjust=0
505  #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
506  #Use an argument to make it easier:
507  InputFileOption = "--filein file:" + previousOutputFile
508  if previousOutputFile =="":
509  InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
510  #Introduce an over-ride of cmsDriverOptions:
511  #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
512  #At the moment the only use case is when running step2 on its own...
513  for eventcontent in EventContents:
514  cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
515  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
516  % (cmsDriver,
517  KeywordToCfi[acandle],
518  NumberOfEvents,
519  stepsStr,
520  InputFileOption,
521  OutputFileOption,
522  CustomisePythonFragment,
523  cmsDriverOptions) )
524  simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
525  return OutputFile
static std::string join(char **cmd)
Definition: RemoteFile.cc:18

Variable Documentation

list cmsRelvalreportInput.AfterPileUpSteps = []

Definition at line 38 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]

Definition at line 37 of file cmsRelvalreportInput.py.

string cmsRelvalreportInput.cmsDriver = 'cmsDriver.py'

Definition at line 33 of file cmsRelvalreportInput.py.

cmsRelvalreportInput.debug = False

Definition at line 35 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.DEF_STEPS = ['GEN,SIM', 'DIGI']

Definition at line 36 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.hypreg = re.compile('-')

Definition at line 34 of file cmsRelvalreportInput.py.

dictionary cmsRelvalreportInput.Profiler
Initial value:
1 = {
2  'TimingReport' : 'Timing_Parser',
3  'TimingReport @@@ reuse' : 'Timing_Parser',
4  'TimeReport' : 'Timereport_Parser',
5  'TimeReport @@@ reuse' : 'Timereport_Parser',
6  'SimpleMemReport' : 'SimpleMem_Parser',
7  'EdmSize' : 'Edm_Size',
8  'IgProfperf' : 'IgProf_perf.PERF_TICKS',
9  'IgProfperf @@@ reuse' : 'IgProf_perf.PERF_TICKS',
10  'IgProfMemTotal' : 'IgProf_mem.MEM_TOTAL',
11  'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
12  'IgProfMemLive' : 'IgProf_mem.MEM_LIVE',
13  'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
14  'IgProfMemMax' : 'IgProf_mem.MEM_MAX',
15  'IgProfMemMax @@@ reuse' : 'IgProf_mem.MEM_MAX',
16  'IgProfMemAnalyse' : 'IgProf_mem.ANALYSE',
17  'valgrind' : 'ValgrindFCE',
18  'memcheck_valgrind' : 'Memcheck_Valgrind',
19  'None' : 'None',
20 }

Definition at line 44 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.THIS_PROG_NAME = os.path.basename(sys.argv[0])

Global variables.

Definition at line 32 of file cmsRelvalreportInput.py.