CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
cmsRelvalreportInput Namespace Reference

Functions

def checkSteps
 
def determineNewProfile
 
def expandHyphens
 
def getFstOccur
 
def getLstIndex
 
def getProfileArray
 
def getSteps
 
def init_vars
 
def main
 
def optionparse
 
def pythonFragment
 
def setInputFile
 
def setOutputFileOption
 
def setupProgramParameters
 
def writeCommands
 
def writeCommandsToReport
 
def writePrerequisteSteps
 
def writeReportFileHeader
 
def writeStepHead
 
def writeUnprofiledSteps
 

Variables

list AfterPileUpSteps = []
 
list AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]
 
string cmsDriver = 'cmsDriver.py'
 
 debug = False
 
list DEF_STEPS = ['GEN,SIM', 'DIGI']
 
tuple hypreg = re.compile('-')
 
dictionary Profiler
 
tuple THIS_PROG_NAME = os.path.basename(sys.argv[0])
 Global variables. More...
 

Function Documentation

def cmsRelvalreportInput.checkSteps (   steps)
Checks user steps <steps> for order against steps defined in AllSteps list.
If they are not, prints a warning and exits.
No return value.

Definition at line 75 of file cmsRelvalreportInput.py.

References split.

Referenced by getSteps().

75 
76 def checkSteps(steps):
77  '''Checks user steps <steps> for order against steps defined in AllSteps list.
78  If they are not, prints a warning and exits.
79  No return value.'''
80 
81  #print steps
82  #if ":" in steps:
83  # steps.split(
84  idx = -2
85  lstidx = -2
86  for step in steps:
87  astep = step
88  split = []
89  #Added special workflow for GEN-FASTSIM, bypass the check of step order:
90  if step=="GEN,FASTSIM":
91  pass
92  else:
93  if "-" in step:
94  split = astep.split("-")
95  astep = split[0].split(":")[0]
96  # print "This is astep:%s"%astep
97  #idx =AllSteps.index("GEN,SIM")
98  #print idx
99  #print astep
100  idx = AllSteps.index(astep.split(":")[0])
101  if not ( idx == -2 ):
102  if lstidx > idx:
103  print "ERROR: Your user defined steps are not in a valid order"
104  sys.exit()
105  lstidx = idx
106  if "-" in step:
107  lstidx = AllSteps.index(split[1].split(":")[0])
108 
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.determineNewProfile (   step,
  Profile,
  SavedProfile 
)

Definition at line 428 of file cmsRelvalreportInput.py.

429 def determineNewProfile(step,Profile,SavedProfile):
430  if 'DIGI2RAW' in step:
431  SavedProfile = Profile
432  Profile = [ ]
433  if 'HLT' in step:
434  Profile = SavedProfile
435 
436  return (Profile, SavedProfile)
def cmsRelvalreportInput.expandHyphens (   step)

Definition at line 260 of file cmsRelvalreportInput.py.

References alcazmumu_cfi.filter, python.multivaluedict.map(), and split.

Referenced by writeCommands().

261 def expandHyphens(step):
262  newsteps = []
263 # for step in steps:
264  if "-" in step:
265  hypsteps = step.split(r"-")
266  #if not (len(hypsteps) == 2):
267  # print "ERROR: Could not parse usersteps. You have too many hypens between commas"
268  # sys.exit()
269  #el
270  if not reduce(lambda x,y,: x and y,map(lambda x: x.split(":")[0] in AllSteps, hypsteps)):
271  #(hypsteps[0].split(":")[0] in AllSteps and hypsteps[1].split(":") in AllSteps):
272  print "ERROR: One of the steps you defined is invalid"
273  sys.exit()
274  else:
275  if (hypsteps[0] == hypsteps[1]):
276  print "WARNING: You should not add a hypenated step that as the same source and destination step, ignoring"
277  newsteps.append(hypsteps[0])
278  else:
279  newsteps.append(hypsteps[0])
280 
281  srt = AllSteps.index(hypsteps[0].split(":")[0]) + 1
282  for n in range(srt,AllSteps.index(hypsteps[-1].split(":")[0])):
283  astep = AllSteps[n]
284  if astep in ( hypsteps[i].split(":")[0] for i in range(1,len(hypsteps))):
285  #break
286  astep=filter(lambda x: astep == x.split(":")[0],hypsteps)[0]
287  if astep==hypsteps[-1].split(":")[0]:
288  break
289  newsteps.append(astep)
290  newsteps.append(hypsteps[-1])
291  else:
292  if not (step.split(":")[0] in AllSteps):
293  print "ERROR: One of the steps you defined is invalid"
294  sys.exit()
295  else:
296  newsteps.append(step)
297 
298  return newsteps
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.getFstOccur (   item,
  list 
)

Definition at line 66 of file cmsRelvalreportInput.py.

References alcazmumu_cfi.filter.

66 
67 def getFstOccur(item, list):
68  return filter(item.__eq__,list)[0]
def cmsRelvalreportInput.getLstIndex (   item,
  list 
)

Definition at line 69 of file cmsRelvalreportInput.py.

69 
70 def getLstIndex(item, list):
71  lenlist = len(list)
72  for x in range(lenlist - 1,0,-1):
73  if list[x] == item:
74  return x
def cmsRelvalreportInput.getProfileArray (   ProfileCode)

Definition at line 376 of file cmsRelvalreportInput.py.

Referenced by main().

377 def getProfileArray(ProfileCode):
378 
379  Profile = []
380 
381  # The allowed profiles are:
382 
383  AllowedProfile = [
384  'TimingReport',
385  'TimeReport',
386  'SimpleMemReport',
387  'EdmSize',
388  'IgProfperf',
389  'IgProfMemTotal',
390  'IgProfMemLive',
391  #'IgProfMemMax', #Should add this here... but instead of changing the profile code numbers (really should, using options instead of numbers...) I am hacking the IgProfMemLive one: when this one is selected I will also include the IgProfMemMax profile (and analysis is analyse is there)
392  'IgProfMemAnalyse',
393  'valgrind',
394  'memcheck_valgrind',
395  'None',
396  ]
397 
398  if _noprof:
399  Profile.append(AllowedProfile[-1])
400  else:
401  #FIXME:
402  #Horrible code!
403  for i in range(10):
404  if str(i) in ProfileCode:
405  firstCase = (i == 0 and (str(1) in ProfileCode or str(2) in ProfileCode)) \
406  or (i == 1 and str(2) in ProfileCode)
407  secCase = (i==4 and str(7) in ProfileCode) \
408  or (i == 5 and (str(6) in ProfileCode or str(7) in ProfileCode)) \
409  or (i == 6 and str(7) in ProfileCode)
410 
411  if firstCase or secCase:
412  Profile.append(AllowedProfile[i] + ' @@@ reuse')
413  #Here's the hack:
414  if i==6: #i.e. IgProfMemLive
415  Profile.append("IgProfMemMax @@@ reuse")
416  else:
417  Profile.append(AllowedProfile[i])
418  if i==6: #i.e. IgProfMemLive
419  Profile.append("IgProfMemMax")
420 
421  return Profile
def cmsRelvalreportInput.getSteps (   userSteps)
Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
Returns a list of steps <steps>.

Definition at line 109 of file cmsRelvalreportInput.py.

References checkSteps().

Referenced by setupProgramParameters().

110 def getSteps(userSteps):
111  '''Analyzes the steps in <userSteps> to translate the cmsRelvalreportInput.py --step option into the cmsDriver.py one. Expanding hyphens for combined steps.
112  Handle the exceptional cases of GEN-, GEN-SIM and GEN-FASTSIM (GEN is never profiled by itself).
113  Returns a list of steps <steps>.'''
114  #Add handling of ":" in steps
115  #print userSteps
116  # Then split the user steps into "steps"
117  gsreg = re.compile('GEN-SIM')
118  greg = re.compile('GEN') #Add a second hack (due to the first) to handle the step 1 case GEN-HLT
119  gfsreg = re.compile('GEN-FASTSIM') #Add a third hack to handle the case GEN-FASTSIM workflow
120  StepsTokens = userSteps.split(",")
121  steps = []
122  for astep in StepsTokens:
123 
124  # Then transform the combined steps (GEN-SIM, RAW2DIGI-RECO)
125  # from using the "-" to using the "," to match cmsDriver.py convention
126 
127  if gsreg.search(astep):
128  astep = gsreg.sub(r"GEN,SIM", astep)
129  elif gfsreg.search(astep):
130  astep = gfsreg.sub(r"GEN,FASTSIM", astep) #could add protection for user screw-ups
131  elif greg.search(astep):
132  astep = greg.sub(r"GEN,SIM", astep)
133 
134 
135  #print astep
136  # Finally collect all the steps into the steps list:
137 
138  steps.append(astep)
139 
140  #steps = expandHypens(steps)
141  checkSteps(steps)
142  return steps
def cmsRelvalreportInput.init_vars ( )

Definition at line 340 of file cmsRelvalreportInput.py.

Referenced by main().

341 def init_vars():
342 
343  ####################
344  # Obtain required environment variables
345  #
346 
347  try:
348  CMSSW_BASE = os.environ['CMSSW_BASE']
349  CMSSW_RELEASE_BASE = os.environ['CMSSW_RELEASE_BASE']
350  CMSSW_VERSION = os.environ['CMSSW_VERSION']
351  except KeyError:
352  print 'Error: An environment variable either CMSSW_{BASE, RELEASE_BASE or VERSION} is not available.'
353  print ' Please run eval `scramv1 runtime -csh` to set your environment variables'
354  sys.exit()
355 
356  return ( CMSSW_BASE,
357  CMSSW_RELEASE_BASE,
358  CMSSW_VERSION)
def cmsRelvalreportInput.main (   argv = sys.argv)

Definition at line 946 of file cmsRelvalreportInput.py.

References getProfileArray(), init_vars(), optionparse(), setupProgramParameters(), writeCommandsToReport(), and writeReportFileHeader().

947 def main(argv=sys.argv):
948 
949  #####################
950  # Options Parser
951  #
952 
953  (options, args) = optionparse()
954 
955  #####################
956  # Set up arguments and option handling
957  #
958 
959  (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, bypasshlt, userInputFile ) = setupProgramParameters(options,args)
960  #print setupProgramParameters(options,args)
961 
962  ######################
963  # Initialize a few variables
964  #
965 
966  (CMSSW_BASE, CMSSW_RELEASE_BASE, CMSSW_VERSION ) = init_vars()
967 
968  ##################
969  # Ok everything is ready now we need to create the input file for the relvalreport script
970  #
971 
972  simcandles = writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE)
973 
974  ##################
975  # Based on the profile code create the array of profiles to run:
976  #
977 
978  Profile = getProfileArray(ProfileCode)
979 
980 
981  ##################
982  # Write the commands for the report to the file
983  #
984 
985  writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile)
986 
987  print "Written out cmsRelvalreport.py input file at:\n%s"%os.path.abspath('./'+simcandles.name)
988 
989  simcandles.close()
def cmsRelvalreportInput.optionparse ( )

Definition at line 143 of file cmsRelvalreportInput.py.

References python.multivaluedict.map().

Referenced by main().

144 def optionparse():
145  global _noprof
146  explanations = map(lambda x: " " + x, Candles)
147  explanation = ""
148  for x in range(len(explanations)):
149  explanation += "%-*s %s\n" % (30, explanations[x],CandDesc[x])
150  parser = opt.OptionParser(usage=("""%s NUM_EVENTS_PER_CFG CANDLES PROFILE [--cmsdriver=cmsDriverOptions] [--usersteps=processingStepsOption]
151 
152  Description - This program creates a configuration file for cmsRelvalreport.py that describes the order in which cmsDriver.py should be run with which candles, steps and profiling so that time spent running cmsDriver.py is minimised. Without this correct ordering we would have to re-run parts of the profiling steps.
153 
154  Arguments:
155  NUM_EVENTS_PER_CFG - The number of events per config file
156 
157  CANDLES - The simulation type to perform profiling on
158  Candles Description
159  AllCandles Run all of the candles below
160 %s
161 
162  PROFILE - the type of profiling to perform (multiple codes can be used):
163  Code Profile Type
164  0 TimingReport
165  1 TimeReport
166  2 SimpleMemoryCheck
167  3 EdmSize
168  4 IgProfPerf
169  5 IgProfMemTotal
170  6 IgProfMemLive
171  7 IgProfAnalyse
172  8 ValgrindFCE
173  9 ValgrindMemCheck
174 
175  Examples (could be obsolete!):
176  Perform Timing Report profiling for all candles, all steps and 10 events per cfg.
177  ./%s 10 AllCandles 1 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
178  Perform Timing Report, Time Report and SimpleMemoryCheck profiling for HiggsZZ4LM200, all steps and 50 events.
179  ./%s 50 \"HiggsZZ4LM200\" 012 \"--conditions FakeConditions --eventcontent FEVTDEBUGHLT\"
180  Perform IgProfPerf and IgProfMemTotal profiling for TTbar, all steps and 100 events.
181  ./%s 100 \"TTbar\" 45 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWRECO\"
182  Perform ValgrindFCE ValgrindMemCheck profiling for Minimum bias and 100 events. Only on GEN,SIM and DIGI steps.
183  ./%s 100 \"MinBias\" 89 --cmsdriver=\"--conditions FakeConditions --eventcontent RAWSIM\" \"--usersteps=GEN-SIM,DIGI"""
184  % ( THIS_PROG_NAME, explanation, THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME,THIS_PROG_NAME)))
185 
186  devel = opt.OptionGroup(parser, "Developer Options",
187  "Caution: use these options at your own risk."
188  "It is believed that some of them bite.\n")
189  #parser.set_defaults(debug=False)
190 
191  parser.add_option(
192  '-b',
193  '--bypass-hlt',
194  action="store_true",
195  dest='bypasshlt',
196  default=False,
197  help='Should we bypass using the HLT root file?'
198  )
199 
200  parser.add_option(
201  '-u',
202  '--usersteps',
203  type='string',
204  dest='userSteps',
205  help='Which steps to run',
206  metavar='<STEPS>',
207  )
208  parser.add_option(
209  '-c',
210  '--cmsdriver',
211  type='string',
212  dest='cmsDriverOptions',
213  help='Option for cmsDriver.py can be specified as a string to be added to all cmsDriver.py commands',
214  metavar='<CONDITION>',
215  )
216  parser.add_option(
217  '-i',
218  '--filein',
219  type='string',
220  dest='userInputFile',
221  help='Eventual input file (for now to be only used with HLT and RAW2DIGI-RECO workflows)',
222  default='',
223  metavar='<INPUTFILE>',
224  )
225  devel.add_option(
226  '-1',
227  '--no-profile',
228  action="store_true",
229  dest='noprof',
230  help='Do not perform profiling, ever',
231  #metavar='DEBUG',
232  )
233 
234  devel.add_option(
235  '-d',
236  '--debug',
237  action="store_true",
238  dest='debug',
239  help='Show debug output',
240  #metavar='DEBUG',
241  )
242 
243  parser.set_defaults(debug=False,noprof=False)
244  parser.add_option_group(devel)
245 
246  (options, args) = parser.parse_args()
247 
248 
249  _noprof = options.noprof
250  debug = options.debug
251  numofargs = len(args)
252 # print "runtime cmsinput " + str(numofargs) + " noprof " + str(_noprof)
253  if ((not numofargs == 3) and (not _noprof)) or (_noprof and not (numofargs == 2)):
254  parser.error("There are not enough arguments specified to run this program."
255  " Please determine the correct arguments from the usage information above."
256  " Run with --help option for more information.")
257  sys.exit()
258 
259  return (options, args)
def cmsRelvalreportInput.pythonFragment (   step,
  cmsdriverOptions 
)

Definition at line 437 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writePrerequisteSteps().

438 def pythonFragment(step,cmsdriverOptions):
439  # Convenient CustomiseFragment dictionary to map the correct customise Python fragment for cmsDriver.py:
440  #It is now living in cmsPerfCommons.py!
441 
442  if "--pileup" in cmsdriverOptions and not (step == "HLT" or step.startswith("RAW2DIGI")):
443  return CustomiseFragment['DIGI-PILEUP']
444  elif CustomiseFragment.has_key(step):
445  return CustomiseFragment[step]
446  else:
447  #This is a safe default in any case,
448  #the maintenance of the customise python fragments goes into cmsPerfCommons.py
449  return CustomiseFragment['DIGI']
450 
def cmsRelvalreportInput.setInputFile (   steps,
  step,
  acandle,
  stepIndex,
  pileup = False,
  bypasshlt = False 
)

Definition at line 451 of file cmsRelvalreportInput.py.

Referenced by writeCommands(), and writeUnprofiledSteps().

452 def setInputFile(steps,step,acandle,stepIndex,pileup=False,bypasshlt=False):
453  #print "StepIndex 2:%s"%stepIndex
454  InputFileOption = ""
455  if pileup and stepIndex == 0:
456  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],"DIGI" )
457  else:
458  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 1] )
459 
460  if pileup:
461  pass
462  else :
463  if 'GEN,SIM' in step: # there is no input file for GEN,SIM!
464  InputFileOption = ''
465  if 'GEN,FASTSIM' in step: # there is no input file for GEN,FASTSIM!
466  InputFileOption = ''
467  elif 'HLT' in steps[stepIndex - 1] and bypasshlt:
468 
469  # Special hand skipping of HLT since it is not stable enough, so it will not prevent
470  # RAW2DIGI,RECO from running
471 
472  InputFileOption = "--filein file:%s_%s" % ( FileName[acandle],steps[stepIndex - 2] )
473 
474  if not InputFileOption == "" :
475  InputFileOption += ".root "
476 
477  return InputFileOption
def cmsRelvalreportInput.setOutputFileOption (   acandle,
  endstep 
)

Definition at line 538 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by writeCommands().

539 def setOutputFileOption(acandle,endstep):
540  return "%s_%s.root" % ( FileName[acandle],endstep)
def cmsRelvalreportInput.setupProgramParameters (   options,
  args 
)

Definition at line 299 of file cmsRelvalreportInput.py.

References getSteps().

Referenced by main().

300 def setupProgramParameters(options,args):
301  steps = []
302  cmsDriverOptions = ""
303  global AfterPileUpSteps
304  NumberOfEvents = int(args[0]) # first arg
305  WhichCandles = str(args[1]) # second arg
306  ProfileCode = ""
307  if not _noprof:
308  ProfileCode = str(args[2]) # third arg
309 
310  if options.cmsDriverOptions:
311 
312  cmsDriverOptions = options.cmsDriverOptions
313  print 'Using user-specified cmsDriver.py options: ' + cmsDriverOptions
314 
315  if options.userSteps:
316 
317  userSteps = options.userSteps
318  steps = getSteps(userSteps)
319 
320  if WhichCandles.lower() == 'allcandles':
321  Candle = Candles
322  print 'ALL standard simulation candles will be PROCESSED:'
323  else:
324  Candle = [WhichCandles]
325  print 'Candle %s will be PROCESSED' % Candle[0]
326 
327  # For now the two steps are built in, this can be added as an argument later
328  # Added the argument option so now this will only be defined if it was not defined already:
329 
330  if not steps:
331  print 'The default steps will be run:'
332  steps = DEF_STEPS
333  else:
334  print "You defined your own steps to run:"
335 
336  for astep in steps:
337  print astep
338 
339  return (NumberOfEvents, ProfileCode, cmsDriverOptions, steps, Candle, options.bypasshlt,options.userInputFile)
def cmsRelvalreportInput.writeCommands (   simcandles,
  Profile,
  acandle,
  steps,
  NumberOfEvents,
  cmsDriverOptions,
  bypasshlt,
  stepIndex = 0,
  pileup = False,
  userInputFile = "" 
)

Definition at line 550 of file cmsRelvalreportInput.py.

References expandHyphens(), alcazmumu_cfi.filter, join(), python.multivaluedict.map(), pythonFragment(), setInputFile(), setOutputFileOption(), split, writePrerequisteSteps(), writeStepHead(), and writeUnprofiledSteps().

Referenced by setOutputFileOption(), and writeCommandsToReport().

551  userInputFile=""):
552  #print "a:%s,b:%s,c:%s"%(stepIndex,pileup,userInputFile)
553  OutputStep = ""
554 
555  stopIndex = len(steps)
556  start = 0
557 
558  userSteps = steps
559  SavedProfile = []
560  fstROOTfile = True
561  fstROOTfileStr = ""
562 
563  #Handling the case of the first user step not being the first step (GEN,SIM):
564  #print "Steps passed to writeCommands %s",steps
565  #Special case for FASTSIM:
566  if (steps[0] == "GEN,FASTSIM"):
567  start=-1
568  stopIndex=1
569  steps = ["GEN,FASTSIM"]
570  else:
571  if not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] != "GEN,SIM"):
572  #print "userInputFile: %s"%userInputFile
573  if ("--pileup" in cmsDriverOptions) and (steps[0]=="HLT" or steps[0].startswith("RAW2DIGI")) :
574  userInputFile = "../INPUT_PILEUP_EVENTS.root"
575  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
576  rootFileStr=""
577 
578  elif userInputFile == "":
579  #Write the necessary line to run without profiling all the steps before the wanted ones in one shot:
580  (stepIndex, rootFileStr) = writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt)
581  else: #To be implemented if we want to have input file capability beyond the HLT and RAW2DIGI-RECO workflows
582  #if "-" in steps[0]:
583  stepIndex=AllSteps.index(steps[0].split("-")[0].split(":")[0])
584  #print "Here!"
585  #else:
586  #print "There!"
587  # stepIndex=AllSteps.index(steps[0])
588  rootFileStr=""
589  #Now take care of setting the indeces and input root file name right for the profiling part...
590  if fstROOTfile:
591  fstROOTfileStr = rootFileStr
592  fstROOTfile = False
593  start = -1
594  #if "-" in steps[0]:
595  start = AllSteps.index(steps[0].split("-")[0].split(":")[0])
596  #else:
597  # start = AllSteps.index(steps[0])
598  lst = - 1
599  if "-" in steps[-1]:
600  lst = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) #here we are assuming that - is always between two steps no GEN-SIM-DIGI, this is GEN-DIGI
601  else:
602  lst = AllSteps.index(steps[-1])
603  runSteps = AllSteps[start:lst]
604  numOfSteps = (lst - start) + 1
605  stopIndex = start + numOfSteps
606  #Handling the case in which the first user step is the same as the first step (GEN,SIM)
607  #elif not (steps[0] == AllSteps[0]) and (steps[0].split("-")[0] == "GEN"):
608  else:
609  #Handling the case of the last step being a composite one:
610  if "-" in steps[-1]:
611  #Set the stop index at the last step of the composite step (WHY???)
612  stopIndex = AllSteps.index(steps[-1].split("-")[1].split(":")[0]) + 1
613  else:
614 
615  stopIndex = AllSteps.index(steps[-1].split(":")[0]) + 1
616  #print "Last step is %s and stopIndex is %s"%(steps[-1],stopIndex)
617  steps = AllSteps
618 
619  unprofiledSteps = []
620  rawreg = re.compile("^RAW2DIGI")
621 
622 #Horrible structure... to be rewritten sooner or later...
623 
624 # FOR step in steps
625 
626  prevPrevOutputFile = ""
627  #Add if to introduce the possibility to run on user selected input file.
628  if userInputFile!="":
629  previousOutputFile=userInputFile
630  else:
631  previousOutputFile = ""
632 
633  for x in range(start,stopIndex,1):
634  if stepIndex >= stopIndex:
635  break
636  step = steps[stepIndex]
637 
638  # One shot Profiling variables
639  befStep = step
640  aftStep = step
641  # We need this in case we are running one-shot profiling or for DIGI-PILEUP
642  stepToWrite = step
643 
644  CustomisePythonFragment = pythonFragment(step,cmsDriverOptions)
645  oneShotProf = False
646  hypsteps = []
647 
648  #Looking for step in userSteps, or for composite steps that step matches the first of a composite step in userSteps
649  if step in userSteps or reduce(lambda x,y : x or y,map(lambda x: step == x.split("-")[0].split(":")[0],userSteps)):
650 
651  if "GEN,FASTSIM" in userSteps: # HLT/L1 menu workaround
652  stepToWrite = stepToWrite + ",HLT:GRun"
653  elif "HLT" in userSteps:
654  stepToWrite = stepToWrite.replace("HLT", "HLT:GRun")
655 
656  #Checking now if the current step matches the first of a composite step in userSteps
657  hypMatch = filter(lambda x: "-" in x,filter(lambda x: step == x.split("-")[0],userSteps))
658  if not len(hypMatch) == 0 :
659  hypsteps = expandHyphens(hypMatch[0])
660  #print hypsteps
661  #print hypMatch[0]
662  stepToWrite = ",".join(hypsteps)
663  if "GEN,SIM-HLT" in userSteps: # HLT/L1 menu workaround
664  stepToWrite = stepToWrite.replace("HLT","HLT:GRun")
665  befStep = hypsteps[0]
666  #Kludge to avoid running HLT in composite steps if the -b option is chosen
667  if bypasshlt and hypsteps[-1]=='HLT':
668  aftStep = hypsteps[-2]
669  else:
670  aftStep = hypsteps[-1]
671  oneShotProf = True
672  if filter(lambda x: stepToWrite==x.split(":")[0],userSteps):
673  #print "Inside the filter if!"
674  #print stepToWrite
675  #Here we are assuming that the steps specified with : are not duplicate in the userSteps command:
676  #i.e. GEN-DIGI:SequenceA,HLT:1E31 is OK
677  #GEN-DIGI:SequenceA,HLT:SequenceB,HLT:1E31 IS NOT (OF COURSE!)
678  stepToWrite=filter(lambda x: stepToWrite==x.split(":")[0],userSteps)[0]
679  #print stepToWrite
680 
681  writeStepHead(simcandles,acandle,stepToWrite)
682 
683  #Set the output file name for Pile up and for regular case:
684  if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change to make root file output work (without colons)
685  stepToWrite = stepToWrite.replace(":","=")
686 
687  if '--pileup' in cmsDriverOptions:
688  outfile = stepToWrite + "_PILEUP"
689  else:
690  outfile = stepToWrite
691 
692  #print stepToWrite
693 
694  if "GEN,SIM-HLT" or "GEN,FASTSIM" or "HLT" in userSteps: # change it back
695  stepToWrite = stepToWrite.replace("=",":")
696 
697  OutputFile = setOutputFileOption(acandle,outfile)
698  if fstROOTfile:
699  fstROOTfileStr = OutputFile
700  fstROOTfile = False
701  OutputFileOption = "--fileout=" + OutputFile
702 
703  for prof in Profile:
704  #First prepare the cmsDriver.py command
705 
706  #Special case of EventEdmSize profiling
707  if 'EdmSize' in prof:
708  EdmFile = "%s_%s.root" % (FileName[acandle],outfile) #stepToWrite) #Bug in the filename for EdmSize for PileUp corrected.
709  #EventEdmSize needs a pre-requisite step that just produces the root file if one decided to run with only EdmSize profiling!
710  if prof == Profile[0] and not os.path.exists("./" + EdmFile):
711  # insert command to generate required state ( need to run one more step
712  # so that EDM can actually check the size of the root file
713  # If the first step to be profiled is something later on in the steps such
714  # as HLT then writePrerequisteSteps() should have got you to the step prior to
715  # HLT, therefore the only thing left to run to profile EDMSIZE is HLT itself
716 
717  InputFileOption = "--filein file:" + previousOutputFile
718  #Kludge to bypass HLT output... here's a problem... since the unprofiled steps could contain HLT!
719  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
720  InputFileOption = "--filein file:" + prevPrevOutputFile
721  if previousOutputFile == "":
722  InputFileOption = setInputFile(steps,stepToWrite,acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
723  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
724  % (cmsDriver,
725  KeywordToCfi[acandle],
726  NumberOfEvents,
727  stepToWrite,
728  InputFileOption,
729  OutputFileOption,
730  CustomisePythonFragment,
731  cmsDriverOptions) )
732  simcandles.write( "%s @@@ None @@@ None\n" % (Command))
733 
734  Command = EdmFile
735  #all other profiles:
736  else:
737  InputFileOption = "--filein file:" + previousOutputFile
738  if rawreg.search(step) and bypasshlt and 'DIGI2RAW.root' in prevPrevOutputFile:
739  InputFileOption = "--filein file:" + prevPrevOutputFile
740 
741  if previousOutputFile == "":
742  InputFileOption = setInputFile(steps,befStep,acandle,stepIndex,pileup,bypasshlt)
743 
744  #if '--pileup' in cmsDriverOptions:
745  # stepToWrite = pileupStep
746  if '--pileup' in cmsDriverOptions and ( stepToWrite=='GEN,SIM' or stepToWrite=='SIM'):
747  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
748  cmsDriver,
749  KeywordToCfi[acandle],
750  NumberOfEvents,
751  stepToWrite,
752  InputFileOption,
753  OutputFileOption,
754  CustomiseFragment['GEN,SIM'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
755  cmsDriverOptions[:cmsDriverOptions.index('--pileup')]
756  ))
757  elif '--pileup' in cmsDriverOptions and (stepToWrite=='GEN,FASTSIM' or stepToWrite=='GEN,FASTSIM,HLT:GRun'): #Adding GEN,FASTIM case exception and use the CustomiseFragment['DIGI']
758  #+exception to the HLT:GRun case
759  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
760  cmsDriver,
761  KeywordToCfi[acandle],
762  NumberOfEvents,
763  stepToWrite,
764  InputFileOption,
765  OutputFileOption,
766  CustomiseFragment['DIGI'],#Done by hand to avoid silly use of MixinModule.py for pre-digi individual steps
767  cmsDriverOptions #For FASTSIM PU need the whole cmsDriverOptions! [:cmsDriverOptions.index('--pileup')]
768  ))
769 
770  else:
771  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s" % (
772  cmsDriver,
773  KeywordToCfi[acandle],
774  NumberOfEvents,
775  stepToWrite,
776  InputFileOption,
777  OutputFileOption,
778  CustomisePythonFragment,
779  cmsDriverOptions
780  ))
781  #Now the cmsDriver command is ready in Command, we just edit the rest of the line and write it to the file!
782  if _noprof:
783  simcandles.write("%s @@@ None @@@ None\n" % Command)
784  else:
785  if 'valgrind' in prof:
786  #I think the if is safer even though all commands in this "else" (i.e. non EdmSize) should be OK.
787  if 'TimeMemoryInfo.py' in Command:
788  Command=Command.replace('--customise=Validation/Performance/TimeMemoryInfo.py', '')
789  #There is actually also the SIM case to take care of:
790  elif 'TimeMemoryG4Info.py' in Command:
791  Command=Command.replace('--customise=Validation/Performance/TimeMemoryG4Info.py', '')
792  stepLabel=stepToWrite
793  #print "stepLabel is %s"%stepLabel
794  if '--pileup' in cmsDriverOptions and not "_PILEUP" in stepToWrite:
795  stepLabel = stepToWrite+"_PILEUP"
796  #Add kludge here to make the IGPROF ANALYSE work:
797  #The idea is to:
798  #1-Check if ANALYSE is there everytime we're handling IGPROF profiles
799  #2-If it is there then write 2 lines instead of 1: add a @@@ reuse to the first line and a second line with ANALYSE @@@ the same metadata
800  #[3-Catch the case of ANALYSE by itself and raise an exception]
801  #4-Catch the case of ANALYSE proper, and do nothing in that case (already taken care by the second lines done after PERF_TICKS, MEM_TOTAL and MEM_LIVE.
802  #print "EEEEEEE %s"%prof
803  #We actually want the analyse to be used also for IgProf perf!
804  if 'IgProf' in prof: # and 'perf' not in prof:
805  #New IgProf naming scheme:
806  #Add a new naming scheme to simplify the handling in igprof-navigator:
807  #Candle___Step___Conditions__PileUpOption__IgProfProfile___Event.gz (and .sql3)
808 
809  #Creating the IgProfProfile variable, to be either IgProfMem or IgProfPerf, the two types of profiling:
810  IgProfProfile="IgProf"
811  if "Mem" in prof:
812  if "reuse" in prof:
813  IgProfProfile="IgProfMem @@@ reuse"
814  else:
815  IgProfProfile="IgProfMem"
816  else:
817  if "reuse" in prof:
818  IgProfProfile="IgProfPerf @@@ reuse"
819  else:
820  IgProfProfile="IgProfPerf"
821 
822  #Conditions and PileUpOption are extracted from the cmsDriverOptions:
823  Conditions="UNKNOWN_CONDITIONS"
824  PileUp="NOPILEUP"
825  EventContent="UNKNOWN_EVENTCONTENT"
826  tokens=cmsDriverOptions.split("--")
827  for token in tokens:
828  keywords=token.split(" ")
829  if "=" in keywords[0]:
830  keywords=keywords[0].split("=")
831  if "conditions" in keywords[0]:
832  # check if we are using the autoCond style of flexible conditions
833  # if so, expand the condition here so that the file names contain the real conditions
834  if "auto:" in keywords[1]:
835  from Configuration.AlCa.autoCond import autoCond
836  fileConditionsOption = autoCond[ keywords[1].split(':')[1] ]
837  Conditions = autoCond[ keywords[1].split(':')[1] ].split("::")[0]
838  else:
839  # old style, hardcoded, conditions ...
840  # Complicated expression, just to get rid of FrontierConditions_GlobalTag,
841  # and ::All at the end:
842  # Added if to handle new --conditions convention for cmsDriver.py (no more
843  # FrontierConditions... in front of it)
844  if "," in keywords[1]:
845  Conditions=keywords[1].split(",")[1].split("::")[0] #"backward" compatibility...
846  else:
847  Conditions=keywords[1].split("::")[0]
848  elif "pileup" in keywords[0]:
849  PileUp=keywords[1]
850  elif "eventcontent" in keywords[0]:
851  EventContent=keywords[1]
852 
853  #so here's our new MetaName:
854  MetaName=acandle+"___"+stepToWrite+"___"+PileUp+"___"+Conditions+"___"+EventContent+"___"+IgProfProfile
855 
856  if 'Analyse' not in prof and (lambda x: 'Analyse' in x,Profile):
857 
858  simcandles.write("%s @@@ %s @@@ %s\n" % (Command,
859  Profiler[prof],
860  MetaName))
861  #FileName[acandle],
862  #stepLabel,
863  #prof))
864  analyse_command = ("%s @@@ %s @@@ %s\n" % (Command,
865  Profiler[prof].split(".")[0]+'.ANALYSE.'+Profiler[prof].split(".")[1], #Add the IgProf counter in here (e.g. IgProfMem.ANALYSE.MEM_TOTAL)
866  MetaName))
867  #FileName[acandle],
868  #stepLabel,
869  #prof))
870 
871  #for the relval case, GEN-SIM,HLT get rid of the last " @@@ reuse":
872  if Profiler[prof] == 'IgProf_mem.MEM_MAX' or Profiler[prof] == 'IgProf_perf.PERF_TICKS':
873  analyse_command = analyse_command.replace("@@@ reuse", "")
874 
875  simcandles.write(analyse_command)
876 
877  elif 'Analyse' in prof:
878  pass
879  else:
880  simcandles.write("%s @@@ %s @@@ %s_%s_%s\n" % (Command,
881  Profiler[prof],
882  FileName[acandle],
883  stepLabel,
884  prof))
885 
886  if debug:
887  print InputFileOption, step, 'GEN,SIM' in step, 'HLT' in steps[stepIndex - 1], steps
888  print "cmsDriveroptions : " + cmsDriverOption
889  prevPrevOutputFile = previousOutputFile
890  previousOutputFile = OutputFile
891  else:
892  unprofiledSteps.append(step)
893  isNextStepForProfile = False # Just an initialization for scoping. don't worry about it being false
894 
895  try:
896  isNextStepForProfile = steps[stepIndex + 1] in userSteps or reduce(lambda x,y : x or y,map(lambda z: steps[ stepIndex + 1 ] == z.split("-")[0].split(":")[0],userSteps))
897  #print "YES found Step %s when looking with standard step %s"%(steps[stepIndex],steps[stepIndex])
898  except IndexError:
899  # This loop should have terminated if x + 1 is out of bounds!
900  print "Error: Something is wrong we shouldn't have come this far"
901  break
902 
903  if isNextStepForProfile:
904  #Minimum intervention solution is to fix it here: I think we need to pass the before and after steps...
905  OutputFile=writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents,stepIndex,pileup,bypasshlt)
906  unprofiledSteps = []
907  prevPrevOutputFile = previousOutputFile
908  previousOutputFile = OutputFile
909  #Dangerous index handling when looping over index x!
910  if oneShotProf:
911  stepIndex += len(hypsteps)
912  else:
913  stepIndex +=1
914  return fstROOTfileStr
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeCommandsToReport (   simcandles,
  Candle,
  Profile,
  debug,
  NumberOfEvents,
  cmsDriverOptions,
  steps,
  bypasshlt,
  userInputFile 
)

Definition at line 915 of file cmsRelvalreportInput.py.

References writeCommands().

Referenced by main().

916 def writeCommandsToReport(simcandles,Candle,Profile,debug,NumberOfEvents,cmsDriverOptions,steps,bypasshlt,userInputFile):
917 
918  OutputStep = ''
919 
920  # This option will not be used for now since the event content names keep changing, it can be
921  # edited by hand pre-release by pre-release if wanted (Still moved all to FEVTDEBUGHLT for now, except RECO, left alone).
922  #
923  #EventContent = {'GEN,SIM': 'FEVTDEBUGHLT', 'DIGI': 'FEVTDEBUGHLT'}
924 
925  for acandle in Candle:
926  print '*Candle ' + acandle
927 
928  ##################
929  # If the first profiling we run is EdmSize we need to create the root file first
930  #
931 
932  #Here all candles are processed with all the same command, and in the pileup case they will have the pileup settings set correctly already:
933  #print "1-userInputFile: %s"%userInputFile
934 
935  fstoutfile = writeCommands(simcandles,
936  Profile,
937  acandle,
938  steps,
939  NumberOfEvents,
940  cmsDriverOptions,
941  bypasshlt,
942  0,
943  False,
944  userInputFile)
945 
def cmsRelvalreportInput.writePrerequisteSteps (   simcandles,
  steps,
  acandle,
  NumberOfEvents,
  cmsDriverOptions,
  pileup,
  bypasshlt 
)

Definition at line 525 of file cmsRelvalreportInput.py.

References pythonFragment(), split, and writeUnprofiledSteps().

Referenced by writeCommands().

526 def writePrerequisteSteps(simcandles,steps,acandle,NumberOfEvents,cmsDriverOptions,pileup,bypasshlt):
527  fstIdx = -1
528  #Hack for GEN-FASTSIM workflow:
529  #if steps=="GEN,FASTSIM":
530  #if "-" in steps[0]:
531  fstIdx = AllSteps.index(steps[0].split("-")[0].split(":")[0])
532  #else:
533  # fstIdx = AllSteps.index(steps[0])
534  CustomisePythonFragment = pythonFragment("GEN,SIM",cmsDriverOptions)
535  previousOutputFile=""
536  OutputFile = writeUnprofiledSteps(simcandles, CustomisePythonFragment, cmsDriverOptions,AllSteps[0:fstIdx],previousOutputFile,acandle,NumberOfEvents, 0,pileup,bypasshlt)
537  return (fstIdx, OutputFile)
double split
Definition: MVATrainer.cc:139
def cmsRelvalreportInput.writeReportFileHeader (   CMSSW_VERSION,
  CMSSW_RELEASE_BASE,
  CMSSW_BASE 
)

Definition at line 359 of file cmsRelvalreportInput.py.

Referenced by main().

360 def writeReportFileHeader(CMSSW_VERSION,CMSSW_RELEASE_BASE,CMSSW_BASE):
361 
362  SimCandlesFile = 'SimulationCandles_%s.txt' % CMSSW_VERSION
363 
364  try:
365  simcandles = open(SimCandlesFile, 'w')
366  except IOError:
367  print "Couldn't open %s to save" % SimCandlesFile
368 
369  simcandles.write('#Candles file automatically generated by %s for %s\n'
370  % (THIS_PROG_NAME, CMSSW_VERSION))
371  simcandles.write("#CMSSW Base : %s\n" % CMSSW_BASE)
372  simcandles.write("#Release Base: %s\n" % CMSSW_RELEASE_BASE)
373  simcandles.write("#Version : %s\n\n" % CMSSW_VERSION)
374 
375  return simcandles
def cmsRelvalreportInput.writeStepHead (   simcandles,
  acandle,
  step 
)

Definition at line 422 of file cmsRelvalreportInput.py.

Referenced by writeCommands().

423 def writeStepHead(simcandles,acandle,step):
424  simcandles.write('#%s\n' % FileName[acandle])
425  simcandles.write('#Step %s\n' % step)
426  #print step
427 
def cmsRelvalreportInput.writeUnprofiledSteps (   simcandles,
  CustomisePythonFragment,
  cmsDriverOptions,
  unprofiledSteps,
  previousOutputFile,
  acandle,
  NumberOfEvents,
  stepIndex,
  pileup,
  bypasshlt 
)

Definition at line 478 of file cmsRelvalreportInput.py.

References join(), and setInputFile().

Referenced by writeCommands(), and writePrerequisteSteps().

479 def writeUnprofiledSteps(simcandles,CustomisePythonFragment,cmsDriverOptions,unprofiledSteps,previousOutputFile,acandle,NumberOfEvents, stepIndex, pileup,bypasshlt):
480  # reduce(lambda x,y : x + "," + "y",unprofiledSteps)
481  #stepsStr = ",".join(unprofiledSteps)
482 
483 
484  #print "unprofiledSteps is %s"%unprofiledSteps
485  #print "22acandle is %s"%acandle
486  #Kludge in case -b option to skip HLT is used...
487  if bypasshlt and unprofiledSteps[-1]=="HLT":
488  stepsStr = ",".join(unprofiledSteps[:-1])
489  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-2])
490  else:
491  stepsStr = ",".join(unprofiledSteps)
492  OutputFile = "%s_%s.root" % ( FileName[acandle],unprofiledSteps[-1])
493  simcandles.write("\n#Run a %s step(s) that has not been selected for profiling but is needed to run the next step to be profiled\n" % (stepsStr))
494  OutputFileOption = "--fileout=%s" % OutputFile
495  #Bug here: should take into account the flag --bypass-hlt instead of assuming hlt should be bypassed
496  #This affects the Step1/Step2 running since Step1 will produce an HLT.root file and Step2 should start from there!
497  #Adding the argument bypasshlt to the calls...
498  #PreviousInputFile=AllSteps[AllSteps.index(unprofiledSteps[0])-1]
499  #print "StepIndex 1:%s"%stepIndex
500  #Correcting a bug: when unprofiled intermediate steps are present it would skip 1 step...
501  #stepIndexAdjust=stepIndex - 2
502  #if stepIndexAdjust < 0: #To avoid issues with negative indeces
503  # stepIndexAdjust=0
504  #InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndexAdjust,bypasshlt=bypasshlt)
505  #Use an argument to make it easier:
506  InputFileOption = "--filein file:" + previousOutputFile
507  if previousOutputFile =="":
508  InputFileOption = setInputFile(AllSteps,unprofiledSteps[0],acandle,stepIndex,pileup=pileup,bypasshlt=bypasshlt)
509  #Introduce an over-ride of cmsDriverOptions:
510  #For the case of unprofiled steps, always run them with FEVTDEBUGHLT eventcontent
511  #At the moment the only use case is when running step2 on its own...
512  for eventcontent in EventContents:
513  cmsDriverOptions=re.sub(eventcontent,'FEVTDEBUGHLT',cmsDriverOptions)
514  Command = ("%s %s -n %s --step=%s %s %s --customise=%s %s"
515  % (cmsDriver,
516  KeywordToCfi[acandle],
517  NumberOfEvents,
518  stepsStr,
519  InputFileOption,
520  OutputFileOption,
521  CustomisePythonFragment,
522  cmsDriverOptions) )
523  simcandles.write( "%s @@@ None @@@ None\n\n" % (Command))
524  return OutputFile
static std::string join(char **cmd)
Definition: RemoteFile.cc:18

Variable Documentation

list cmsRelvalreportInput.AfterPileUpSteps = []

Definition at line 37 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.AllSteps = ["GEN,SIM", "DIGI", "L1", "DIGI2RAW", "HLT", "RAW2DIGI","RECO"]

Definition at line 36 of file cmsRelvalreportInput.py.

string cmsRelvalreportInput.cmsDriver = 'cmsDriver.py'

Definition at line 32 of file cmsRelvalreportInput.py.

cmsRelvalreportInput.debug = False

Definition at line 34 of file cmsRelvalreportInput.py.

list cmsRelvalreportInput.DEF_STEPS = ['GEN,SIM', 'DIGI']

Definition at line 35 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.hypreg = re.compile('-')

Definition at line 33 of file cmsRelvalreportInput.py.

dictionary cmsRelvalreportInput.Profiler
Initial value:
1 = {
2  'TimingReport' : 'Timing_Parser',
3  'TimingReport @@@ reuse' : 'Timing_Parser',
4  'TimeReport' : 'Timereport_Parser',
5  'TimeReport @@@ reuse' : 'Timereport_Parser',
6  'SimpleMemReport' : 'SimpleMem_Parser',
7  'EdmSize' : 'Edm_Size',
8  'IgProfperf' : 'IgProf_perf.PERF_TICKS',
9  'IgProfperf @@@ reuse' : 'IgProf_perf.PERF_TICKS',
10  'IgProfMemTotal' : 'IgProf_mem.MEM_TOTAL',
11  'IgProfMemTotal @@@ reuse': 'IgProf_mem.MEM_TOTAL',
12  'IgProfMemLive' : 'IgProf_mem.MEM_LIVE',
13  'IgProfMemLive @@@ reuse' : 'IgProf_mem.MEM_LIVE',
14  'IgProfMemMax' : 'IgProf_mem.MEM_MAX',
15  'IgProfMemMax @@@ reuse' : 'IgProf_mem.MEM_MAX',
16  'IgProfMemAnalyse' : 'IgProf_mem.ANALYSE',
17  'valgrind' : 'ValgrindFCE',
18  'memcheck_valgrind' : 'Memcheck_Valgrind',
19  'None' : 'None',
20 }

Definition at line 43 of file cmsRelvalreportInput.py.

tuple cmsRelvalreportInput.THIS_PROG_NAME = os.path.basename(sys.argv[0])

Global variables.

Definition at line 31 of file cmsRelvalreportInput.py.