CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
cmsPerfSuite.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 import os, time, sys, re, glob, exceptions
3 import optparse as opt
4 import cmsRelRegress as crr
5 from cmsPerfCommons import Candles, KeywordToCfi, CandFname, cmsDriverPileUpOption, getVerFromLog
6 import cmsRelValCmd,cmsCpuInfo
7 import threading #Needed in threading use for Valgrind
8 import subprocess #Nicer subprocess management than os.popen
9 import datetime #Used to time the running of the performance suite
10 import pickle #Used to dump the running timing information
11 
12 #Redefine _cleanup() function not to poll active processes
13 #[This is necessary to avoid issues when threading]
14 #So let's have it do nothing:
15 def _cleanup():
16  pass
17 #Override the function in subprocess
18 subprocess._cleanup=_cleanup
19 
20 class PerfThread(threading.Thread):
21  def __init__(self,**args):
22  self.args=args
23  threading.Thread.__init__(self)
24  def run(self):
25  self.suite=PerfSuite()
26  #print "Arguments inside the thread instance:"
27  #print type(self.args)
28  #print self.args
29  self.suite.runPerfSuite(**(self.args))#self.args)
30 
31 class PerfSuiteTimer:
32  """A class defining timing objects to time the running of the various parts of the performance suite. The class depends on module datetime."""
33  def __init__(self,start=None):
34  """Initialize the start time and set the end time to some indefinite time in the future"""
35  self.start = start
36  self.end = datetime.datetime.max
37  self.duration = self.start - self.end
38 
39  #Setters:
40  def set_start(self,start=None):
41  self.start = start
42  def set_end(self,end=None):
43  #print "Setting end time to %s"%end.ctime()
44  self.end = end
45  self.duration = self.end - self.start
46  #Getters
47  def get_start(self):
48  """Return the start time in ctime timestamp format"""
49  return self.start.ctime()
50  def get_end(self):
51  """Return the end time in ctime timestamp format"""
52  return self.end.ctime()
53  def get_duration(self):
54  """Return the duration between start and end as a dictionary with keys 'hours', 'minutes', 'seconds' to express the total duration in the favourite (most appropriate) unit. The function returns truncated integers."""
55  self.duration_seconds = self.duration.days*86400 + self.duration.seconds
56  self.duration_minutes = self.duration_seconds/60
57  self.duration_hours = self.duration_seconds/3600
58  return {'hours':self.duration_hours, 'minutes':self.duration_minutes, 'seconds':self.duration_seconds}
59 
60 class PerfSuite:
61  def __init__(self):
62 
63  self.ERRORS = 0
64  #Swtiching from CASTOR to EOS (using xrdcp instead of rfcp and root://eoscms//eos/ instead of /castor/cern.ch/
65  #NOT YET!
66  #FIX ME... do the migration to EOS eventually, taking care of PerFDB implications for tarball location!
67  self._CASTOR_DIR = "/castor/cern.ch/cms/store/relval/performance/"
68  self._dryrun = False
69  self._debug = False
70  self._unittest = False
71  self._noexec = False
72  self._verbose = True
73  self.logh = sys.stdout
74 
75  #Get some environment variables to use
76  try:
77  self.cmssw_arch = os.environ["SCRAM_ARCH"]
78  self.cmssw_version= os.environ["CMSSW_VERSION"]
79  self.host = os.environ["HOST"]
80  self.user = os.environ["USER"]
81  except KeyError:
82  self.logh.write('Error: An environment variable either SCRAM_ARCH, CMSSW_VERSION, HOST or USER is not available.\n')
83  self.logh.write(' Please run eval `scramv1 runtime -csh` to set your environment variables\n')
84  self.logh.flush()
85  sys.exit()
86 
87  #Scripts used by the suite:
88  self.Scripts =["cmsDriver.py","cmsRelvalreport.py","cmsRelvalreportInput.py","cmsScimark2"]
89  self.AuxiliaryScripts=["cmsScimarkLaunch.csh","cmsScimarkParser.py","cmsScimarkStop.py"]
90 
91 
92  #Threading the execution of IgProf, Memcheck and Callgrind using the same model used to thread the whole performance suite:
93  #1-Define a class simpleGenReportThread() that has relevant methods needed to handle PerfTest()
94  #2-Instantiate one with the necessary arguments to run simpleGenReport on core N
95  #3-Execute its "run" method by starting the thread
96  #Simplest way maybe is to keep 2 global lists:
97  #AvailableCores
98  #TestsToDo
99  #PerfSuite will fill the TestsToDo list with dictionaries, to be used as keyword arguments to instantiate a relevant thread.
100  #Once all the TestsToDo are "scheduled" into the list (FirstInLastOut buffer since we use pop()) PerfSuite will look into the
101  #AvailableCores list and start popping cores onto which to instantiate the relevant threads, then it will start the thread,
102  #appending it to the activePerfTestThread{},a dictionary with core as key and thread object as value, to facilitate bookkeeping.
103  #An infinite loop will take care of checking for AvailableCores as long as there are TestsToDo and keep submitting.
104  #In the same loop the activePerfTestThread{} will be checked for finished threads and it will re-append the relevant cpu back
105  #to the AvailableCores list.
106  #In the same loop a check for the case of all cores being back into AvailableCores with no more TestsToDo will break the infinite loop
107  #and declare the end of all tests.As else to this if a sleep statement of 5 seconds will delay the repetition of the loop.
108 
109  def createIgVolume(self):
110  igcommand = '/afs/cern.ch/cms/sdt/internal/scripts/requestPerfIgprofSpace.py --version ' + self.cmssw_version + ' --platform ' + self.cmssw_arch
111  subprocess.Popen(igcommand,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
112 
113 
114  class simpleGenReportThread(threading.Thread):
115  def __init__(self,cpu,perfsuiteinstance,**simpleGenReportArgs): #Passing around the perfsuite object to be able to access simpleGenReport
116  self.cpu=cpu
117  self.simpleGenReportArgs=simpleGenReportArgs
118  self.perfsuiteinstance=perfsuiteinstance
119  threading.Thread.__init__(self)
120  def run(self):
121  self.PerfTest=self.perfsuiteinstance.PerfTest(self.cpu,self.perfsuiteinstance,**(self.simpleGenReportArgs))
122  self.PerfTest.runPerfTest()
123 
124  class PerfTest:
125  def __init__(self,cpu,perfsuiteinstance,**simpleGenReportArgs):
126  self.cpu=cpu
127  self.simpleGenReportArgs=simpleGenReportArgs
128  self.perfsuiteinstance=perfsuiteinstance
129  def runPerfTest(self):
130 # self.PerfTestTotalTimer=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
131 # TimerInfo.update({self.simpleGenReportArgs['Name']:{'TotalTime':self.PerfTestTotalTimer}}) #Add the TimeSize timer to the dictionary
132  if "--pileup" in self.simpleGenReportArgs['cmsdriverOptions']:
133  self.perfsuiteinstance.logh.write("Launching the PILE UP %s tests on cpu %s with %s events each\n"%(self.simpleGenReportArgs['Name'],self.cpu,self.simpleGenReportArgs['NumEvents']))
134  self.PerfTestPUTimer=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
135  TimerInfo[self.simpleGenReportArgs['Name']].update({'PileUpTime':self.PerfTestPUTimer}) #Add the TimeSize timer to the dictionary
136 
137  else:
138  self.perfsuiteinstance.logh.write("Launching the %s tests on cpu %s with %s events each\n"%(self.simpleGenReportArgs['Name'],self.cpu,self.simpleGenReportArgs['NumEvents']))
139  self.PerfTestTimer=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
140  TimerInfo[self.simpleGenReportArgs['Name']].update({'NoPileUpTime':self.PerfTestTimer}) #Add the TimeSize timer to the dictionary
141  self.perfsuiteinstance.logh.flush()
142  #Cut and paste in bulk, should see if this works...
143  self.perfsuiteinstance.printDate()
144  self.perfsuiteinstance.logh.flush()
145  self.exitcode=self.perfsuiteinstance.simpleGenReport([self.cpu],**(self.simpleGenReportArgs)) #Returning ReportExit code
146  #Stop the timers on the threaded PileUp and NoPileUp tests:
147  if "--pileup" in self.simpleGenReportArgs['cmsdriverOptions']:
148  self.PerfTestPUTimer.set_end(datetime.datetime.now())
149  else:
150  self.PerfTestTimer.set_end(datetime.datetime.now())
151  return self.exitcode
152 
153  #Options handling
154  def optionParse(self,argslist=None):
155  parser = opt.OptionParser(usage='''./cmsPerfSuite.py [options]
156 
157  Examples:
158 
159  cmsPerfSuite.py --step GEN-HLT -t 5 -i 2 -c 1 -m 5 --RunTimeSize MinBias,TTbar --RunIgProf TTbar --RunCallgrind TTbar --RunMemcheck TTbar --RunDigiPileUp TTbar --PUInputFile /store/relval/CMSSW_2_2_1/RelValMinBias/GEN-SIM-DIGI-RAW-HLTDEBUG/IDEAL_V9_v2/0001/101C84AF-56C4-DD11-A90D-001D09F24EC0.root --cmsdriver="--eventcontent FEVTDEBUGHLT --conditions FrontierConditions_GlobalTag,IDEAL_V9::All"
160  (this will run the suite with 5 events for TimeSize tests on MinBias and TTbar, 2 for IgProf tests on TTbar only, 1 for Callgrind tests on TTbar only, 5 for Memcheck on MinBias and TTbar, it will also run DIGI PILEUP for all TTbar tests defined, i.e. 5 TimeSize, 2 IgProf, 1 Callgrind, 5 Memcheck. The file /store/relval/CMSSW_2_2_1/RelValMinBias/GEN-SIM-DIGI-RAW-HLTDEBUG/IDEAL_V9_v2/0001/101C84AF-56C4-DD11-A90D-001D09F24EC0.root will be copied locally as INPUT_PILEUP_EVENTS.root and it will be used as the input file for the MixingModule pile up events. All these tests will be done for the step GEN-HLT, i.e. GEN,SIM,DIGI,L1,DIGI2RAW,HLT at once)
161  OR
162  cmsPerfSuite.py --step GEN-HLT -t 5 -i 2 -c 1 -m 5 --RunTimeSize MinBias,TTbar --RunIgProf TTbar --RunCallgrind TTbar --RunMemcheck TTbar --RunTimeSizePU TTbar --PUInputFile /store/relval/CMSSW_2_2_1/RelValMinBias/GEN-SIM-DIGI-RAW-HLTDEBUG/IDEAL_V9_v2/0001/101C84AF-56C4-DD11-A90D-001D09F24EC0.root
163  (this will run the suite with 5 events for TimeSize tests on MinBias and TTbar, 2 for IgProf tests on TTbar only, 1 for Callgrind tests on TTbar only, 5 for Memcheck on MinBias and TTbar, it will also run DIGI PILEUP on TTbar but only for 5 TimeSize events. All these tests will be done for the step GEN-HLT, i.e. GEN,SIM,DIGI,L1,DIGI2RAW,HLT at once)
164  OR
165  cmsPerfSuite.py --step GEN-HLT -t 5 -i 2 -c 1 -m 5 --RunTimeSize MinBias,TTbar --RunIgProf TTbar --RunCallgrind TTbar --RunMemcheck TTbar --RunTimeSizePU TTbar --PUInputFile /store/relval/CMSSW_2_2_1/RelValMinBias/GEN-SIM-DIGI-RAW-HLTDEBUG/IDEAL_V9_v2/0001/101C84AF-56C4-DD11-A90D-001D09F24EC0.root --cmsdriver="--eventcontent RAWSIM --conditions FrontierConditions_GlobalTag,IDEAL_V9::All"
166  (this will run the suite with 5 events for TimeSize tests on MinBias and TTbar, 2 for IgProf tests on TTbar only, 1 for Callgrind tests on TTbar only, 5 for Memcheck on MinBias and TTbar, it will also run DIGI PILEUP on TTbar but only for 5 TimeSize events. All these tests will be done for the step GEN-HLT, i.e. GEN,SIM,DIGI,L1,DIGI2RAW,HLT at once. It will also add the options "--eventcontent RAWSIM --conditions FrontierConditions_GlobalTag,IDEAL_V9::All" to all cmsDriver.py commands executed by the suite. In addition it will run only 2 cmsDriver.py "steps": "GEN,SIM" and "DIGI". Note the syntax GEN-SIM for combined cmsDriver.py steps)
167 
168  Legal entries for individual candles (--RunTimeSize, --RunIgProf, --RunCallgrind, --RunMemcheck options):
169  %s
170  ''' % ("\n".join(Candles)))
171 
172  parser.set_defaults(TimeSizeEvents = 0 ,
173  IgProfEvents = 0 ,
174  CallgrindEvents = 0 ,
175  MemcheckEvents = 0 ,
176  cmsScimark = 10 ,
177  cmsScimarkLarge = 10 ,
178  cmsdriverOptions = "--eventcontent FEVTDEBUGHLT", # Decided to avoid using the automatic parsing of cmsDriver_highstats_hlt.txt: cmsRelValCmd.get_cmsDriverOptions(), #Get these options automatically now!
179  #"Release Integrators" will create another file relative to the performance suite and the operators will fetch from that file the --cmsdriver option... for now just set the eventcontent since that is needed in order for things to run at all now...
180  stepOptions = "" ,
181  profilers = "" ,
182  outputdir = "" ,
183  logfile = os.path.join(os.getcwd(),"cmsPerfSuite.log"),
184  runonspare = True ,
185  bypasshlt = False ,
186  quicktest = False ,
187  unittest = False ,
188  noexec = False ,
189  dryrun = False ,
190  verbose = True ,
191  create = False ,
192  previousrel = "" ,
193  castordir = self._CASTOR_DIR,
194  cores = cmsCpuInfo.get_NumOfCores(), #Get Number of cpu cores on the machine from /proc/cpuinfo
195  cpu = "1" , #Cpu core on which the suite is run:
196  RunTimeSize = "" ,
197  RunIgProf = "" ,
198  RunCallgrind = "" ,
199  RunMemcheck = "" ,
200  RunDigiPileUP = "" ,
201  RunTimeSizePU = "" ,
202  RunIgProfPU = "" ,
203  RunCallgrindPU = "" ,
204  RunMemcheckPU = "" ,
205  PUInputFile = "" ,
206  userInputFile = "" )
207  parser.add_option('--createIgVol', action="store_true", dest='create',
208  help = 'Create IgProf AFS volume for the release and architecture')
209  parser.add_option('-q', '--quiet' , action="store_false", dest='verbose' ,
210  help = 'Output less information' )
211  parser.add_option('-b', '--bypass-hlt' , action="store_true" , dest='bypasshlt' ,
212  help = 'Bypass HLT root file as input to RAW2DIGI')
213  parser.add_option('-n', '--notrunspare', action="store_false", dest='runonspare',
214  help = 'Do not run cmsScimark on spare cores')
215  parser.add_option('-t', '--timesize' , type='int' , dest='TimeSizeEvents' , metavar='<#EVENTS>' ,
216  help = 'specify the number of events for the TimeSize tests' )
217  parser.add_option('-i', '--igprof' , type='int' , dest='IgProfEvents' , metavar='<#EVENTS>' ,
218  help = 'specify the number of events for the IgProf tests' )
219  parser.add_option('-c', '--callgrind' , type='int' , dest='CallgrindEvents' , metavar='<#EVENTS>' ,
220  help = 'specify the number of events for the Callgrind tests' )
221  parser.add_option('-m', '--memcheck' , type='int' , dest='MemcheckEvents' , metavar='<#EVENTS>' ,
222  help = 'specify the number of events for the Memcheck tests' )
223  parser.add_option('--cmsScimark' , type='int' , dest='cmsScimark' , metavar='' ,
224  help = 'specify the number of times the cmsScimark benchmark is run before and after the performance suite on cpu1')
225  parser.add_option('--cmsScimarkLarge' , type='int' , dest='cmsScimarkLarge' , metavar='' ,
226  help = 'specify the number of times the cmsScimarkLarge benchmark is run before and after the performance suite on cpu1')
227  parser.add_option('--cores' , type='int', dest='cores' , metavar='<CORES>' ,
228  help = 'specify the number of cores of the machine (can be used with 0 to stop cmsScimark from running on the other cores)')
229  parser.add_option('--cmsdriver' , type='string', dest='cmsdriverOptions', metavar='<OPTION_STR>',
230  help = 'specify special options to use with the cmsDriver.py commands (designed for integration build use')
231  parser.add_option('-a', '--archive' , type='string', dest='castordir' , metavar='<DIR>' ,
232  help = 'specify the wanted CASTOR directory where to store the results tarball')
233  parser.add_option('-L', '--logfile' , type='string', dest='logfile' , metavar='<FILE>' ,
234  help = 'file to store log output of the script')
235  parser.add_option('-o', '--output' , type='string', dest='outputdir' , metavar='<DIR>' ,
236  help = 'specify the directory where to store the output of the script')
237  parser.add_option('-r', '--prevrel' , type='string', dest='previousrel' , metavar='<DIR>' ,
238  help = 'Top level dir of previous release for regression analysis')
239  parser.add_option('--step' , type='string', dest='stepOptions' , metavar='<STEPS>' ,
240  help = 'specify the processing steps intended (instead of the default ones)' )
241  parser.add_option('--cpu' , type='string', dest='cpu' , metavar='<CPU>' ,
242  help = 'specify the core on which to run the performance suite')
243 
244  #Adding new options to put everything configurable at command line:
245  parser.add_option('--RunTimeSize' , type='string', dest='RunTimeSize' , metavar='<CANDLES>' ,
246  help = 'specify on which candles to run the TimeSize tests')
247  parser.add_option('--RunIgProf' , type='string', dest='RunIgProf' , metavar='<CANDLES>' ,
248  help = 'specify on which candles to run the IgProf tests')
249  parser.add_option('--RunCallgrind' , type='string', dest='RunCallgrind' , metavar='<CANDLES>' ,
250  help = 'specify on which candles to run the Callgrind tests')
251  parser.add_option('--RunMemcheck' , type='string', dest='RunMemcheck' , metavar='<CANDLES>' ,
252  help = 'specify on which candles to run the Memcheck tests')
253  parser.add_option('--RunDigiPileUp' , type='string', dest='RunDigiPileUp' , metavar='<CANDLES>' ,
254  help = 'specify the candle on which to run DIGI PILE UP and repeat all the tests set to run on that candle with PILE UP')
255  parser.add_option('--PUInputFile' , type='string', dest='PUInputFile' , metavar='<FILE>' ,
256  help = 'specify the root file to pick the pile-up events from')
257  parser.add_option('--RunTimeSizePU' , type='string', dest='RunTimeSizePU' , metavar='<CANDLES>' ,
258  help = 'specify on which candles to run the TimeSize tests with PILE UP')
259  parser.add_option('--RunIgProfPU' , type='string', dest='RunIgProfPU' , metavar='<CANDLES>' ,
260  help = 'specify on which candles to run the IgProf tests with PILE UP')
261  parser.add_option('--RunCallgrindPU' , type='string', dest='RunCallgrindPU' , metavar='<CANDLES>' ,
262  help = 'specify on which candles to run the Callgrind tests with PILE UP')
263  parser.add_option('--RunMemcheckPU' , type='string', dest='RunMemcheckPU' , metavar='<CANDLES>' ,
264  help = 'specify on which candles to run the Memcheck tests with PILE UP')
265 
266  #Adding a filein option to use pre-processed RAW file for RECO and HLT:
267  parser.add_option('--filein' , type='string', dest='userInputFile' , metavar='<FILE>', #default="",
268  help = 'specify input RAW root file for HLT and RAW2DIGI-RECO (list the files in the same order as the candles for the tests)')
269 
270  #Adding an option to handle additional (to the default user) email addresses to the email notification list (that sends the cmsPerfSuite.log once the performance suite is done running):
271  parser.add_option('--mail', type='string', dest='MailLogRecipients', metavar='<EMAIL ADDRESS>', default=self.user, help='specify valid email address(es) name@domain in order to receive notification at the end of the performance suite running with the cmsPerfSuite.log file')
272 
273  #Adding option to turn off tarball creation at the end of the execution of the performance suite:
274  parser.add_option('--no_tarball', action="store_false", dest='tarball', default=True, help='Turn off automatic tarball creation at the end of the performance suite execution')
275 
276  #####################
277  #
278  # Developer options
279  #
280 
281  devel = opt.OptionGroup(parser, "Developer Options",
282  "Caution: use these options at your own risk."
283  "It is believed that some of them bite.\n")
284 
285  devel.add_option('-p', '--profile' , type="str" , dest='profilers', metavar="<PROFILERS>" ,
286  help = 'Profile codes to use for cmsRelvalInput' )
287  devel.add_option('-f', '--false-run', action="store_true", dest='dryrun' ,
288  help = 'Dry run' )
289  devel.add_option('-d', '--debug' , action='store_true', dest='debug' ,
290  help = 'Debug' )
291  devel.add_option('--quicktest' , action="store_true", dest='quicktest',
292  help = 'Quick overwrite all the defaults to small numbers so that we can run a quick test of our chosing.' )
293  devel.add_option('--test' , action="store_true", dest='unittest' ,
294  help = 'Perform a simple test, overrides other options. Overrides verbosity and sets it to false.' )
295  devel.add_option('--no_exec' , action="store_true", dest='noexec' ,
296  help = 'Run the suite without executing the cmsRelvalreport.py commands in the various directories. This is a useful debugging tool.' )
297  parser.add_option_group(devel)
298  (options, args) = parser.parse_args(argslist)
299 
300 
301  self._debug = options.debug
302  self._unittest = options.unittest
303  self._noexec = options.noexec
304  self._verbose = options.verbose
305  self._dryrun = options.dryrun
306  create = options.create
307  castordir = options.castordir
308  TimeSizeEvents = options.TimeSizeEvents
309  IgProfEvents = options.IgProfEvents
310  CallgrindEvents = options.CallgrindEvents
311  MemcheckEvents = options.MemcheckEvents
312  cmsScimark = options.cmsScimark
313  cmsScimarkLarge = options.cmsScimarkLarge
314  cmsdriverOptions = options.cmsdriverOptions
315  stepOptions = options.stepOptions
316  quicktest = options.quicktest
317  #candleoption = options.candleOptions
318  runonspare = options.runonspare
319  profilers = options.profilers.strip()
320  cpu = options.cpu.strip()
321  bypasshlt = options.bypasshlt
322  cores = options.cores
323  logfile = options.logfile
324  prevrel = options.previousrel
325  outputdir = options.outputdir
326  RunTimeSize = options.RunTimeSize
327  RunIgProf = options.RunIgProf
328  RunCallgrind = options.RunCallgrind
329  RunMemcheck = options.RunMemcheck
330  RunDigiPileUp = options.RunDigiPileUp
331  RunTimeSizePU = options.RunTimeSizePU
332  RunIgProfPU = options.RunIgProfPU
333  RunCallgrindPU = options.RunCallgrindPU
334  RunMemcheckPU = options.RunMemcheckPU
335  PUInputFile = options.PUInputFile
336  userInputFile = options.userInputFile
337  if options.MailLogRecipients !="" and self.user not in options.MailLogRecipients: #To allow for the --mail "" case of suppressing the email and the default user case
338  MailLogRecipients= self.user+","+options.MailLogRecipients #Add the user by default if there is a mail report
339  else:
340  MailLogRecipients=options.MailLogRecipients
341  tarball = options.tarball
342 
343  #################
344  # Check logfile option
345  #
346  if not logfile == None:
347  logfile = os.path.abspath(logfile)
348  logdir = os.path.dirname(logfile)
349  if not os.path.exists(logdir):
350  parser.error("Directory to output logfile does not exist")
351  sys.exit()
352  logfile = os.path.abspath(logfile)
353 
354  #############
355  # Check step Options
356  #
357  if "GEN,SIM" in stepOptions:
358  self.logh.write("WARNING: Please use GEN-SIM with a hypen not a \",\"!\n")
359  #Using the step option as a switch between different dictionaries for:
360  #RunTimeSize,RunIgProf,RunCallgrind,RunMemCheck,RunDigiPileUp:
361  if stepOptions == "" or stepOptions == 'Default':
362  pass
363  else:
364  stepOptions='--usersteps=%s' % (stepOptions)
365 
366  ###############
367  # Check profile option
368  #
369  isnumreg = re.compile("^-?[0-9]*$")
370  found = isnumreg.search(profilers)
371  if not found :
372  parser.error("profile codes option contains non-numbers")
373  sys.exit()
374 
375  ###############
376  # Check output directory option
377  #
378  if outputdir == "":
379  outputdir = os.getcwd()
380  else:
381  outputdir = os.path.abspath(outputdir)
382 
383  if not os.path.isdir(outputdir):
384  parser.error("%s is not a valid output directory" % outputdir)
385  sys.exit()
386 
387  ################
388  # Check cpu option
389  #
390  numetcomreg = re.compile("^[0-9,]*")
391  if not numetcomreg.search(cpu):
392  parser.error("cpu option needs to be a comma separted list of ints or a single int")
393  sys.exit()
394 
395  cpustr = cpu
396  cpu = []
397  if "," in cpustr:
398  cpu = map(lambda x: int(x),cpustr.split(","))
399  else:
400  cpu = [ int(cpustr) ]
401 
402  ################
403  # Check previous release directory
404  #
405  if not prevrel == "":
406  prevrel = os.path.abspath(prevrel)
407  if not os.path.exists(prevrel):
408  self.logh.write("ERROR: Previous release dir %s could not be found" % prevrel)
409  sys.exit()
410 
411  #############
412  # Setup quicktest option
413  #
414  if quicktest:
415  TimeSizeEvents = 1
416  IgProfEvents = 1
417  CallgrindEvents = 0
418  MemcheckEvents = 0
419  cmsScimark = 1
420  cmsScimarkLarge = 1
421 
422  #############
423  # Setup unit test option
424  #
425  if self._unittest:
426  self._verbose = False
427  if stepOptions == "":
428  stepOptions = "GEN-SIM,DIGI,L1,DIGI2RAW,HLT,RAW2DIGI-RECO"
429  cmsScimark = 0
430  cmsScimarkLarge = 0
431  CallgrindEvents = 0
432  MemcheckEvents = 0
433  IgProfEvents = 0
434  TimeSizeEvents = 1
435 
436  #Split all the RunTimeSize etc candles in lists:
437  TimeSizeCandles=[]
438  IgProfCandles=[]
439  CallgrindCandles=[]
440  MemcheckCandles=[]
441  TimeSizePUCandles=[]
442  IgProfPUCandles=[]
443  CallgrindPUCandles=[]
444  MemcheckPUCandles=[]
445  userInputRootFiles=[]
446  if RunTimeSize:
447  TimeSizeCandles = RunTimeSize.split(",")
448  if RunIgProf:
449  IgProfCandles = RunIgProf.split(",")
450  if RunCallgrind:
451  CallgrindCandles = RunCallgrind.split(",")
452  if RunMemcheck:
453  MemcheckCandles = RunMemcheck.split(",")
454  if RunDigiPileUp:
455  for candle in RunDigiPileUp.split(","):
456  if candle in TimeSizeCandles:
457  TimeSizePUCandles.append(candle)
458  if candle in IgProfCandles:
459  IgProfPUCandles.append(candle)
460  if candle in CallgrindCandles:
461  CallgrindPUCandles.append(candle)
462  if candle in MemcheckCandles:
463  MemcheckPUCandles.append(candle)
464  if RunTimeSizePU:
465  TimeSizePUCandles.extend(RunTimeSizePU.split(","))
466  #Some smart removal of duplicates from the list!
467  temp=set(TimeSizePUCandles)
468  TimeSizePUCandles=list(temp) #Doing it in 2 steps to avoid potential issues with type of arguments
469  if RunIgProfPU:
470  IgProfPUCandles.extend(RunIgProfPU.split(","))
471  #Some smart removal of duplicates from the list!
472  temp=set(IgProfPUCandles)
473  IgProfPUCandles=list(temp) #Doing it in 2 steps to avoid potential issues with type of arguments
474  if RunCallgrindPU:
475  CallgrindPUCandles.extend(RunCallgrindPU.split(","))
476  #Some smart removal of duplicates from the list!
477  temp=set(CallgrindPUCandles)
478  CallgrindPUCandles=list(temp) #Doing it in 2 steps to avoid potential issues with type of arguments
479  if RunMemcheckPU:
480  MemcheckPUCandles.extend(RunMemcheckPU.split(","))
481  #Some smart removal of duplicates from the list!
482  temp=set(MemcheckPUCandles)
483  MemcheckPUCandles=list(temp) #Doing it in 2 steps to avoid potential issues with type of arguments
484  if userInputFile:
485  userInputRootFiles=userInputFile.split(",")
486 
487 
488 
489  #############
490  # Setup cmsdriver and eventual cmsdriverPUoption
491  #
492  cmsdriverPUOptions=""
493  if cmsdriverOptions:
494  #Set the eventual Pile Up cmsdriver options first:
495  if TimeSizePUCandles or IgProfPUCandles or CallgrindPUCandles or MemcheckPUCandles:
496  #Bug fixed: no space between --pileup= and LowLumiPileUp (otherwise could omit the =)
497  cmsdriverPUOptions = '--cmsdriver="%s %s%s"'%(cmsdriverOptions," --pileup=",cmsDriverPileUpOption)
498  #Set the regular ones too:
499  cmsdriverOptions = '--cmsdriver="%s"'%cmsdriverOptions
500 
501  return (create ,
502  castordir ,
503  TimeSizeEvents ,
504  IgProfEvents ,
505  CallgrindEvents ,
506  MemcheckEvents ,
507  cmsScimark ,
508  cmsScimarkLarge ,
509  cmsdriverOptions,
510  cmsdriverPUOptions,
511  stepOptions ,
512  quicktest ,
513  profilers ,
514  cpu ,
515  cores ,
516  prevrel ,
517  bypasshlt ,
518  runonspare ,
519  outputdir ,
520  logfile ,
521  TimeSizeCandles ,
522  IgProfCandles ,
523  CallgrindCandles,
524  MemcheckCandles ,
525  TimeSizePUCandles ,
526  IgProfPUCandles ,
527  CallgrindPUCandles,
528  MemcheckPUCandles ,
529  PUInputFile ,
530  userInputRootFiles,
531  MailLogRecipients,
532  tarball)
533 
534  #def usage(self):
535  # return __doc__
536 
537  ############
538  # Run a list of commands using system
539  # ! We should rewrite this not to use system (most cases it is unnecessary)
540  def runCmdSet(self,cmd):
541  exitstat = 0
542  if len(cmd) <= 1:
543  exitstat = self.runcmd(cmd)
544  if self._verbose:
545  self.printFlush(cmd)
546  else:
547  for subcmd in cmd:
548  if self._verbose:
549  self.printFlush(subcmd)
550  exitstat = self.runcmd(" && ".join(cmd))
551  if self._verbose:
552  self.printFlush(self.getDate())
553  return exitstat
554 
555  #############
556  # Print and flush a string (for output to a log file)
557  #
558  def printFlush(self,command):
559  if self._verbose:
560  self.logh.write(str(command) + "\n")
561  self.logh.flush()
562 
563  #############
564  # Run a command and return the exit status
565  #
566  def runcmd(self,command):
567  #Substitute popen with subprocess.Popen!
568  #Using try/except until Popen becomes thread safe (it seems that everytime it is called
569  #all processes are checked to reap the ones that are done, this creates a race condition with the wait()... that
570  #results into an error with "No child process".
571  #os.popen(command)
572  try:
573  process = subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
574  pid=process.pid
575  exitstat= process.wait()
576  cmdout = process.stdout.read()
577  exitstat = process.returncode
578  except OSError, detail:
579  self.logh.write("Race condition in subprocess.Popen has robbed us of the exit code of the %s process (PID %s).Assume it failed!\n %s\n"%(command,pid,detail))
580  self.logh.flush()
581  exitstat=999
582  cmdout="Race condition in subprocess.Popen has robbed us of the exit code of the %s process (PID %s).Assume it failed!\n %s"%(command,pid,detail)
583  if self._verbose:
584  self.logh.write(cmdout)# + "\n") No need of extra \n!
585  self.logh.flush()
586  if exitstat == None:
587  self.logh.write("Something strange is going on! Exit code was None for command %s: check if it really ran!"%command)
588  self.logh.flush()
589  exitstat=0
590  return exitstat
591 
592  def getDate(self):
593  return time.ctime()
594 
595  def printDate(self):
596  self.logh.write(self.getDate() + "\n")
597  self.logh.flush()
598  #############
599  # Make directory for a particular candle and profiler.
600  # ! This is really unnecessary code and should be replaced with a os.mkdir() call
601  def mkCandleDir(self,pfdir,candle,profiler):
602  adir = os.path.join(pfdir,"%s_%s" % (candle,profiler))
603  self.runcmd( "mkdir -p %s" % adir )
604  if self._verbose:
605  self.printDate()
606  return adir
607 
608  #############
609  # Copy root file from another candle's directory
610  # ! Again this is messy.
611 
612  def cprootfile(self,dir,candle,NumOfEvents,cmsdriverOptions=""):
613  cmds = ("cd %s" % dir,
614  "cp -pR ../%s_IgProf/%s_GEN,SIM.root ." % (candle,CandFname[candle]))
615 
616  if self.runCmdSet(cmds):
617  self.logh.write("Since there was no ../%s_IgProf/%s_GEN,SIM.root file it will be generated first\n"%(candle,CandFname[candle]))
618 
619  cmd = "cd %s ; cmsDriver.py %s -s GEN,SIM -n %s --fileout %s_GEN,SIM.root %s>& %s_GEN_SIM_for_valgrind.log" % (dir,KeywordToCfi[candle],str(NumOfEvents),candle,cmsdriverOptions,candle)
620 
621  self.printFlush(cmd)
622  #Obsolete popen4-> subprocess.Popen
623  #cmdout=os.popen3(cmd)[2].read()
624  cmdout=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
625  if cmdout:
626  self.printFlush(cmdout)
627  return cmdout
628 
629  #############
630  # Display G4 cerr errors and CMSExceptions in the logfile
631  #
632  def displayErrors(self,file):
633  try:
634  for line in open(file,"r"):
635  if "cerr" in line or "CMSException" in line:
636  self.logh.write("ERROR: %s\n" % line)
637  self.ERRORS += 1
638  except OSError, detail:
639  self.logh.write("WARNING: %s\n" % detail)
640  self.ERRORS += 1
641  except IOError, detail:
642  self.logh.write("WARNING: %s\n" % detail)
643  self.ERRORS += 1
644 
645  ##############
646  # Filter lines in the valgrind report that match GEN,SIM
647  #
648  def valFilterReport(self,dir):
649  #cmds = ("cd %s" % dir,
650  # "grep -v \"step=GEN,SIM\" SimulationCandles_%s.txt > tmp" % (self.cmssw_version),
651  # "mv tmp SimulationCandles_%s.txt" % (self.cmssw_version))
652  #FIXME:
653  #Quick and dirty hack to have valgrind MemCheck run on 5 events on both GEN,SIM and DIGI in QCD_80_120, while removing the line for GEN,SIM for Callgrind
654  InputFileName=os.path.join(dir,"SimulationCandles_%s.txt"%(self.cmssw_version))
655  InputFile=open(InputFileName,"r")
656  InputLines=InputFile.readlines()
657  InputFile.close()
658  Outputfile=open(InputFileName,"w")
659  simRegxp=re.compile("step=GEN,SIM")
660  digiRegxp=re.compile("step=DIGI")
661  CallgrindRegxp=re.compile("ValgrindFCE")
662  MemcheckRegxp=re.compile("Memcheck")
663  NumEvtRegxp=re.compile("-n 1")#FIXME Either use the ValgrindEventNumber or do a more general match!
664  for line in InputLines:
665  if simRegxp.search(line) and CallgrindRegxp.search(line):
666  continue
667  elif simRegxp.search(line) and MemcheckRegxp.search(line):
668  #Modify
669  if NumEvtRegxp.search(line):
670  line=NumEvtRegxp.sub(r"-n 5",line)
671  else:
672  self.logh.write("The number of Memcheck event was not changed since the original number of Callgrind event was not 1!\n")
673  Outputfile.write(line)
674  elif digiRegxp.search(line) and MemcheckRegxp.search(line):
675  #Modify
676  if NumEvtRegxp.search(line):
677  line=NumEvtRegxp.sub(r"-n 5",line)
678  else:
679  self.logh.write("The number of Memcheck event was not changed since the original number of Callgrind event was not 1!\n")
680  Outputfile.write(line)
681  else:
682  Outputfile.write(line)
683  self.logh.flush()
684  Outputfile.close()
685 
686  #self.runCmdSet(cmds)
687 
688  ##################
689  # Run cmsScimark benchmarks a number of times
690  #
691  def benchmarks(self,cpu,pfdir,name,bencher,large=False):
692  cmd = self.Commands[cpu][3]
693  redirect = ""
694  if large:
695  redirect = " -large >>"
696  else:
697  redirect = " >>"
698 
699  for i in range(bencher):
700  #Check first for the existence of the file so that we can append:
701  if not os.path.exists(os.path.join(pfdir,os.path.basename(name))):
702  #Equivalent of touch to make sure the file exist to be able to append to it.
703  open(os.path.join(pfdir,os.path.basename(name)))
704 
705  command= cmd + redirect + os.path.join(pfdir,os.path.basename(name))
706  self.printFlush(command + " [%s/%s]" % (i+1,bencher))
707  self.runcmd(command)
708  self.logh.flush()
709 
710  ##################
711  # This function is a wrapper around cmsRelvalreport
712  #
713  def runCmsReport(self,cpu,dir,candle):
714  cmd = self.Commands[cpu][1]
715  cmds = ("cd %s" % (dir),
716  "%s -i SimulationCandles_%s.txt -t perfreport_tmp -R -P >& %s.log" % (cmd,self.cmssw_version,candle))
717  exitstat = 0
718  if not self._debug:
719  exitstat = self.runCmdSet(cmds)
720 
721  if self._unittest and (not exitstat == 0):
722  self.logh.write("ERROR: CMS Report returned a non-zero exit status \n")
723  sys.exit(exitstat)
724  else:
725  return(exitstat) #To return the exit code of the cmsRelvalreport.py commands to the runPerfSuite function
726 
727  ##################
728  # Test cmsDriver.py (parses the simcandles file, removing duplicate lines, and runs the cmsDriver part)
729  #
730  def testCmsDriver(self,cpu,dir,cmsver,candle):
731  cmsdrvreg = re.compile("^cmsDriver.py")
732  cmd = self.Commands[cpu][0]
733  noExit = True
734  stepreg = re.compile("--step=([^ ]*)")
735  previousCmdOnline = ""
736  for line in open(os.path.join(dir,"SimulationCandles_%s.txt" % (cmsver))):
737  if (not line.lstrip().startswith("#")) and not (line.isspace() or len(line) == 0):
738  cmdonline = line.split("@@@",1)[0]
739  if cmsdrvreg.search(cmdonline) and not previousCmdOnline == cmdonline:
740  stepbeingrun = "Unknown"
741  matches = stepreg.search(cmdonline)
742  if not matches == None:
743  stepbeingrun = matches.groups()[0]
744  if "PILEUP" in cmdonline:
745  stepbeingrun += "_PILEUP"
746  self.logh.write(cmdonline + "\n")
747  cmds = ("cd %s" % (dir),
748  "%s >& ../cmsdriver_unit_test_%s_%s.log" % (cmdonline,candle,stepbeingrun))
749  if self._dryrun:
750  self.logh.write(cmds + "\n")
751  else:
752  out = self.runCmdSet(cmds)
753  if not out == None:
754  sig = out >> 16 # Get the top 16 bits
755  xstatus = out & 0xffff # Mask out all bits except the first 16
756  self.logh.write("FATAL ERROR: CMS Driver returned a non-zero exit status (which is %s) when running %s for candle %s. Signal interrupt was %s\n" % (xstatus,stepbeingrun,candle,sig))
757  sys.exit()
758  previousCmdOnline = cmdonline
759 
760  ##############
761  # Wrapper for cmsRelvalreportInput
762  #
763  def runCmsInput(self,cpu,dir,numevents,candle,cmsdrvopts,stepopt,profiles,bypasshlt,userInputFile):
764 
765  #Crappy fix for optional options with special synthax (bypasshlt and userInputFile)
766  bypass = ""
767  if bypasshlt:
768  bypass = "--bypass-hlt"
769  userInputFileOption=""
770  if userInputFile:
771  userInputFileOption = "--filein %s"%userInputFile
772  cmd = self.Commands[cpu][2]
773  cmds=[]
774  #print cmds
775  cmds = ("cd %s" % (dir),
776  "%s %s \"%s\" %s %s %s %s %s" % (cmd,
777  numevents,
778  candle,
779  profiles,
780  cmsdrvopts,
781  stepopt,
782  bypass,userInputFileOption))
783  exitstat=0
784  exitstat = self.runCmdSet(cmds)
785  if self._unittest and (not exitstat == 0):
786  self.logh.write("ERROR: CMS Report Input returned a non-zero exit status \n" )
787  return exitstat
788  ##############
789  # Prepares the profiling directory and runs all the selected profiles (if this is not a unit test)
790  #
791  #Making parameters named to facilitate the handling of arguments (especially with the threading use case)
792  def simpleGenReport(self,cpus,perfdir=os.getcwd(),NumEvents=1,candles=['MinBias'],cmsdriverOptions='',stepOptions='',Name='',profilers='',bypasshlt='',userInputRootFiles=''):
793  callgrind = Name == "Callgrind"
794  memcheck = Name == "Memcheck"
795 
796  profCodes = {"TimeSize" : "0123",
797  "IgProf" : "4567",
798  "IgProf_Perf":"47", #Added the Analyse to IgProf_Perf #FIXME: At the moment Analyse is always run whether 7 is selected or not! Issue to solve in cmsRelvalreportInput.py... but not really important (it's always been there, not impacting our use-cases).
799  "IgProf_Mem":"567",
800  "Callgrind": "8",
801  "Memcheck" : "9",
802  None : "-1"}
803 
804  profiles = profCodes[Name]
805  if not profilers == "":
806  profiles = profilers
807 
808  RelvalreportExitCode=0
809 
810  for cpu in cpus:
811  pfdir = perfdir
812  if len(cpus) > 1:
813  pfdir = os.path.join(perfdir,"cpu_%s" % cpu)
814  for candle in candles:
815  #Create the directory for cmsRelvalreport.py running (e.g. MinBias_TimeSize, etc)
816  #Catch the case of PILE UP:
817  if "--pileup" in cmsdriverOptions:
818  candlename=candle+"_PU"
819  else:
820  candlename=candle
821  adir=self.mkCandleDir(pfdir,candlename,Name)
822  if self._unittest:
823  # Run cmsDriver.py
824  if userInputRootFiles:
825  self.logh.write(userInputRootFiles)
826  userInputFile=userInputRootFiles[0]
827  else:
828  userInputFile=""
829  self.logh.flush()
830  self.runCmsInput(cpu,adir,NumEvents,candle,cmsdriverOptions,stepOptions,profiles,bypasshlt,userInputFile)
831  self.testCmsDriver(cpu,adir,candle)
832  else:
833  if userInputRootFiles:
834  self.logh.write("Variable userInputRootFiles is %s\n"%userInputRootFiles)
835  #Need to use regexp, cannot rely on the order... since for different tests there are different candles...
836  #userInputFile=userInputRootFiles[candles.index(candle)]
837  #FIXME:
838  #Note the issue that the input files HAVE to have in their name the candle as is used in cmsPerfSuite.py command line!
839  #This is currently caught by a printout in the log: should be either taken care of with some exception to throw?
840  #Will put this in the documentation
841  userInputFile=""
842  candleregexp=re.compile(candle)
843  for file in userInputRootFiles:
844  if candleregexp.search(file):
845  userInputFile=file
846  self.logh.write("For these %s %s tests will use user input file %s\n"%(candlename,Name,userInputFile))
847  if userInputFile == "":
848  self.logh.write("***WARNING: For these %s %s tests could not find a matching input file in %s: will try to do without it!!!!!\n"%(candlename,Name,userInputRootFiles))
849  self.logh.flush()
850  else:
851  userInputFile=""
852  DummyTestName=candlename+"_"+stepOptions.split("=")[1]
853  DummyTimer=PerfSuiteTimer(start=datetime.datetime.now()) #Start the timer (DummyTimer is just a reference, but we will use the dictionary to access this later...
854  TimerInfo[Name].update({DummyTestName:DummyTimer}) #Add the TimeSize timer to the dictionary
855  #The following command will create the appropriate SimulationCandlesX.txt file in the relevant directory, ready to run cmsRelvalreport.py on it.
856  self.runCmsInput(cpu,adir,NumEvents,candle,cmsdriverOptions,stepOptions,profiles,bypasshlt,userInputFile)
857  #Here where the no_exec option kicks in (do everything but do not launch cmsRelvalreport.py, it also prevents cmsScimark spawning...):
858  if self._noexec:
859  self.logh.write("Running in debugging mode, without executing cmsRelvalreport.py\n")
860  self.logh.flush()
861  pass
862  else:
863  #The following command will launch cmsRelvalreport.py on the SimulationCandlesX.txt input file created above.
864  ExitCode=self.runCmsReport(cpu,adir,candle)
865  self.logh.write("Individual cmsRelvalreport.py ExitCode %s\n"%ExitCode)
866  RelvalreportExitCode=RelvalreportExitCode+ExitCode
867  self.logh.write("Summed cmsRelvalreport.py ExitCode %s\n"%RelvalreportExitCode)
868  self.logh.flush()
869  DummyTimer.set_end(datetime.datetime.now())
870 
871  #for proflog in proflogs:
872  #With the change from 2>1&|tee to >& to preserve exit codes, we need now to check all logs...
873  #less nice... we might want to do this externally so that in post-processing its a re-usable tool
874  globpath = os.path.join(adir,"*.log") #"%s.log"%candle)
875  self.logh.write("Looking for logs that match %s\n" % globpath)
876  logs = glob.glob(globpath)
877  for log in logs:
878  self.logh.write("Found log %s\n" % log)
879  self.displayErrors(log)
880  self.printFlush("Returned cumulative RelvalreportExitCode is %s"%RelvalreportExitCode)
881  return RelvalreportExitCode
882 
883  ############
884  # Runs benchmarking, cpu spinlocks on spare cores and profiles selected candles
885  #
886  #FIXME:
887  #Could redesign interface of functions to use keyword arguments:
888  #def runPerfSuite(**opts):
889  #then instead of using castordir variable, would use opts['castordir'] etc
890  def runPerfSuite(self,
891  create = False,
892  #Swtiching from CASTOR to EOS (using xrdcp instead of rfcp and root://eoscms//eos/ instead of /castor/cern.ch/
893  #Actually not yet... for consistency we will keep it on CASTOR for now
894  #FIXME! Do the migration, following its implication in PerfDB application!
895  castordir = "/castor/cern.ch/cms/store/relval/performance/",
896  TimeSizeEvents = 100 ,
897  IgProfEvents = 5 ,
898  CallgrindEvents = 1 ,
899  MemcheckEvents = 5 ,
900  cmsScimark = 10 ,
901  cmsScimarkLarge = 10 ,
902  cmsdriverOptions = "" ,#Could use directly cmsRelValCmd.get_Options()
903  cmsdriverPUOptions= "" ,
904  stepOptions = "" ,
905  quicktest = False ,
906  profilers = "" ,
907  cpus = [1] ,
908  cpu_list = [1] ,
909  cores = 4 ,#Could use directly cmsCpuInfo.get_NumOfCores()
910  prevrel = "" ,
911  bypasshlt = False ,
912  runonspare = True ,
913  perfsuitedir = os.getcwd(),
914  logfile = None,
915  TimeSizeCandles = "" ,
916  IgProfCandles = "" ,
917  CallgrindCandles = "" ,
918  MemcheckCandles = "" ,
919  TimeSizePUCandles = "" ,
920  IgProfPUCandles = "" ,
921  CallgrindPUCandles = "" ,
922  MemcheckPUCandles = "" ,
923  PUInputFile = "" ,
924  userInputFile = "" ,
925  MailLogRecipients = "" ,
926  tarball = "" ):
927 
928  #Set up a variable for the FinalExitCode to be used as the sum of exit codes:
929  FinalExitCode=0
930 
931  #Set up the logfile first!
932  if not logfile == None:
933  try:
934  self.logh = open(logfile,"a")
935  except (OSError, IOError), detail:
936  self.logh.write(detail + "\n")
937  self.logh.flush()
938 
939  #Adding HEPSPEC06 score if available in /build/HEPSPEC06.score file
940  self.HEPSPEC06 = 0 #Set it to 0 by default (so it is easy to catch in the DB too)
941  try:
942  HEPSPEC06_file=open("/build/HEPSPEC06.score","r")
943  for line in HEPSPEC06_file.readlines():
944  if not line.startswith("#") and "HEPSPEC06" in line:
945  self.HEPSPEC06= line.split()[2]
946  except IOError:
947  self.logh.write("***Warning***: Could not find file /build/HEPSPEC06.score file on this machine!\n")
948  self.logh.flush()
949 
950  #Adding a copy of /proc/cpuinfo and /proc/meminfo in the working directory so it can be kept in the tarball on CASTOR:
951  localcpuinfo=os.path.join(perfsuitedir,"cpuinfo")
952  cpuinfo_exitcode=-1
953  if os.path.exists(localcpuinfo):
954  cpuinfo_exitcode=0
955  else:
956  self.logh.write("Copying /proc/cpuinfo in current working directory (%s)\n"%perfsuitedir)
957  cpuinfo_exitcode=self.runcmd("cp /proc/cpuinfo %s"%perfsuitedir)
958  localmeminfo=os.path.join(perfsuitedir,"meminfo")
959  meminfo_exitcode=-1
960  if os.path.exists(localmeminfo):
961  meminfo_exitcode=0
962  else:
963  self.logh.write("Copying /proc/meminfo in current working directory (%s)\n"%perfsuitedir)
964  meminfo_exitcode=self.runcmd("cp /proc/meminfo %s"%perfsuitedir)
965  if cpuinfo_exitcode or meminfo_exitcode:
966  self.logh.write("There was an issue copying the cpuinfo or meminfo files!\n")
967  self.logh.flush()
968 
969  try:
970  if not prevrel == "":
971  self.logh.write("Production of regression information has been requested with release directory %s\n" % prevrel)
972  if not cmsdriverOptions == "":
973  self.logh.write("Running cmsDriver.py with user defined options: %s\n" % cmsdriverOptions)
974  #Attach the full option synthax for cmsRelvalreportInput.py:
975  cmsdriverOptionsRelvalInput="--cmsdriver="+cmsdriverOptions
976  #FIXME: should import cmsRelvalreportInput.py and avoid these issues...
977  if not stepOptions == "":
978  self.logh.write("Running user defined steps only: %s\n" % stepOptions)
979  #Attach the full option synthax for cmsRelvalreportInput.py:
980  setpOptionsRelvalInput="--usersteps="+stepOptions
981  #FIXME: should import cmsRelvalreportInput.py and avoid these issues...
982  if bypasshlt:
983  #Attach the full option synthax for cmsRelvalreportInput.py:
984  bypasshltRelvalInput="--bypass-hlt"
985  #FIXME: should import cmsRelvalreportInput.py and avoid these issues...
986  self.logh.write("Current Architecture is %s\n"%self.cmssw_arch)
987  self.logh.write("Current CMSSW version is %s\n"%self.cmssw_version)
988  self.logh.write("This machine ( %s ) is assumed to have %s cores, and the suite will be run on cpu %s\n" %(self.host,cores,cpus))
989  self.logh.write("This machine's HEPSPEC06 score is: %s \n"%self.HEPSPEC06)
990  path=os.path.abspath(".")
991  self.logh.write("Performance Suite started running at %s on %s in directory %s, run by user %s\n" % (self.getDate(),self.host,path,self.user))
992  #Start the timer for the total performance suite running time:
993  TotalTime=PerfSuiteTimer(start=datetime.datetime.now())
994  #Also initialize the dictionary that will contain all the timing information:
995  global TimerInfo
996  TimerInfo={'TotalTime':{'TotalTime':TotalTime}} #Structure will be {'key':[PerfSuiteTimerInstance,...],...}
997  #Obsolete popen4-> subprocess.Popen
998  #showtags=os.popen4("showtags -r")[1].read()
999  showtags=subprocess.Popen("showtags -r",shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
1000  self.logh.write(showtags) # + "\n") No need for extra \n!
1001  self.logh.flush()
1002  #For the log:
1003  if self._verbose:
1004  self.logh.write("The performance suite results tarball will be stored in CASTOR at %s\n" % self._CASTOR_DIR)
1005  self.logh.write("%s TimeSize events\n" % TimeSizeEvents)
1006  self.logh.write("%s IgProf events\n" % IgProfEvents)
1007  self.logh.write("%s Callgrind events\n" % CallgrindEvents)
1008  self.logh.write("%s Memcheck events\n" % MemcheckEvents)
1009  self.logh.write("%s cmsScimark benchmarks before starting the tests\n" % cmsScimark)
1010  self.logh.write("%s cmsScimarkLarge benchmarks before starting the tests\n" % cmsScimarkLarge)
1011  self.logh.flush()
1012  #Actual script actions!
1013  #Will have to fix the issue with the matplotlib pie-charts:
1014  #Used to source /afs/cern.ch/user/d/dpiparo/w0/perfreport2.1installation/share/perfreport/init_matplotlib.sh
1015  #Need an alternative in the release
1016 
1017  #Code for the architecture benchmarking use-case
1018  if len(cpus) > 1:
1019  for cpu in cpus:
1020  cpupath = os.path.join(perfsuitedir,"cpu_%s" % cpu)
1021  if not os.path.exists(cpupath):
1022  os.mkdir(cpupath)
1023 
1024  self.Commands = {}
1025  AllScripts = self.Scripts + self.AuxiliaryScripts
1026 
1027  for cpu in range(cmsCpuInfo.get_NumOfCores()): #FIXME use the actual number of cores of the machine here!
1028  self.Commands[cpu] = []
1029 
1030  #Information for the log:
1031  self.logh.write("Full path of all the scripts used in this run of the Performance Suite:\n")
1032  for script in AllScripts:
1033  which="which " + script
1034 
1035  #Logging the actual version of cmsDriver.py, cmsRelvalreport.py, cmsSimPyRelVal.pl
1036  #Obsolete popen4-> subprocess.Popen
1037  #whichstdout=os.popen4(which)[1].read()
1038  whichstdout=subprocess.Popen(which,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
1039  self.logh.write(whichstdout) # + "\n") No need of the extra \n!
1040  if script in self.Scripts:
1041  for cpu in range(cmsCpuInfo.get_NumOfCores()):#FIXME use the actual number of cores of the machine here!
1042  command="taskset -c %s %s" % (cpu,script)
1043  self.Commands[cpu].append(command)
1044 
1045  #First submit the cmsScimark benchmarks on the unused cores:
1046  scimark = ""
1047  scimarklarge = ""
1048  if not (self._unittest or self._noexec):
1049  if (len(cpu_list) != cores):
1050  for core in range(cores):
1051  if (not core in cpus) and runonspare:
1052  self.logh.write("Submitting cmsScimarkLaunch.csh to run on core cpu "+str(core) + "\n")
1053  subcmd = "cd %s ; cmsScimarkLaunch.csh %s" % (perfsuitedir, str(core))
1054  command="taskset -c %s sh -c \"%s\" &" % (str(core), subcmd)
1055  self.logh.write(command + "\n")
1056 
1057  #cmsScimarkLaunch.csh is an infinite loop to spawn cmsScimark2 on the other
1058  #cpus so it makes no sense to try reading its stdout/err
1059  #Obsolete popen4-> subprocess.Popen
1060  #os.popen4(command)
1061  subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
1062 
1063  self.logh.flush()
1064 
1065  #Don't do benchmarking if in debug mode... saves time
1066  benching = not self._debug
1067  ##FIXME:
1068  #We may want to introduce a switch here or agree on a different default (currently 10 cmsScimark and 10 cmsScimarkLarge)
1069  if benching and not (self._unittest or self._noexec):
1070  #Submit the cmsScimark benchmarks on the cpu where the suite will be run:
1071  for cpu in cpus:
1072  scimark = open(os.path.join(perfsuitedir,"cmsScimark2.log") ,"w")
1073  scimarklarge = open(os.path.join(perfsuitedir,"cmsScimark2_large.log"),"w")
1074  if cmsScimark > 0:
1075  self.logh.write("Starting with %s cmsScimark on cpu%s\n" % (cmsScimark,cpu))
1076  cmsScimarkInitialTime=PerfSuiteTimer(start=datetime.datetime.now()) #Create the cmsScimark PerfSuiteTimer
1077  TimerInfo.update({'cmsScimarkTime':{'cmsScimarkInitial':cmsScimarkInitialTime}}) #Add the cmsScimarkInitialTime information to the general TimerInfo dictionary
1078  self.benchmarks(cpu,perfsuitedir,scimark.name,cmsScimark)
1079  cmsScimarkInitialTime.set_end(datetime.datetime.now()) #Stop the cmsScimark initial timer
1080 
1081  if cmsScimarkLarge > 0:
1082  self.logh.write("Following with %s cmsScimarkLarge on cpu%s\n" % (cmsScimarkLarge,cpu))
1083  cmsScimarkLargeInitialTime=PerfSuiteTimer(start=datetime.datetime.now()) #Create the cmsScimarkLarge PerfSuiteTimer
1084  TimerInfo['cmsScimarkTime'].update({'cmsScimarkLargeInitial':cmsScimarkLargeInitialTime}) #Add the cmsScimarkLargeInitialTime information to the general TimerInfo dictionary
1085  self.benchmarks(cpu,perfsuitedir,scimarklarge.name,cmsScimarkLarge, large=True)
1086  cmsScimarkLargeInitialTime.set_end(datetime.datetime.now()) #Stop the cmsScimarkLarge Initial timer
1087  self.logh.flush()
1088  #Handling the Pile up input file here:
1089  if (TimeSizePUCandles or IgProfPUCandles or CallgrindPUCandles or MemcheckPUCandles) and not ("FASTSIM" in stepOptions):
1090  #Note the FASTSIM exclusion... since there is no need to copy the file for FASTSIM.
1091  PUInputName=os.path.join(perfsuitedir,"INPUT_PILEUP_EVENTS.root")
1092  if PUInputFile:
1093  #Define the actual command to copy the file locally:
1094  #Allow the file to be mounted locally (or accessible via AFS)
1095  copycmd="cp"
1096  #Allow the file to be on CASTOR (taking a full CASTOR path)
1097  if '/store/relval/' in PUInputFile:
1098  #Switching from CASTOR to EOS, i.e. from rfcp to xrdcp
1099  copycmd="xrdcp"
1100  #Accept plain LFNs from DBS for RelVal CASTOR files:
1101  #Minor fix to allow the case of user using the full path /castor/cern.ch/cms...
1102  if PUInputFile.startswith('/store/relval/'):
1103  #Switching to EOS from CASTOR:
1104  #PUInputFile="/castor/cern.ch/cms"+PUInputFile
1105  PUInputFile="root://eoscms//eos/cms"+PUInputFile
1106  #Copy the file locally
1107  self.logh.write("Copying the file %s locally to %s\n"%(PUInputFile,PUInputName))
1108  self.logh.flush()
1109  GetPUInput=subprocess.Popen("%s %s %s"%(copycmd,PUInputFile,PUInputName), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1110  GetPUInputExitCode=GetPUInput.wait()
1111  #Allow even the potential copy of a local file (even one already named INPUT_PILEUP_EVENTS.root!)
1112  if GetPUInputExitCode:
1113  self.logh.write("The copying of the pile-up input file returned a non-zero exit code: %s \nThis is the stdout+stderr if the command:\n%s\n"%(GetPUInputExitCode,GetPUInput.stdout))
1114  #Ultimately accept the case of the file being already there and not being specified in the --PUInputFile option
1115  if not os.path.exists(PUInputName):
1116  self.logh.write("The necessary INPUT_PILEUP_EVENTS.root file was not found in the working directory %s\nExiting now!"%perfsuitedir)
1117  self.logh.flush()
1118  sys.exit(1)
1119  else:
1120  #Set up here the DIGI PILE UP options
1121  self.printFlush("Some PILE UP tests will be run!")
1122  #Actually setting them earlier... when handling options... May not need this else after all... or just as a log entry.
1123  self.printFlush("cmsdriverPUOptions is %s"%cmsdriverPUOptions)
1124  pass
1125 
1126  #TimeSize tests:
1127  if TimeSizeEvents > 0:
1128  TimeSizeTime=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
1129  TimerInfo.update({'TimeSize':{'TotalTime':TimeSizeTime}}) #Add the TimeSize timer to the dictionary
1130  if TimeSizeCandles:
1131  self.logh.write("Launching the TimeSize tests (TimingReport, TimeReport, SimpleMemoryCheck, EdmSize) with %s events each\n" % TimeSizeEvents)
1132  NoPileUpTime=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
1133  TimerInfo['TimeSize'].update({'NoPileUpTime':NoPileUpTime}) #Add the TimeSize No Pile Up tests timer to the list
1134  self.printDate()
1135  self.logh.flush()
1136  ReportExit=self.simpleGenReport(cpus,perfsuitedir,TimeSizeEvents,TimeSizeCandles,cmsdriverOptions,stepOptions,"TimeSize",profilers,bypasshlt,userInputFile)
1137  FinalExitCode=FinalExitCode+ReportExit
1138  #Adding a time stamp here to parse for performance suite running time data
1139  self.printFlush("Regular TimeSize tests were finished at %s"%(self.getDate()))
1140  NoPileUpTime.set_end(datetime.datetime.now()) #Stop TimeSize timer
1141 
1142  #Launch eventual Digi Pile Up TimeSize too:
1143  if TimeSizePUCandles:
1144  self.logh.write("Launching the PILE UP TimeSize tests (TimingReport, TimeReport, SimpleMemoryCheck, EdmSize) with %s events each\n" % TimeSizeEvents)
1145  PileUpTime=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
1146  TimerInfo['TimeSize'].update({'PileUpTime':PileUpTime}) #Add the TimeSize Pile Up tests timer to the list
1147  self.printDate()
1148  self.logh.flush()
1149  ReportExit=self.simpleGenReport(cpus,perfsuitedir,TimeSizeEvents,TimeSizePUCandles,cmsdriverPUOptions,stepOptions,"TimeSize",profilers,bypasshlt,userInputFile)
1150  FinalExitCode=FinalExitCode+ReportExit
1151  #Adding a time stamp here to parse for performance suite running time data
1152  self.printFlush("Pileup TimeSize tests were finished at %s"%(self.getDate()))
1153  PileUpTime.set_end(datetime.datetime.now()) #Stop TimeSize timer
1154 
1155  #Check for issue with
1156  if not (TimeSizeCandles or TimeSizePUCandles):
1157  self.printFlush("A number of events (%s) for TimeSize tests was selected, but no candle for regular or pileup tests was selected!"%(TimeSizeEvents))
1158  #Adding a time stamp here to parse for performance suite running time data
1159  self.printFlush("All TimeSize tests were finished at %s"%(self.getDate()))
1160  TimeSizeTime.set_end(datetime.datetime.now()) #Stop TimeSize timer
1161 
1162  #Stopping all cmsScimark jobs and analysing automatically the logfiles
1163  #No need to waste CPU while the load does not affect Valgrind measurements!
1164  if not (self._unittest or self._noexec):
1165  self.logh.write("Stopping all cmsScimark jobs now\n")
1166  subcmd = "cd %s ; %s" % (perfsuitedir,self.AuxiliaryScripts[2])
1167  stopcmd = "sh -c \"%s\"" % subcmd
1168  self.printFlush(stopcmd)
1169  #os.popen(stopcmd)
1170  #Obsolete popen4-> subprocess.Popen
1171  #self.printFlush(os.popen4(stopcmd)[1].read())
1172  self.printFlush(subprocess.Popen(stopcmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read())
1173 
1174  #From here on we can use all available cores to speed up the performance suite remaining tests:
1175  if cores==0: #When specifying the cpu to run the suite on, one has to set cores to 0 to avoid threading of PerfSuite itself...
1176  #So we need to catch this case for the IB tests case where we assign the test to a specific cpu.
1177  AvailableCores=cpus
1178  elif len(cpu_list) == cores: # For the new relval case, when running all the tests on one machine,
1179  # specifying the same number of cores and cpus (like: --cores 3, --cpu 3,4,5)
1180  AvailableCores=cpus
1181  else:
1182  AvailableCores=range(cores)
1183 
1184  #Initialize a list that will contain all the simpleGenReport keyword arguments (1 dictionary per test):
1185  TestsToDo=[]
1186  #IgProf tests:
1187  if IgProfEvents > 0:
1188  if IgProfCandles:
1189  self.printFlush("Preparing IgProf tests")
1190  #Special case for IgProf: user could pick with the option --profilers to run only IgProf perf or Mem (or Mem_Total alone etc)
1191  #So in general we want to be able to split the perf and mem tests...
1192  #For the case of --profiler option we will run only 1 test (i.e. it will get one core slot until it is done with whatever profiling choosen)
1193  if profilers:
1194  self.printFlush("Special profiler option for IgProf was indicated by the user: %s"%profilers)
1195  #Prepare the simpleGenReport arguments for this test:
1196  IgProfProfilerArgs={
1197  'perfdir':perfsuitedir,
1198  'NumEvents':IgProfEvents,
1199  'candles':IgProfCandles,
1200  'cmsdriverOptions':cmsdriverOptions,
1201  'stepOptions':stepOptions,
1202  'Name':"IgProf",
1203  'profilers':profilers,
1204  'bypasshlt':bypasshlt,
1205  'userInputRootFiles':userInputFile
1206  }
1207  #Append the test to the TestsToDo list:
1208  TestsToDo.append(IgProfProfilerArgs)
1209  self.printFlush("Appended IgProf test with profiler option %s to the TestsToDo list"%profilers)
1210  #For the default case (4,5,6,7) we split the tests into 2 jobs since they naturally are 2 cmsRun jobs and for machines with many cores this will
1211  #make the performance suite run faster.
1212  else:
1213  self.printFlush("Splitting the IgProf tests into Perf and Mem to parallelize the cmsRun execution as much as possible:")
1214  ##PERF##
1215  #Prepare the simpleGenReport arguments for this test:
1216  IgProfPerfArgs={
1217  'perfdir':perfsuitedir,
1218  'NumEvents':IgProfEvents,
1219  'candles':IgProfCandles,
1220  'cmsdriverOptions':cmsdriverOptions,
1221  'stepOptions':stepOptions,
1222  'Name':"IgProf_Perf",
1223  'profilers':profilers,
1224  'bypasshlt':bypasshlt,
1225  'userInputRootFiles':userInputFile
1226  }
1227  #Append the test to the TestsToDo list:
1228  TestsToDo.append(IgProfPerfArgs)
1229  self.printFlush("Appended IgProf PERF test to the TestsToDo list")
1230  ##MEM##
1231  #Prepare the simpleGenReport arguments for this test:
1232  IgProfMemArgs={
1233  'perfdir':perfsuitedir,
1234  'NumEvents':IgProfEvents,
1235  'candles':IgProfCandles,
1236  'cmsdriverOptions':cmsdriverOptions,
1237  'stepOptions':stepOptions,
1238  'Name':"IgProf_Mem",
1239  'profilers':profilers,
1240  'bypasshlt':bypasshlt,
1241  'userInputRootFiles':userInputFile
1242  }
1243  #Append the test to the TestsToDo list:
1244  TestsToDo.append(IgProfMemArgs)
1245  self.printFlush("Appended IgProf MEM test to the TestsToDo list")
1246  #The following will be handled in the while loop that handles the starting of the threads:
1247  #ReportExit=self.simpleGenReport(cpus,perfsuitedir,IgProfEvents,IgProfCandles,cmsdriverOptions,stepOptions,"IgProf",profilers,bypasshlt,userInputFile)
1248  #FinalExitCode=FinalExitCode+ReportExit
1249  #Launch eventual Digi Pile Up IgProf too:
1250  if IgProfPUCandles:
1251  self.printFlush("Preparing IgProf PileUp tests")
1252  #Special case for IgProf: user could pick with the option --profilers to run only IgProf perf or Mem (or Mem_Total alone etc)
1253  #So in general we want to be able to split the perf and mem tests...
1254  #For the case of --profiler option we will run only 1 test (i.e. it will get one core slot until it is done with whatever profiling choosen)
1255  if profilers:
1256  self.printFlush("Special profiler option for IgProf was indicated by the user: %s"%profilers)
1257  #Prepare the simpleGenReport arguments for this test:
1258  IgProfProfilerPUArgs={
1259  'perfdir':perfsuitedir,
1260  'NumEvents':IgProfEvents,
1261  'candles':IgProfPUCandles,
1262  'cmsdriverOptions':cmsdriverPUOptions,
1263  'stepOptions':stepOptions,
1264  'Name':"IgProf",
1265  'profilers':profilers,
1266  'bypasshlt':bypasshlt,
1267  'userInputRootFiles':userInputFile
1268  }
1269  #Append the test to the TestsToDo list:
1270  TestsToDo.append(IgProfProfilerPUArgs)
1271  self.printFlush("Appended IgProf PileUp test with profiler option %s to the TestsToDo list"%profilers)
1272  else:
1273  self.printFlush("Splitting the IgProf tests into Perf and Mem to parallelize the cmsRun execution as much as possible:")
1274  ##PERF##
1275  #Prepare the simpleGenReport arguments for this test:
1276  IgProfPerfPUArgs={
1277  'perfdir':perfsuitedir,
1278  'NumEvents':IgProfEvents,
1279  'candles':IgProfPUCandles,
1280  'cmsdriverOptions':cmsdriverPUOptions,
1281  'stepOptions':stepOptions,
1282  'Name':"IgProf_Perf",
1283  'profilers':profilers,
1284  'bypasshlt':bypasshlt,
1285  'userInputRootFiles':userInputFile
1286  }
1287  #Append the test to the TestsToDo list:
1288  TestsToDo.append(IgProfPerfPUArgs)
1289  self.printFlush("Appended IgProf MEM PileUp test to the TestsToDo list")
1290  ##MEM##
1291  #Prepare the simpleGenReport arguments for this test:
1292  IgProfMemPUArgs={
1293  'perfdir':perfsuitedir,
1294  'NumEvents':IgProfEvents,
1295  'candles':IgProfPUCandles,
1296  'cmsdriverOptions':cmsdriverPUOptions,
1297  'stepOptions':stepOptions,
1298  'Name':"IgProf_Mem",
1299  'profilers':profilers,
1300  'bypasshlt':bypasshlt,
1301  'userInputRootFiles':userInputFile
1302  }
1303  #Append the test to the TestsToDo list:
1304  TestsToDo.append(IgProfMemPUArgs)
1305  self.printFlush("Appended IgProf MEM PileUp test to the TestsToDo list")
1306  if not (IgProfCandles or IgProfPUCandles):
1307  self.printFlush("A number of events (%s) for IgProf tests was selected, but no candle for regular or pileup tests was selected!"%(IgProfEvents))
1308 
1309 
1310  #Valgrind tests:
1311  if CallgrindEvents > 0:
1312  if CallgrindCandles:
1313  self.printFlush("Preparing Callgrind tests")
1314  CallgrindArgs={
1315  'perfdir':perfsuitedir,
1316  'NumEvents':CallgrindEvents,
1317  'candles':CallgrindCandles,
1318  'cmsdriverOptions':cmsdriverOptions,
1319  'stepOptions':stepOptions,
1320  'Name':"Callgrind",
1321  'profilers':profilers,
1322  'bypasshlt':bypasshlt,
1323  'userInputRootFiles':userInputFile
1324  }
1325  #Append the test to the TestsToDo list:
1326  TestsToDo.append(CallgrindArgs)
1327  self.printFlush("Appended Callgrind test to the TestsToDo list")
1328  #Launch eventual Digi Pile Up Callgrind too:
1329  if CallgrindPUCandles:
1330  self.printFlush("Preparing Callgrind PileUp tests")
1331  CallgrindPUArgs={
1332  'perfdir':perfsuitedir,
1333  'NumEvents':CallgrindEvents,
1334  'candles':CallgrindPUCandles,
1335  'cmsdriverOptions':cmsdriverPUOptions,
1336  'stepOptions':stepOptions,
1337  'Name':"Callgrind",
1338  'profilers':profilers,
1339  'bypasshlt':bypasshlt,
1340  'userInputRootFiles':userInputFile
1341  }
1342  #Append the test to the TestsToDo list:
1343  TestsToDo.append(CallgrindPUArgs)
1344  self.printFlush("Appended Callgrind PileUp test to the TestsToDo list")
1345  if not (CallgrindCandles or CallgrindPUCandles):
1346  self.printFlush("A number of events (%s) for Callgrind tests was selected, but no candle for regular or pileup tests was selected!"%(CallgrindEvents))
1347 
1348  if MemcheckEvents > 0:
1349  if MemcheckCandles:
1350  self.printFlush("Preparing Memcheck tests")
1351  MemcheckArgs={
1352  'perfdir':perfsuitedir,
1353  'NumEvents':MemcheckEvents,
1354  'candles':MemcheckCandles,
1355  'cmsdriverOptions':cmsdriverOptions,
1356  'stepOptions':stepOptions,
1357  'Name':"Memcheck",
1358  'profilers':profilers,
1359  'bypasshlt':bypasshlt,
1360  'userInputRootFiles':userInputFile
1361  }
1362  #Append the test to the TestsToDo list:
1363  TestsToDo.append(MemcheckArgs)
1364  self.printFlush("Appended Memcheck test to the TestsToDo list")
1365  #Launch eventual Digi Pile Up Memcheck too:
1366  if MemcheckPUCandles:
1367  self.printFlush("Preparing Memcheck PileUp tests")
1368  MemcheckPUArgs={
1369  'perfdir':perfsuitedir,
1370  'NumEvents':MemcheckEvents,
1371  'candles':MemcheckPUCandles,
1372  'cmsdriverOptions':cmsdriverPUOptions,
1373  'stepOptions':stepOptions,
1374  'Name':"Memcheck",
1375  'profilers':profilers,
1376  'bypasshlt':bypasshlt,
1377  'userInputRootFiles':userInputFile
1378  }
1379  #Append the test to the TestsToDo list:
1380  TestsToDo.append(MemcheckPUArgs)
1381  self.printFlush("Appended Memcheck PileUp test to the TestsToDo list")
1382  if not (MemcheckCandles or MemcheckPUCandles):
1383  self.printFlush("A number of events (%s) for Memcheck tests was selected, but no candle for regular or pileup tests was selected!"%(MemcheckEvents))
1384 
1385  #Here if there are any IgProf, Callgrind or MemcheckEvents to be run,
1386  #run the infinite loop that submits the PerfTest() threads on the available cores:
1387  if IgProfEvents or CallgrindEvents or MemcheckEvents:
1388  #FIXME:We should consider what behavior makes most sense in case we use the --cores option at this time only the cores=0 care is considered...
1389  self.printFlush("Threading all remaining tests on all %s available cores!"%len(AvailableCores))
1390  self.printDate()
1391  self.logh.flush()
1392  #Save the original AvailableCores list to use it as a test to break the infinite loop:
1393  #While in the regular RelVal use-case it makes sense to use the actual number of cores of the machines, in
1394  #the IB case the AvailableCores will always consist of only 1 single core..
1395  OriginalAvailableCores=list(AvailableCores) #Tricky list copy bug! without the list() OriginalAvalaibleCores would point to AvailableCores!
1396  #Print this out in the log for debugging reasons
1397  self.printFlush("Original available cores list: %s"%AvailableCores)
1398 
1399  #Create a dictionary to keep track of running threads on the various cores:
1400  activePerfTestThreads={}
1401  #Flag for waiting messages:
1402  Waiting=False
1403  while 1:
1404  #Check if there are tests to run:
1405  if TestsToDo:
1406  #Using the Waiting flag to avoid writing this message every 5 seconds in the case
1407  #of having more tests to do than available cores...
1408  if not Waiting:
1409  self.printFlush("Currently %s tests are scheduled to be run:"%len(TestsToDo))
1410  self.printFlush(TestsToDo)
1411  #Check the available cores:
1412  if AvailableCores:
1413  #Set waiting flag to False since we'll be doing something
1414  Waiting=False
1415  self.printFlush("There is/are %s core(s) available"%len(AvailableCores))
1416  cpu=AvailableCores.pop()
1417  self.printFlush("Let's use cpu %s"%cpu)
1418  simpleGenReportArgs=TestsToDo.pop()
1419  self.printFlush("Let's submit %s test on core %s"%(simpleGenReportArgs['Name'],cpu))
1420  #Adding a Total timer for each of the threaded tests:
1421  if simpleGenReportArgs['Name'] not in TimerInfo.keys():
1422  #if 'TotalTime' not in TimerInfo[simpleGenReportArgs['Name']].keys():
1423  self.PerfTestTotalTimer=PerfSuiteTimer(start=datetime.datetime.now()) #Start the TimeSize timer
1424  TimerInfo.update({simpleGenReportArgs['Name']:{'TotalTime':self.PerfTestTotalTimer}}) #Add the TimeSize timer to the dictionary
1425  threadToDo=self.simpleGenReportThread(cpu,self,**simpleGenReportArgs) #Need to send self too, so that the thread has access to the PerfSuite.simpleGenReport() function
1426  self.printFlush("Starting thread %s"%threadToDo)
1427  ReportExitCode=threadToDo.start()
1428  self.printFlush("Adding thread %s to the list of active threads"%threadToDo)
1429  activePerfTestThreads[cpu]=threadToDo
1430  #If there is no available core, pass, there will be some checking of activeThreads, a little sleep and then another check.
1431  else:
1432  pass
1433  #Test activePerfThreads:
1434  activeTestNames=[]
1435  activeTestNamesPU=[]
1436  for cpu in activePerfTestThreads.keys():
1437  if activePerfTestThreads[cpu].isAlive():
1438  #print "%% cpu %s activerPerfTestThreads[cpu] %s activePerfTestThreads[cpu].simpleGenReportArgs['cmsdriverOptions'] %s"%(cpu,activePerfTestThreads[cpu],activePerfTestThreads[cpu].simpleGenReportArgs['cmsdriverOptions'])
1439  if "--pileup" in activePerfTestThreads[cpu].simpleGenReportArgs['cmsdriverOptions']:
1440  activeTestNamesPU.append(activePerfTestThreads[cpu].simpleGenReportArgs['Name'])
1441  else:
1442  activeTestNames.append(activePerfTestThreads[cpu].simpleGenReportArgs['Name'])
1443  pass
1444  elif cpu not in AvailableCores:
1445  #Set waiting flag to False since we'll be doing something
1446  Waiting=False
1447  self.printFlush(time.ctime())
1448  self.printFlush("%s test, in thread %s is done running on core %s"%(activePerfTestThreads[cpu].simpleGenReportArgs['Name'],activePerfTestThreads[cpu],cpu) )
1449  self.printFlush("About to append cpu %s to AvailableCores list"%cpu)
1450  AvailableCores.append(cpu)
1451  #Eliminate from activeTestNames lists:
1452  #print activeTestNames
1453  #print activeTestNamesPU
1454  #print activePerfTestThreads[cpu].simpleGenReportArgs['Name']
1455  if "--pileup" in activePerfTestThreads[cpu].simpleGenReportArgs['cmsdriverOptions']:
1456  try:
1457  activeTestNamesPU.remove(activePerfTestThreads[cpu].simpleGenReportArgs['Name'])
1458  except:
1459  pass
1460  else:
1461  try:
1462  activeTestNames.remove(activePerfTestThreads[cpu].simpleGenReportArgs['Name'])
1463  except:
1464  pass
1465  #Eliminate also from activePErfTestThreads dictionary:
1466  activePerfTestThreads.pop(cpu)
1467  #FIXME:
1468  #Delicate check to stop the timer on the individual threaded test!
1469  #Need to thik about it still...
1470  #FIXME:
1471  #Delicate check to stop the timers on the threaded tests:
1472  #Check activePerfTestThreads dictionary for "Name" if any name is missing, the total can be stopped for that name.
1473  #self.PerfTestTotalTimer
1474  for TestName in ["IgProf_Perf","IgProf_Mem","Memcheck","Valgrind"]:
1475  if (TestName not in activeTestNames) and (TestName not in activeTestNamesPU) :
1476  try:
1477  TimerInfo[TestName]['TotalTime'].set_end(datetime.datetime.now())
1478  except:
1479  #print "No %s test was running"%TestName
1480  pass
1481  #Buggy if... it seems we don't wait for the running thread to be finished...
1482  #We should request:
1483  #-All OriginalAvailableCores should be actually available.
1484  if not AvailableCores==[] and (set(AvailableCores)==set(range(cmsCpuInfo.get_NumOfCores())) or set(AvailableCores)==set(OriginalAvailableCores)) and not TestsToDo:
1485  self.printFlush("PHEW! We're done... all TestsToDo are done... at %s "%(self.getDate()))
1486  #Debug printouts:
1487  #print "AvailableCores",AvailableCores
1488  #print "set(AvailableCores)",set(AvailableCores)
1489  #print "set(range(cmsCpuInfo.get_NumOfCores())",set(range(cmsCpuInfo.get_NumOfCores()))
1490  #print "OriginalAvailableCores",OriginalAvailableCores
1491  #print "set(OriginalAvailableCores)",set(OriginalAvailableCores)
1492  #print "TestsToDo",TestsToDo
1493  break
1494  else:
1495  #Putting the sleep statement first to avoid writing Waiting... before the output of the started thread reaches the log...
1496  time.sleep(5)
1497  #Use Waiting flag to writing 1 waiting message while waiting and avoid having 1 message every 5 seconds...
1498  if not Waiting:
1499  self.printFlush(time.ctime())
1500  self.printFlush("Waiting for tests to be done...")
1501  sys.stdout.flush()
1502  Waiting=True
1503  #End of the if for IgProf, Callgrind, Memcheck tests
1504 
1505  if benching and not (self._unittest or self._noexec):
1506  #Ending the performance suite with the cmsScimark benchmarks again:
1507  for cpu in cpus:
1508  if cmsScimark > 0:
1509  self.logh.write("Ending with %s cmsScimark on cpu%s\n" % (cmsScimark,cpu))
1510  cmsScimarkFinalTime=PerfSuiteTimer(start=datetime.datetime.now()) #Create the cmsScimark PerfSuiteTimer
1511  TimerInfo['cmsScimarkTime'].update({'cmsScimarkFinal':cmsScimarkFinalTime}) #Add the cmsScimarkFinalTime information to the general TimerInfo dictionary
1512 
1513  self.benchmarks(cpu,perfsuitedir,scimark.name,cmsScimark)
1514  cmsScimarkFinalTime.set_end(datetime.datetime.now()) #Stop the cmsScimarkLarge Initial timer
1515  if cmsScimarkLarge > 0:
1516  self.logh.write("Following with %s cmsScimarkLarge on cpu%s\n" % (cmsScimarkLarge,cpu))
1517  cmsScimarkLargeFinalTime=PerfSuiteTimer(start=datetime.datetime.now()) #Create the cmsScimarkLargePerfSuiteTimer
1518  TimerInfo['cmsScimarkTime'].update({'cmsScimarkLargeFinal':cmsScimarkLargeFinalTime}) #Add the cmsScimarkLargeFinalTime information to the general TimerInfo dictionary
1519  self.benchmarks(cpu,perfsuitedir,scimarklarge.name,cmsScimarkLarge,large=True)
1520  cmsScimarkLargeFinalTime.set_end(datetime.datetime.now()) #Stop the cmsScimarkLarge Initial timer
1521 
1522  if prevrel:
1523  self.logh.write("Running the regression analysis with respect to %s\n"%getVerFromLog(prevrel))
1524  self.logh.write(time.ctime(time.time()))
1525  self.logh.flush()
1526 
1527  crr.regressReports(prevrel,os.path.abspath(perfsuitedir),oldRelName = getVerFromLog(prevrel),newRelName=self.cmssw_version)
1528 
1529  #Create a tarball of the work directory
1530  if tarball:
1531  tarballTime=PerfSuiteTimer(start=datetime.datetime.now()) #Create the tarball PerfSuiteTimer
1532  TimerInfo.update({'tarballTime':{'TotalTime':tarballTime}})
1533  # Adding the str(stepOptions to distinguish the tarballs for 1 release
1534  # (GEN->DIGI, L1->RECO will be run in parallel)
1535 
1536  # Cleaning the stepOptions from the --usersteps=:
1537  if "=" in str(stepOptions):
1538  fileStepOption=str(stepOptions).split("=")[1]
1539  else:
1540  fileStepOption=str(stepOptions)
1541  if fileStepOption=="":
1542  fileStepOption="UnknownStep"
1543  # Add the working directory used to avoid overwriting castor files (also put a check...)
1544  fileWorkingDir=os.path.basename(perfsuitedir)
1545 
1546  # Also add the --conditions and --eventcontent options used in the --cmsdriver options since it
1547  # is possible that the same tests will be run with different conditions and/or event content:
1548  # Parse it out of --cmsdriver option:
1549  fileEventContentOption="UnknownEventContent"
1550  fileConditionsOption="UnknownConditions"
1551  for token in cmsdriverOptions.split("--"):
1552  if token!='' and 'cmsdriver' not in token:
1553  if "=" in token:
1554  fileOption=token.split("=")[0]
1555  fileOptionValue=token.split("=")[1].strip("'").strip('"')
1556  else:
1557  fileOption=token.split()[0]
1558  fileOptionValue=token.split()[1].strip("'").strip('"')
1559  if "eventcontent" or "conditions" in fileOption:
1560  if "eventcontent" in fileOption:
1561  fileEventContentOption=fileOptionValue
1562  elif "conditions" in fileOption:
1563  # check if we are using the autoCond style of flexible conditions
1564  # if so, expand the condition here so that the file names contain the real conditions
1565  if "auto:" in fileOptionValue:
1566  from Configuration.AlCa.autoCond import autoCond
1567  fileConditionsOption = autoCond[ fileOptionValue.split(':')[1] ]
1568  else:
1569  # "old style" conditions, hardcoded values ...
1570  # FIXME:
1571  # Should put at least the convention in cmsPerfCommons to know how to parse it...
1572  # Potential weak point if the conditions tag convention changes...
1573  if "," in fileOptionValue: #Since 330, conditions don't have FrontierConditions_GlobalTag, in front of them anymore...
1574  fileConditionsOption=fileOptionValue.split("::")[0].split(",")[1] #"Backward" compatibility
1575  else:
1576  fileConditionsOption=fileOptionValue.split("::")[0]
1577  else: # empty token
1578  #print "Print this is the token: %s"%token
1579  pass
1580 
1581  #self.printFlush("Conditions label to add to the tarball name is %s"%fileConditionsOption)
1582  #self.printFlush("Eventcontent label to add to the tarball name is %s"%fileEventContentOption)
1583  #FIXME:
1584  #Could add the allowed event contents in the cmsPerfCommons.py file and use those to match in the command line options... This assumes maintenance of cmsPerfCommons.py
1585 
1586 
1587  #Create a tarball with just the logfiles
1588  subprocess.Popen("ls -R | grep .root > rootFiles",shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
1589  LogFile = "%s_%s_%s_%s_%s_%s_%s_%s_log.tgz" % (self.cmssw_arch, self.cmssw_version, fileStepOption, fileConditionsOption, fileEventContentOption.split()[0], fileWorkingDir, self.host, self.user)
1590  AbsTarFileLOG = os.path.join(perfsuitedir,LogFile)
1591  tarcmd = "tar zcfX %s %s %s" %(AbsTarFileLOG, "rootFiles", os.path.join(perfsuitedir,"*"))
1592  self.printFlush("Creating a tarball for the logfiles")
1593  self.printFlush(tarcmd)
1594  self.printFlush(subprocess.Popen(tarcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read())
1595  self.printFlush(subprocess.Popen("rm rootFiles",shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read())
1596 
1597  fullcastorpathlog=os.path.join(castordir,LogFile)
1598 
1599 
1600  #Create the tarball with the contents of the directory + md5 checksum
1601  TarFile = "%s_%s_%s_%s_%s_%s_%s_%s.tgz" % (self.cmssw_arch, self.cmssw_version, fileStepOption, fileConditionsOption, fileEventContentOption.split()[0], fileWorkingDir, self.host, self.user)
1602  AbsTarFile = os.path.join(perfsuitedir,TarFile)
1603  tarcmd = "tar -zcf %s %s" %(AbsTarFile, os.path.join(perfsuitedir,"*"))
1604  md5cmd = "md5sum %s" %(AbsTarFile)
1605  self.printFlush("Creating a tarball with the content of the directory")
1606  self.printFlush(tarcmd)
1607  self.printFlush(md5cmd)
1608  #FIXME:
1609  #Anything that will be logged after the tar command below will not enter the cmsPerfSuite.log in the tarball (by definition)...
1610  #To remain backward compatible the harvesting script needs to be based on the command above to identify the tarball location.
1611  #Obsolete popen4-> subprocess.Popen
1612  #self.printFlush(os.popen3(tarcmd)[2].read()) #Using popen3 to get only stderr we don't want the whole stdout of tar!
1613  self.printFlush(subprocess.Popen(tarcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read())
1614  md5sum = subprocess.Popen(md5cmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.read().split()[0]
1615  self.printFlush("The md5 checksum of the tarball: %s" %(md5sum))
1616  AbsTarFileMD5 = AbsTarFile + ".md5"
1617  md5filecmd = "echo %s > %s" % (md5sum, AbsTarFileMD5)
1618  self.printFlush(subprocess.Popen(md5filecmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read())
1619 
1620  #Archive it on CASTOR
1621  #Before archiving check if it already exist if it does print a message, but do not overwrite, so do not delete it from local dir:
1622  fullcastorpathfile=os.path.join(castordir,TarFile)
1623  fullcastorpathmd5=os.path.join(castordir,TarFile + ".md5")
1624 
1625  checkcastor="nsls %s" % fullcastorpathfile
1626  #Obsolete os.popen-> subprocess.Popen
1627  #checkcastorout=os.popen3(checkcastor)[1].read()
1628  checkcastorout=subprocess.Popen(checkcastor,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.read()
1629  if checkcastorout.rstrip()==fullcastorpathfile:
1630  castorcmdstderr="File %s is already on CASTOR! Will NOT OVERWRITE!!!"%fullcastorpathfile
1631  else:
1632  #Switching from CASTOR TO EOS, i.e. rfcp to xrdcp!
1633  #Not YET!!!
1634  #FIXME! Migrate to EOS eventually, taking into account implications for PerfDB logs linking!
1635  castorcmd="rfcp %s %s" % (AbsTarFile,fullcastorpathfile)
1636  castormd5cmd="rfcp %s %s" % (AbsTarFileMD5,fullcastorpathmd5)
1637  castorlogcmd="rfcp %s %s" % (AbsTarFileLOG,fullcastorpathlog)
1638  self.printFlush(castorcmd)
1639  self.printFlush(castormd5cmd)
1640  self.printFlush(castorlogcmd)
1641  #Obsolete os.popen-> subprocess.Popen
1642  #castorcmdstderr=os.popen3(castorcmd)[2].read()
1643  castorcmdstderr=subprocess.Popen(castorcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
1644  subprocess.Popen(castormd5cmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
1645  subprocess.Popen(castorlogcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stderr.read()
1646  #Checking the stderr of the rfcp command to copy the tarball (.tgz) on CASTOR:
1647  if castorcmdstderr:
1648  #If it failed print the stderr message to the log and tell the user the tarball (.tgz) is kept in the working directory
1649  self.printFlush(castorcmdstderr)
1650  self.printFlush("Since the CASTOR archiving for the tarball failed the file %s is kept in directory %s"%(TarFile, perfsuitedir))
1651  else:
1652  #If it was successful then remove the tarball from the working directory:
1653  self.printFlush("Successfully archived the tarball %s in CASTOR!"%(TarFile))
1654  self.printFlush("The tarball can be found: %s"%(fullcastorpathfile))
1655  self.printFlush("The logfile can be found: %s"%(fullcastorpathlog))
1656  self.printFlush("Deleting the local copy of the tarballs")
1657  rmtarballcmd="rm -Rf %s"%(AbsTarFile)
1658  rmtarballmd5cmd="rm -Rf %s"%(AbsTarFileMD5)
1659  rmtarballlogcmd="rm -Rf %s"%(AbsTarFileLOG)
1660  self.printFlush(rmtarballcmd)
1661  self.printFlush(rmtarballmd5cmd)
1662  self.printFlush(rmtarballlogcmd)
1663  #Obsolete os.popen-> subprocess.Popen
1664  #self.printFlush(os.popen4(rmtarballcmd)[1].read())
1665  self.printFlush(subprocess.Popen(rmtarballcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() )
1666  self.printFlush(subprocess.Popen(rmtarballmd5cmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() )
1667  self.printFlush(subprocess.Popen(rmtarballlogcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() )
1668  tarballTime.set_end(datetime.datetime.now())
1669  else:
1670  self.printFlush("Performance Suite directory will not be archived in a tarball since --no_tarball option was chosen")
1671 
1672  #End of script actions!
1673 
1674  #Print a time stamp at the end:
1675  date=time.ctime(time.time())
1676  self.logh.write("Performance Suite finished running at %s on %s in directory %s\n" % (date,self.host,path))
1677  if self.ERRORS == 0:
1678  self.logh.write("There were no errors detected in any of the log files!\n")
1679  else:
1680  self.logh.write("ERROR: There were %s errors detected in the log files, please revise!\n" % self.ERRORS)
1681  #print "No exit code test"
1682  #sys.exit(1)
1683  except exceptions.Exception, detail:
1684  self.logh.write(str(detail) + "\n")
1685  self.logh.flush()
1686  if not self.logh.isatty():
1687  self.logh.close()
1688  raise
1689  #Add the possibility to send as an email the execution logfile to the user and whoever else interested:
1690  if MailLogRecipients != "": #Basically leave the option to turn it off too.. --mail ""
1691  self.printFlush("Sending email notification for this execution of the performance suite with command:")
1692  sendLogByMailcmd='cat cmsPerfSuite.log |mail -s "Performance Suite finished running on %s" '%self.host + MailLogRecipients
1693  self.printFlush(sendLogByMailcmd)
1694  #Obsolete os.popen-> subprocess.Popen
1695  #self.printFlush(os.popen4(sendLogByMailcmd)[1].read())
1696  self.printFlush(subprocess.Popen(sendLogByMailcmd,shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.read() )
1697  else:
1698  self.printFlush('No email notification will be sent for this execution of the performance suite since option --mail "" was used')
1699 
1700  TotalTime.set_end(datetime.datetime.now())
1701  self.printFlush("Total Running Time\t%s hrs (%s mins)"%(TotalTime.get_duration()['hours'],TotalTime.get_duration()['minutes']))
1702 
1703  #Dump of the TimerInfo information
1704  #First dump it as a pickleq file...
1705  #Well in order to do so without the complication of serializing a custom class instance need to make the dictionary fully string-made:
1706  TimerInfoStr={}
1707  PerfSuiteTimerInfo=open("PerfSuiteTimerInfo.pkl","wb")
1708  #pickle.dump(TimerInfo,PerfSuiteTimerInfo)
1709  #PerfSuiteTimerInfo.close()
1710  #For now print it at the bottom of the log:
1711  self.logh.write("Test type\tActual Test\tDuration\tStart Time\tEnd Time\n")
1712  for key in TimerInfo.keys():
1713  #self.printFlush(key)
1714  TimerInfoStr.update({key:{}})
1715  for test in TimerInfo[key].keys():
1716  TimerInfoStr[key].update({test:[str(TimerInfo[key][test].get_duration()['hours'])+" hrs ("+str(TimerInfo[key][test].get_duration()['minutes'])+" mins)",TimerInfo[key][test].get_start(),TimerInfo[key][test].get_end()]})
1717  self.logh.write(key+"\t"+test+"\t")
1718  self.logh.write("%s hrs (%s mins)\t"%(TimerInfo[key][test].get_duration()['hours'],TimerInfo[key][test].get_duration()['minutes']))
1719  self.logh.write("%s\t"%TimerInfo[key][test].get_start())
1720  self.logh.write("%s\n"%TimerInfo[key][test].get_end())
1721  pickle.dump(TimerInfoStr,PerfSuiteTimerInfo)
1722  PerfSuiteTimerInfo.close()
1723 
1724  self.logh.write("Final Performance Suite exit code was %s"%FinalExitCode)
1725  self.logh.flush()
1726  sys.exit(FinalExitCode)
1727 
1728 def main(argv=[__name__]): #argv is a list of arguments.
1729  #Valid ways to call main with arguments:
1730  #main(["--cmsScimark",10])
1731  #main(["-t100"]) #With the caveat that the options.timeSize will be of type string... so should avoid using this!
1732  #main(["-timeSize,100])
1733  #Invalid ways:
1734  #main(["One string with all options"])
1735 
1736  #Let's instatiate the class:
1737  suite=PerfSuite()
1738 
1739  #print suite
1740  #Uncomment this for tests with main() in inteactive python:
1741  #print suite.optionParse(argv)
1742 
1743  PerfSuiteArgs={}
1744  (PerfSuiteArgs['create'],
1745  PerfSuiteArgs['castordir'],
1746  PerfSuiteArgs['TimeSizeEvents'],
1747  PerfSuiteArgs['IgProfEvents'],
1748  PerfSuiteArgs['CallgrindEvents'],
1749  PerfSuiteArgs['MemcheckEvents'],
1750  PerfSuiteArgs['cmsScimark'],
1751  PerfSuiteArgs['cmsScimarkLarge'],
1752  PerfSuiteArgs['cmsdriverOptions'],
1753  PerfSuiteArgs['cmsdriverPUOptions'],
1754  PerfSuiteArgs['stepOptions'],
1755  PerfSuiteArgs['quicktest'],
1756  PerfSuiteArgs['profilers'],
1757  PerfSuiteArgs['cpus'],
1758  PerfSuiteArgs['cores'],
1759  PerfSuiteArgs['prevrel'],
1760  PerfSuiteArgs['bypasshlt'],
1761  PerfSuiteArgs['runonspare'],
1762  PerfSuiteArgs['perfsuitedir'],
1763  PerfSuiteArgs['logfile'],
1764  PerfSuiteArgs['TimeSizeCandles'],
1765  PerfSuiteArgs['IgProfCandles'],
1766  PerfSuiteArgs['CallgrindCandles'],
1767  PerfSuiteArgs['MemcheckCandles'],
1768  PerfSuiteArgs['TimeSizePUCandles'],
1769  PerfSuiteArgs['IgProfPUCandles'],
1770  PerfSuiteArgs['CallgrindPUCandles'],
1771  PerfSuiteArgs['MemcheckPUCandles'],
1772  PerfSuiteArgs['PUInputFile'],
1773  PerfSuiteArgs['userInputFile'],
1774  PerfSuiteArgs['MailLogRecipients'],
1775  PerfSuiteArgs['tarball']
1776  ) = suite.optionParse(argv)
1777 
1778  if PerfSuiteArgs['create']: # Before anything, request the AFS volume (it takes some time...)
1779  suite.createIgVolume()
1780 
1781  if not PerfSuiteArgs['logfile'] == None:
1782  if os.path.exists(PerfSuiteArgs['logfile']):
1783  oldlogfile=PerfSuiteArgs['logfile']+"_"+time.strftime("%d-%m-%Y_%H:%M:%S")
1784  #Move old logfile to a file with the same filename plus a timestamp appended
1785  mvOldLogfile=subprocess.Popen("mv %s %s"%(PerfSuiteArgs['logfile'],oldlogfile), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1786  mvOldLogfileExitCode=mvOldLogfile.wait()
1787  #Finally open the logfile and put the information above in it:
1788  try:
1789  ActualLogfile = open(PerfSuiteArgs['logfile'],"w")
1790  if mvOldLogfileExitCode:
1791  ActualLogfile.write("Please check what happened: A file named %s existed already and the attempt to move it to %s produced the following output: %s\n"%(PerfSuiteArgs['logfile'],oldlogfile,mvOldLogfile.stdout))
1792  else:
1793  ActualLogfile.write("***WARNING! A file named %s existed already!\n***It has been moved to %s before starting the current logfile!\n"%(PerfSuiteArgs['logfile'],oldlogfile))
1794  except (OSError, IOError), detail:
1795  ActualLogfile.write("Failed to open the intended logfile %s, detail error:\n%s"%(PerfSuiteArgs['logfile'],detail))
1796 
1797  else:
1798  try:
1799  ActualLogfile = open(PerfSuiteArgs['logfile'],"w")
1800  except (OSError, IOError), detail:
1801  ActualLogfile.write("Failed to open the intended logfile %s, detail error:\n%s"%(PerfSuiteArgs['logfile'],detail))
1802  ActualLogfile.flush()
1803 
1804  #Three lines to add the exact command line used to call the performance suite directly in the log.
1805  ActualLogfile.write("Performance suite invoked with command line:\n")
1806  cmdline=reduce(lambda x,y:x+" "+y,sys.argv)
1807  ActualLogfile.write(cmdline+"\n")
1808  ActualLogfile.flush()
1809 
1810  #Debug printout that we could silence...
1811  ActualLogfile.write("Initial PerfSuite Arguments:\n")
1812  for key in PerfSuiteArgs.keys():
1813  ActualLogfile.write("%s %s\n"%(key,PerfSuiteArgs[key]))
1814  ActualLogfile.flush()
1815  #print PerfSuiteArgs
1816 
1817  PerfSuiteArgs['cpu_list'] = PerfSuiteArgs['cpus'] #To access the actual number of cpus used inside the threads..
1818 
1819  #Handle in here the case of multiple cores and the loading of cores with cmsScimark:
1820  if len(PerfSuiteArgs['cpus']) > 1:
1821  ActualLogfile.write("More than 1 cpu: threading the Performance Suite!\n")
1822  outputdir=PerfSuiteArgs['perfsuitedir']
1823  runonspare=PerfSuiteArgs['runonspare'] #Save the original value of runonspare for cmsScimark stuff
1824  cpus=PerfSuiteArgs['cpus']
1825  cores=PerfSuiteArgs['cores']
1826  if runonspare:
1827  for core in range(PerfSuiteArgs['cores']):
1828  cmsScimarkLaunch_pslist={}
1829  if len(cpus) != cores: #In case of this (relval), don't load the others with cmsScimark
1830  if (core not in cpus):
1831  #self.logh.write("Submitting cmsScimarkLaunch.csh to run on core cpu "+str(core) + "\n")
1832  ActualLogfile.write("Submitting cmsScimarkLaunch.csh to run on core cpu "+str(core)+"\n")
1833  subcmd = "cd %s ; cmsScimarkLaunch.csh %s" % (outputdir, str(core))
1834  command="taskset -c %s sh -c \"%s\" &" % (str(core), subcmd)
1835  #self.logh.write(command + "\n")
1836  ActualLogfile.write(command+"\n")
1837  #cmsScimarkLaunch.csh is an infinite loop to spawn cmsScimark2 on the other
1838  #cpus so it makes no sense to try reading its stdout/err
1839  cmsScimarkLaunch_pslist[core]=subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
1840  ActualLogfile.write("Spawned %s \n with PID %s"%(command,cmsScimarkLaunch_pslist[core].pid))
1841  ActualLogfile.flush()
1842  PerfSuiteArgs['runonspare']=False #Set it to false to avoid cmsScimark being spawned by each thread
1843  logfile=PerfSuiteArgs['logfile']
1844  suitethread={}
1845  for cpu in cpus:
1846  #Make arguments "threaded" by setting for each instance of the suite:
1847  #1-A different output (sub)directory
1848  #2-Only 1 core on which to run
1849  #3-Automatically have a logfile... otherwise stdout is lost?
1850  #To be done:[3-A flag for Valgrind not to "thread" itself onto the other cores..]
1851  cpudir = os.path.join(outputdir,"cpu_%s" % cpu)
1852  if not os.path.exists(cpudir):
1853  os.mkdir(cpudir)
1854  PerfSuiteArgs['perfsuitedir']=cpudir
1855  PerfSuiteArgs['cpus']=[cpu] #Keeping the name cpus for now FIXME: change it to cpu in the whole code
1856  if PerfSuiteArgs['logfile']:
1857  PerfSuiteArgs['logfile']=os.path.join(cpudir,os.path.basename(PerfSuiteArgs['logfile']))
1858  else:
1859  PerfSuiteArgs['logfile']=os.path.join(cpudir,"cmsPerfSuiteThread.log")
1860  #Now spawn the thread with:
1861  suitethread[cpu]=PerfThread(**PerfSuiteArgs)
1862  #ActualLogfile.write(suitethread[cpu])
1863  ActualLogfile.write("Launching PerfSuite thread on cpu%s"%cpu)
1864  ActualLogfile.flush()
1865  #print "With arguments:"
1866  #print PerfSuiteArgs
1867  suitethread[cpu].start()
1868 
1869  while reduce(lambda x,y: x or y, map(lambda x: x.isAlive(),suitethread.values())):
1870  try:
1871  time.sleep(5.0)
1872  sys.stdout.flush()
1873  except (KeyboardInterrupt, SystemExit):
1874  raise
1875  ActualLogfile.write("All PerfSuite threads have completed!\n")
1876  ActualLogfile.flush()
1877 
1878  else: #No threading, just run the performance suite on the cpu core selected
1879  suite.runPerfSuite(**PerfSuiteArgs)
1880 
1881 if __name__ == "__main__":
1882 
1883  main(sys.argv)
cmsPerfSuite.PerfSuite.runcmd
def runcmd
Run a command and return the exit status.
Definition: cmsPerfSuite.py:566
start
Definition: start.py:1
cmsPerfSuite.PerfSuite.mkCandleDir
def mkCandleDir
Make directory for a particular candle and profiler.
Definition: cmsPerfSuite.py:601
cmsPerfSuite.PerfSuite.benchmarks
def benchmarks
Run cmsScimark benchmarks a number of times.
Definition: cmsPerfSuite.py:691
cmsPerfSuite.PerfSuite.runCmdSet
def runCmdSet
Run a list of commands using system ! We should rewrite this not to use system (most cases it is unne...
Definition: cmsPerfSuite.py:540
cmsPerfSuite.PerfSuite.valFilterReport
def valFilterReport
Filter lines in the valgrind report that match GEN,SIM.
Definition: cmsPerfSuite.py:648
cmsPerfSuite.PerfSuite._verbose
_verbose
Check logfile option.
Definition: cmsPerfSuite.py:72
cmsPerfSuite.PerfSuite.runCmsInput
def runCmsInput
Wrapper for cmsRelvalreportInput.
Definition: cmsPerfSuite.py:763
mathSSE::return
return((rh^lh)&mask)
cmsPerfSuite.PerfSuite.printFlush
def printFlush
Print and flush a string (for output to a log file)
Definition: cmsPerfSuite.py:558
cmsPerfSuite.PerfSuite.simpleGenReport
def simpleGenReport
Prepares the profiling directory and runs all the selected profiles (if this is not a unit test) ...
Definition: cmsPerfSuite.py:792
cmsCpuInfo.get_NumOfCores
def get_NumOfCores
Definition: cmsCpuInfo.py:6
cmsPerfSuite.PerfSuite.PerfTestTotalTimer
PerfTestTotalTimer
FIXME: We may want to introduce a switch here or agree on a different default (currently 10 cmsScimar...
Definition: cmsPerfSuite.py:1423
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
cmsPerfSuite.PerfSuite.cprootfile
def cprootfile
Copy root file from another candle&#39;s directory ! Again this is messy.
Definition: cmsPerfSuite.py:612
cmsPerfSuite.PerfSuite.runPerfSuite
def runPerfSuite
Runs benchmarking, cpu spinlocks on spare cores and profiles selected candles.
Definition: cmsPerfSuite.py:926
cmsPerfSuite.PerfSuite.testCmsDriver
def testCmsDriver
Test cmsDriver.py (parses the simcandles file, removing duplicate lines, and runs the cmsDriver part)...
Definition: cmsPerfSuite.py:730
update
#define update(a, b)
Definition: TrackClassifier.cc:10
main
Definition: main.py:1
cmsPerfSuite.PerfSuite._debug
_debug
Developer options.
Definition: cmsPerfSuite.py:69
cmsPerfSuite.PerfSuite.displayErrors
def displayErrors
Display G4 cerr errors and CMSExceptions in the logfile.
Definition: cmsPerfSuite.py:632
cmsPerfSuite.PerfSuite.runCmsReport
def runCmsReport
This function is a wrapper around cmsRelvalreport.
Definition: cmsPerfSuite.py:713
split
double split
Definition: MVATrainer.cc:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
Generated for CMSSW Reference Manual by doxygen 1.8.5