CMS 3D CMS Logo
cmsPerfPublish.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #G.Benelli Jan 22 2007, J. Nicolson 12 Aug 2008
3 #A little script to move Simulation Performance Suite
4 #relevant html and log files into our public area
5 #/afs/cern.ch/cms/sdt/web/performance/simulation/
6 #Set here the standard number of events (could become an option... or could be read from the log...)
7 
8 ###############################
9 #
10 # Warning!!! We should use copytree() instead of our self defined copytree4
11 # However, copytree does not use ignore patterns (for filtering files)
12 # before python v2.6, when we upgrade to python 2.6 we should use this
13 # functionality.
14 import tempfile as tmp
15 import optparse as opt
16 import cmsPerfRegress as cpr
17 import re, os, sys, time, glob, socket, fnmatch
18 from shutil import copy2, copystat
19 from stat import *
20 from cmsPerfCommons import CandFname, Step, ProductionSteps, Candles
21 import ROOT
22 #Switching from os.popen4 to subprocess.Popen for Python 2.6 (340_pre5 onwards):
23 import subprocess
24 from functools import reduce
25 
26 PROG_NAME = os.path.basename(sys.argv[0])
27 DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal"
28 DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation"
29 TMP_DIR = ""
30 cpFileFilter = ( "*.root", ) # Unix pattern matching not regex
31 cpDirFilter = ( ) # Unix pattern matching not regex
32 
33 TimeSizeNumOfEvents = -9999
34 IgProfNumOfEvents = -9999
35 CallgrindNumOfEvents = -9999
36 MemcheckNumOfEvents = -9999
37 
38 DirName=( #These need to match the candle directory names ending (depending on the type of profiling)
39  "TimeSize",
40  "IgProf",
41  "IgProf_Perf",
42  "IgProf_Mem",
43  "Callgrind",
44  "Memcheck",
45  #Adding the extra PU directories:
46  "PU_TimeSize",
47  "PU_IgProf",
48  "PU_IgProf_Perf",
49  "PU_IgProf_Mem",
50  "PU_Callgrind",
51  "PU_Memcheck"
52  )
53 #Defining Steps as a union of Step and ProductionSteps:
54 Steps=set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"]) #Adding GEN,FASTSIM too by hand.
55 print Steps
56 
57 ##################
58 #
59 # Small functions
60 #
61 
62 class ReldirExcept(Exception):
63  "Relative directory could not be determined"
64 
65 def fail(errstr=""):
66  print errstr
67  delTmpDir()
68  sys.exit()
69 
70 def addtrailingslash(adir):
71  trail = re.compile("/$")
72  if os.path.isdir(adir) and not trail.search(adir):
73  adir = adir + "/"
74  return adir
75 
76 def getDate():
77  return time.ctime()
78 
79 def getcmdBasic(cmd):
80  #Obsolete popen4-> subprocess.Popen
81  #return os.popen4(cmd)[1].read().strip()
82  return subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
83 
84 def getcmd(command):
85  if _debug > 2:
86  print command
87  #Obsolete popen4-> subprocess.Popen
88  #return os.popen4(command)[1].read().strip()
89  return subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
90 
91 def prettySize(size):
92  nega = size < 0
93  if nega:
94  size = -size
95  suffixes = [("B",2**10), ("k",2**20), ("M",2**30), ("G",2**40), ("T",2**50)]
96  for suf, lim in suffixes:
97  if size > lim:
98  continue
99  else:
100  if nega:
101  return "-" + round(size/float(lim/2**10),2).__str__() + suf
102  else:
103  return round(size/float(lim/2**10),2).__str__()+suf
104 
105 class Row(object):
106 
107  def __init__(self,table):
108  self.table = table
109  self.coldata = {}
110 
111  def __str__(self):
112  return str(self.coldata)
113 
114  def addEntry(self,colname,value):
115  self.table._newCol(colname)
116  self.coldata[colname] = value
117 
118  def getRowDict(self):
119  return self.coldata
120 
121 class Table(object):
122 
123  def __init__(self):
124  self.colNames = []
125  self.keys = [None]
126  self.rows = {None: self.colNames}
127 
128  def __str__(self):
129  out = self.keys
130  out += "\n" + self.rows
131  return out
132 
133  def _newCol(self,name):
134  if name in self.colNames:
135  pass
136  else:
137  self.colNames.append(name)
138 
139  def getCols(self):
140  return self.colNames
141 
142  def newRow(self,name):
143  if name in self.rows.keys():
144  pass
145  else:
146  self.keys.append(name)
147  self.rows[name] = Row(self)
148 
149  return self.rows[name]
150 
151  def getTable(self,mode=0):
152  name = "Total"
153 
154  for key in self.keys:
155  if key == None:
156  pass
157  else:
158  total1 = 0
159  total2 = 0
160  rowobj = self.rows[key]
161  rowdict = rowobj.getRowDict()
162  for col in self.colNames:
163  if col == None:
164  pass
165  elif col in rowdict and not col == name:
166  if mode == 1:
167  total1 += rowdict[col]
168  else:
169  (step_tot1, step_tot2) = rowdict[col]
170  total1 += step_tot1
171  total2 += step_tot2
172  if mode == 1:
173  rowobj.addEntry(name,total1)
174  else:
175  rowobj.addEntry(name,(total1,total2))
176 
177  return (self.keys, self.rows)
178 
179  def addRow(self,row,name):
180  if name in self.rows.keys():
181  pass
182  else:
183  self.keys.append(name)
184  self.rows[name] = row
185 
186  def transpose(self):
187  transp = Table()
188  for col in self.colnames:
189  rowobj = transp.newRow(col)
190  for key in self.keys:
191  if key == None:
192  pass
193  else:
194  row_dict = self.rows[key].getRowDict()
195  if key in row_dict:
196  rowobj.addEntry(key,row_dict[col])
197  return transp
198 
199 ######################
200 #
201 # Main
202 #
203 def main():
204  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
205  #Bad design... why is this function defined here?
206  #Either no function, or define somewhere else.
207  def _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage):
208  """Use function syscp to copy LogFiles and cmsScimarkDir over to staging area"""
209  if _verbose:
210  print "Copying the logfiles to %s/." % stage
211  print "Copying the cmsScimark2 results to the %s/." % stage
212 
213  syscp(LogFiles , stage + "/")
214  syscp(cmsScimarkDir, stage + "/")
215 
216  #Same comment as above about the opportunity of this function definition here.
217  def _createLogFile(LogFile,date,LocalPath,ShowTagsResult):
218  """Creating a small logfile with basic publication script running information (never used really by other scripts in the suite)."""
219  try:
220  LOG = open(LogFile,"w")
221  if _verbose:
222  print "Writing Production Host, Location, Release and Tags information in %s" % LogFile
223  LOG.write("These performance tests were executed on host %s and published on %s" % (HOST,date))
224  LOG.write("They were run in %s" % LocalPath)
225  LOG.write("Results of showtags -r in the local release:\n%s" % ShowTagsResult)
226  LOG.close()
227  except IOError as detail:
228  print "WARNING: Can't create log file"
229  print detail
230 
231  # Print Program header
232  print_header()
233 
234  # Get environment variables
235  #FIXME: should check into this and make sure the proper variables for the tests being published are read from logfile (case of deprecated releases...)
236  print "\n Getting Environment variables..."
237  (LocalPath, ShowTagsResult) = get_environ()
238 
239  # Parse options
240  (options,args) = optionparse()
241 
242  # Determine program parameters and input/staging locations
243  print "\n Determining locations for input and staging..."
244  (drive,path,remote,stage,port,repdir,prevrev,igprof_remotedir) = getStageRepDirs(options,args)
245 
246  #Get the number of events for each test from logfile:
247  print "\n Getting the number of events for each test..."
248  #Let's do a quick implementation of something that looks at the logfile:
249  cmsPerfSuiteLogfile="%s/cmsPerfSuite.log"%repdir
250 
251  if os.path.exists(cmsPerfSuiteLogfile):
252  try:
253  (TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents)=getNumOfEventsFromLog(cmsPerfSuiteLogfile)
254  #Get the CMSSW version and SCRAM architecture from log (need these for the IgProf new publishing with igprof-navigator)
255  (CMSSW_arch,CMSSW_version)=getArchVersionFromLog(cmsPerfSuiteLogfile)
256  #For now keep the dangerous default? Better set it to a negative number...
257  except:
258  print "There was an issue in reading out the number of events for the various tests or the architecture/CMSSW version using the standard logfile %s"%cmsPerfSuiteLogFile
259  print "Check that the format was not changed: this scripts relies on the initial perfsuite arguments to be dumped in the logfile one per line!"
260  print "For now taking the default values for all tests (0)!"
261 
262  print "\n Scan report directory..."
263  # Retrieve some directories and information about them
264  (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir) = scanReportArea(repdir)
265  print "cmsScimarkResults are %s"%cmsScimarkResults
266  print "\n Copy report files to staging directory..."
267  # Copy reports to staging area
268  _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage)
269 
270  print "\n Creating log file..."
271  # Produce a small logfile with basic info on the Production area
272  _createLogFile("%s/ProductionLog.txt" % stage,date,repdir,ShowTagsResult)
273 
274  #Check if there are IgProf tests:
275  for dirname in os.listdir(repdir):
276  if "IgProf" in dirname:
277  print "\n Handling IgProf reports..."
278  # add a function to handle the IgProf reports
279  stageIgProfReports(igprof_remotedir,CMSSW_arch,CMSSW_version)
280 
281  print "\n Creating HTML files..."
282  # create HTML files
283  createWebReports(stage,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
284 
285  print "\n Copy profiling logs to staging directory..."
286  # Copy over profiling logs...
287  getDirnameDirs(repdir,stage)
288 
289  # Send files to remote location
290  if remote:
291  print "\n Uploading web report to remote location..."
292  syncToRemoteLoc(stage,drive,path,port)
293  print "\n Finished uploading! Now removing staging directory..."
294  delTmpDir()
295 
296  print "\n Finished!!!"
297 
298 ##########################
299 #
300 # Get require environment variables
301 #
302 def get_environ():
303  global CMSSW_VERSION, CMSSW_RELEASE_BASE, CMSSW_BASE, HOST, USER, BASE_PERFORMANCE, CMSSW_WORK
304  global DEF_RELVAL, DEF_SIMUL
305 
306  try:
307  CMSSW_VERSION=os.environ['CMSSW_VERSION']
308  CMSSW_RELEASE_BASE=os.environ['CMSSW_RELEASE_BASE']
309  CMSSW_BASE=os.environ['CMSSW_BASE']
310  HOST=os.environ['HOST']
311  USER=os.environ['USER']
312  CMSSW_WORK = os.path.join(CMSSW_BASE,"work/Results")
313  except KeyError as detail:
314  fail("ERROR: Could not retrieve some necessary environment variables. Have you ran scramv1 runtime -csh yet?. Details: %s" % detail)
315 
316  LocalPath=getcmdBasic("pwd")
317  ShowTagsResult=getcmdBasic("showtags -r")
318 
319  #Adding a check for a local version of the packages
320  PerformancePkg="%s/src/Validation/Performance" % CMSSW_BASE
321  if (os.path.exists(PerformancePkg)):
322  BASE_PERFORMANCE=PerformancePkg
323  print "**Using LOCAL version of Validation/Performance instead of the RELEASE version**"
324  else:
325  BASE_PERFORMANCE="%s/src/Validation/Performance" % CMSSW_RELEASE_BASE
326 
327  return (LocalPath,ShowTagsResult)
328 
329 ###############
330 #Option parser
331 #
332 def optionparse():
333  global PROG_NAME, _debug, _dryrun, _verbose
334 
335  parser = opt.OptionParser(usage=("""%s [HOST:]DEST_PATH [Options]
336 
337  Arguments:
338  [HOST:]DEST_PATH - This is where the report should be published, you can specify a local path or a directory on a remote machine (HOST is optional)
339 
340  Examples:
341  Publish report to default local directory
342  ./%s
343  Publish report to \"/some/local/dir\"
344  ./%s /some/local/dir
345  Publish report to \"/some/other/dir\" remote host \"hal.cern.ch\"
346  ./%s hal.cern.ch:/some/other/dir
347  Publish report to default relval location (this could be remote or local depending on the hardcoded default)
348  ./%s --relval"""
349  % ( PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME)))
350 
351  devel = opt.OptionGroup(parser, "Developer Options",
352  "Caution: use these options at your own risk."
353  "It is believed that some of them bite.\n")
354  parser.add_option(
355  '--relval',
356  action="store_true",
357  dest='relval',
358  help='Use the default RelVal location',
359  #metavar='<STEPS>',
360  )
361 
362  parser.add_option(
363  '-v',
364  '--verbose',
365  action="store_true",
366  dest='verbose',
367  help='output more information',
368  #metavar='<STEPS>',
369  )
370 
371  parser.add_option(
372  '--simul',
373  action="store_true",
374  dest='simulation',
375  help='Use the default simulation location',
376  #metavar='<STEPS>',
377  )
378 
379  parser.add_option(
380  '--prev',
381  type="string",
382  dest='previousrev',
383  help='The override the name of the previous release. Default is the string obtain from the identification file REGRESSION.<prevrel>.vs.<newrel>',
384  metavar='<NAME>',
385  default="",
386  )
387 
388  parser.add_option(
389  '--input',
390  type="string",
391  dest='repdir',
392  help='The location of the report files to be published',
393  metavar='<DIR>'
394  )
395 
396  parser.add_option(
397  '-p',
398  '--port',
399  type='int',
400  dest='port',
401  help='Use a particular port number to rsync material to a remote server',
402  metavar='<PORT>'
403  )
404  parser.add_option(
405  '--igprof',
406  type='string',
407  dest='ig_remotedir',
408  default='IgProfData', #Reverting to local default publication for now#'/afs/cern.ch/cms/sdt/web/qa/igprof-testbed/data', #For now going straight into AFS... later implement security via local disk on cmsperfvm and cron job there:
409  #default='cmsperfvm:/data/projects/conf/PerfSuiteDB/IgProfData', #This should not change often! In this virtual machine a cron job will run to move stuff to AFS.
410  help='Specify an AFS or host:mydir remote directory instead of default one',
411  metavar='<IGPROF REMOTE DIRECTORY>'
412  )
413 
414  devel.add_option(
415  '-d',
416  '--debug',
417  type='int',
418  dest='debug',
419  help='Show debug output',
420  #metavar='DEBUG',
421  )
422 
423  devel.add_option(
424  '--dry-run',
425  action="store_true",
426  dest='dryrun',
427  help='Do not send files to remote server, but run everything else',
428  #metavar='DEBUG',
429  )
430 
431  repdirdef = os.getcwd()
432  parser.set_defaults(debug=0,simulation=False,relval=False,port=873,pretend=False,repdir=repdirdef,verbose=False)
433  parser.add_option_group(devel)
434 
435  (options, args) = parser.parse_args()
436 
437  _debug = options.debug
438  _dryrun = options.dryrun
439  _verbose = options.verbose
440 
441  numofargs = len(args)
442 
443  if (options.simulation and options.relval) or ((options.simulation or options.relval) and numofargs >= 1):
444  parser.error("You can not specify simulation and relval together. Neither can you specify simulation or relval AND a path")
445  sys.exit()
446 
447  return (options, args)
448 
449 
450 #A function to get the number of events for each test from the logfile:
451 def getNumOfEventsFromLog(logfile):
452  '''A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!'''
453  log=open(logfile,"r")
454  TimeSizeEvents=0
455  IgProfEvents=0
456  CallgrindEvents=0
457  MemcheckEvents=0
458  for line in log:
459  #FIXME:
460  #For robustness could read this from the Launching the X tests (.....) with N events each and keep that format decent for parsing.
461  #One more place where XML would seem a better choice to extract information (Timing of the performance suite in general is also, but publishing and harvesting some other info as well).
462  if 'TimeSizeEvents' in line and not TimeSizeEvents:
463  lineitems=line.split()
464  TimeSizeEvents=lineitems[lineitems.index('TimeSizeEvents')+1]
465  if 'IgProfEvents' in line and not IgProfEvents:
466  lineitems=line.split()
467  IgProfEvents=lineitems[lineitems.index('IgProfEvents')+1]
468  if 'CallgrindEvents' in line and not CallgrindEvents:
469  lineitems=line.split()
470  CallgrindEvents=lineitems[lineitems.index('CallgrindEvents')+1]
471  if 'MemcheckEvents' in line and not MemcheckEvents:
472  lineitems=line.split()
473  MemcheckEvents=lineitems[lineitems.index('MemcheckEvents')+1]
474  return (TimeSizeEvents,IgProfEvents,CallgrindEvents,MemcheckEvents)
475 
476 def getArchVersionFromLog(logfile):
477  '''Another very fragile function to get the architecture and the CMSSW version parsing the logfile...'''
478  log=open(logfile,"r")
479  arch=re.compile("^Current Architecture is")
480  version=re.compile("^Current CMSSW version is")
481  CMSSW_arch="UNKNOWN_ARCH"
482  CMSSW_version="UNKNOWN_VERSION"
483  for line in log:
484  if arch.search(line):
485  CMSSW_arch=line.split()[3]
486  if version.search(line):
487  CMSSW_version=line.split()[4]
488  return(CMSSW_arch,CMSSW_version)
489 
490 
491 #####################
492 #
493 # Determine locations of staging and report dirs
494 #
495 def getStageRepDirs(options,args):
496  global TMP_DIR, IS_TMP, DEF_LOCAL, CMSSW_VERSION
497  DEF_LOCAL = CMSSW_WORK
498  numofargs = len(args)
499 
500  repdir = os.path.abspath(options.repdir)
501  repdir = addtrailingslash(repdir)
502 
503  if not os.path.exists(repdir):
504  fail("ERROR: The specified report directory %s to retrieve report information from does not exist, exiting" % repdir)
505 
506  previousrev = options.previousrev
507  if previousrev == "":
508  regressfiles = glob.glob("%s/REGRESSION.*.vs.*" % repdir)
509  if not len(regressfiles) == 0:
510  regressID = regressfiles[0]
511  base = os.path.basename(regressID)
512  split = base.split(".")
513  previousrev = split[1]
514  currentrel = split[3]
515  print "Regression Identification file exists, renaming report title for regression report. Old ver: %s" % previousrev
516  else:
517  print "No regression ID file exists and previous release name was not specified. Producing normal report."
518  else:
519  print "Previous release name was specified, renaming report title for regression report. Old ver %s" % previousrev
520 
521  uri = ""
522  defaultlocal = False
523  if options.simulation:
524  uri = DEF_SIMUL
525  elif options.relval:
526  uri = DEF_RELVAL
527  elif numofargs >= 1:
528  uri = args[0] # Add directory CMSSW_VERSION later in temp! Not now, otherwise we get into a mess if this is a remote dir
529  else:
530  defaultlocal = True
531  uri = DEF_LOCAL
532 
533  ####
534  #
535  # Determine if location is remote
536  #
537  # Try not to re-arrange we don't want to assume that default locations are not remote
538  #
539  ####
540 
541  drive = ""
542  path = ""
543  if ":" in uri:
544  drive, path = uri.split(":",1)
545  else:
546  path = uri
547 
548  if drive == "":
549  path = os.path.abspath(path)
550  remote = not drive == ""
551 
552  if remote:
553  unResolved = True
554  try:
555  socket.getaddrinfo(drive,53)
556  unResolved = False
557  except socket.gaierror:
558  unResolved = True
559 
560  # try see if it's an ipaddress
561  if unResolved:
562  try:
563  socket.gethostbyaddr(drive)
564  unResolved = False
565  except socket.gaierror:
566  unResolved = True
567  if unResolved:
568  print "ERROR: Can not determine your hostname or ipv{4,6} address %s" % drive
569  if not (_dryrun or _test):
570  fail("exiting...")
571 
572  if (not remote) and (not options.port == 873) :
573  print "WARNING: Can not use a port if not performing a remote copy, ignoring"
574  port = options.port
575 
576  ###
577  #
578  # Determine Staging Area
579  #
580 
581  StagingArea=""
582  localExists = os.path.exists("%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
583 
584  if remote:
585  #Cannot use this since the /tmp is limited to 2GB on lxbuild machines!
586  #TMP_DIR=tmp.mkdtemp(prefix="/tmp/%s" % PROG_NAME)
587  TMP_DIR=tmp.mkdtemp(prefix="/build/%s" % PROG_NAME)
588  StagingArea = TMP_DIR
589  #Local but dir already exists
590  elif defaultlocal and localExists:
591  TMP_DIR=tmp.mkdtemp(prefix="%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
592  StagingArea = TMP_DIR
593  print "WARNING: %s already exists, creating a temporary staging area %s" % (CMSSW_WORK,TMP_DIR)
594  #Local cases
595  elif defaultlocal:
596  StagingArea = CMSSW_WORK
597  try:
598  os.mkdir(os.path.join(CMSSW_BASE,"work"))
599  os.mkdir(os.path.join(CMSSW_BASE,"work","Results"))
600  except OSError:
601  pass
602  print "**User did not specify location of results, staging in default %s**" % StagingArea
603  else:
604  print "**User chose to publish results in a local directory**"
605  StagingArea = path
606  if not os.path.exists(path):
607  try:
608  os.mkdir("%s" % path)
609  except OSError as detail:
610  if detail.errno == 13:
611  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
612  elif detail.errno == 17:
613  #If the directory already exists just carry on
614  pass
615  else:
616  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
617 
618  IS_TMP = not TMP_DIR == ""
619  ######
620  #
621  # create Version dir
622  # This is why when we create a Tmp dir we get the version(tmpname)/version
623  # structure. We should remove this if we don't want it but i dont think it matters
624  StagingArea="%s/%s" % (StagingArea,CMSSW_VERSION)
625  try:
626  os.mkdir("%s" % StagingArea)
627  except OSError as detail:
628  if detail.errno == 13:
629  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
630  elif detail.errno == 17:
631  #If the directory already exists just carry on
632  pass
633  else:
634  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
635 
636  return (drive,path,remote,StagingArea,port,repdir,previousrev,options.ig_remotedir)
637 
638 ####################
639 #
640 # Scan report area for required things
641 #
642 def scanReportArea(repdir):
643  """Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results.
644  It returns Execution date (completion), current date, list of logfiles and cmsScimark results"""
645  date=getDate()
646  LogFiles = glob.glob(repdir + "cms*.log")
647  if _verbose:
648  print "Found the following log files:"
649  print LogFiles
650 
651  cmsScimarkDir = glob.glob(repdir + "cmsScimarkResults_*")
652  if _verbose:
653  print "Found the following cmsScimark2 results directories:"
654  print cmsScimarkDir
655 
656  cmsScimarkResults = []
657  for adir in cmsScimarkDir:
658  htmlfiles = glob.glob(adir + "/*.html")
659  #FIXME:
660  #Really unnecessary use of map in my opinion
661  #Could do with cmsScimarkResults.extend(htmlfiles)
662  map(cmsScimarkResults.append,htmlfiles)
663 
664  ExecutionDateLast = ""
665  ExecutionDate = ""
666  ExecutionDateSec=0
667  cmsreg = re.compile("^cmsPerfSuite")
668  for logf in LogFiles:
669  if cmsreg.search(logf):
670  ExecutionDateLastSec = os.stat(logf)[ST_CTIME]
671  ExecutionDateLast = os.stat(logf)[ST_MTIME]
672  if _verbose:
673  print "Execution (completion) date for %s was: %s" % (logf,ExecutionDateLast)
674  if (ExecutionDateLastSec > ExecutionDateSec):
675  ExecutionDateSec = ExecutionDateLastSec
676  ExecutionDate = ExecutionDateLast
677 
678  if ExecutionDate == "":
679  ExecutionDate = ExecutionDateLast
680 
681  return (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir)
682 
683 def createRegressHTML(reghtml,repdir,outd,CurrentCandle,htmNames):
684  RegressTmplHTML="%s/doc/regress.html" % (BASE_PERFORMANCE)
685  candnreg = re.compile("CandleName")
686  candhreg = re.compile("CandlesHere")
687  try:
688  REGR = open(reghtml,"w")
689  for line in open(RegressTmplHTML):
690  if candhreg.search(line):
691  html = "<table>"
692 
693  for x in htmNames:
694  abspath = os.path.join(repdir,outd)
695  if os.path.exists(abspath):
696  html += "<tr><td><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></td></tr>\n" % (outd,x,outd,x)
697  else:
698  html += "<tr><td> %s does not exist probably because the log file for the previous release was missing</td></tr>" % (abspath)
699  html += "</table>"
700  REGR.write(html)
701  elif candnreg.search(line):
702  REGR.write(CurrentCandle)
703  else:
704  REGR.write(line)
705  except IOError as detail:
706  print "ERROR: Could not write regression html %s because %s" % (os.path.basename(reghtml),detail)
707 
708 def getOutputNames(base,reportName):
709  logreg = re.compile("(.*)\.log$")
710  matches = logreg.search(reportName)
711  logdir = logreg.sub(matches.groups()[0],reportName)
712  outd = os.path.join(base,logdir)
713  nologext = matches.groups()[0]
714  return (nologext,outd)
715 
716 def rootfile_cmp(x,y):
717  (fname,x) = x
718  (fname,y) = y
719  x = os.path.basename(x)
720  y = os.path.basename(y)
721  stepreg = re.compile("%s_(..*)\.root" % fname)
722  if stepreg.search(x):
723  x = stepreg.search(x).groups()[0]
724  if stepreg.search(y):
725  y = stepreg.search(y).groups()[0]
726  return step_cmp(x,y)
727 
728 def dirname_cmp(x,y):
729  (candle,prof,x) = x
730  (candle,prof,y) = y
731  x = os.path.basename(x)
732  y = os.path.basename(y)
733  stepreg = re.compile("%s_(..*)_%s" % (candle,prof))
734  if stepreg.search(x):
735  x = stepreg.search(x).groups()[0]
736  if stepreg.search(y):
737  y = stepreg.search(y).groups()[0]
738  return step_cmp(x,y)
739 
740 def reg_dirname_cmp(x,y):
741  (candle,prof,x) = x
742  (candle,prof,y) = y
743  x = os.path.basename(x)
744  y = os.path.basename(y)
745  stepreg = re.compile("%s_(..*)_%s_regression" % (candle,prof))
746  if stepreg.search(x):
747  x = stepreg.search(x).groups()[0]
748  if stepreg.search(y):
749  y = stepreg.search(y).groups()[0]
750  return step_cmp(x,y)
751 
752 def logrep_cmp(x,y):
753  (fname,x) = x
754  (fname,y) = y
755  x = os.path.basename(x)
756  y = os.path.basename(y)
757  stepreg = re.compile("%s_(..*)_TimingReport.log" % fname)
758  if stepreg.search(x):
759  x = stepreg.search(x).groups()[0]
760  if stepreg.search(y):
761  y = stepreg.search(y).groups()[0]
762  return step_cmp(x,y)
763 
764 def timerep_cmp(x,y):
765  (fname,x) = x
766  (fname,y) = y
767  x = os.path.basename(x)
768  y = os.path.basename(y)
769  stepreg = re.compile("%s_(..*)_TimingReport" % fname)
770  if stepreg.search(x):
771  x = stepreg.search(x).groups()[0]
772  if stepreg.search(y):
773  y = stepreg.search(y).groups()[0]
774  return step_cmp(x,y)
775 
776 def step_cmp(x,y):
777  xstr = x
778  ystr = y
779  x_idx = -1
780  y_idx = -1
781  bestx_idx = -1
782  besty_idx = -1
783  sndbst_x = -1
784  sndbst_y = -1
785  last_x = -1
786  last_y = -1
787  for i in range(len(Step)):
788  stepreg = re.compile("^%s.*" % Step[i])
789  # fallback
790  if Step[i] in xstr and sndbst_x == -1:
791  last_x = i
792  if Step[i] in ystr and sndbst_y == -1:
793  last_y = i
794  # second best
795  if xstr in Step[i] and bestx_idx == -1:
796  sndbst_x = i
797  if ystr in Step[i] and besty_idx == -1:
798  sndbst_y = i
799  # next best
800  if stepreg.search(xstr) and x_idx == -1:
801  bestx_idx = i # If an exact match has not been found but something similar has, set x best index
802  if stepreg.search(ystr) and y_idx == -1:
803  besty_idx = i # If an exact match has not been found but something similar has, set y best index
804  # actual
805  if Step[i] == xstr and x_idx == -1:
806  x_idx = i # if an exact match has been found then set x index
807  if Step[i] == ystr and y_idx == -1:
808  y_idx = i # if an exact match has been found then set y index
809  if not ( x_idx == -1 or y_idx == -1):
810  break # if an exact match has been found for both, stop
811 
812  # use best match if we still can't find indices
813  if x_idx == -1:
814  x_idx = bestx_idx
815  if y_idx == -1:
816  y_idx = besty_idx
817 
818  # use second best if we still can't find indices
819  if x_idx == -1:
820  x_idx = sndbst_x
821  if y_idx == -1:
822  y_idx = sndbst_y
823 
824  # use fallback if we still can't find indices
825  if x_idx == -1:
826  x_idx = last_x
827  if y_idx == -1:
828  y_idx = last_y
829 
830  if x_idx == -1 or y_idx == -1:
831  print "WARNING: No valid step names could be found in the logfiles or root filenames being sorted: x: %s y: %s." % (xstr,ystr)
832  print "x", x_idx, "y", y_idx
833 
834  if x_idx < y_idx:
835  return -1
836  elif x_idx == y_idx:
837  return 0
838  elif y_idx < x_idx:
839  return 1
840 
841 
842 ######################
843 #
844 # Create HTML pages for candles
845 
846 def createCandlHTML(tmplfile,candlHTML,CurrentCandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
847  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
848  def _stripbase(base, astr):
849  basereg = re.compile("^%s/?(.*)" % base)
850  out = astr
851  found = basereg.search(astr)
852  if found:
853  out = found.groups()[0]
854  return out
855 
856  def _getProfileReportLink(repdir,CurrentCandle,CurDir,step,CurrentProfile,Profiler):
857  #FIXME:
858  #Pileup now has it own directory... should add it in the DirName dictionary at the beginning?
859  ProfileTemplate=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step,CurrentProfile),Profiler)
860  #print ProfileTemplate
861  #There was the issue of SIM vs sim (same for DIGI) between the previous RelVal based performance suite and the current.
862  ProfileTemplateLowCaps=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step.lower(),CurrentProfile),Profiler)
863  ProfileReportLink = glob.glob(ProfileTemplate)
864  #Filter out the Pile up directories when step does not contain Pile up
865  #if not ('PILEUP' in step):
866  # print "NPNPNPNP BEFORE %s"%ProfileReportLink
867  # ProfileReportLink=filter(lambda x: "PU" not in x,ProfileReportLink)
868  # print "NPNPNPNP %s"%ProfileReportLink
869  #print ProfileReportLink
870  if len(ProfileReportLink) > 0:
871  #print ProfileReportLink
872  if not reduce(lambda x,y: x or y,map(lambda x: CurrentCandle in x,ProfileReportLink)):# match with caps try low caps
873  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
874  else:
875  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
876  ProfileReportLink = map(lambda x: _stripbase(repdir,x),ProfileReportLink)
877  #print ProfileReportLink
878  return ProfileReportLink
879 
880  def _writeReportLink(INDEX,ProfileReportLink,CurrentProfile,step,NumOfEvents,Profiler=""):
881  if Profiler == "":
882  INDEX.write("<li><a href=\"%s\">%s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,step,NumOfEvents))
883  else:
884  #FIXME: need to fix this: why do we have the number of Memcheck events hardcoded?
885  if CurrentProfile == "memcheck_valgrind":#FIXME:quick and dirty hack to report we have changed memcheck to 5 events now...
886  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,"5"))
887  else:
888  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,NumOfEvents))
889  def IgProfDumpsTable(INDEX,ProfileLinks,step):
890  #See if the end of job profiles IgProfMemTotal.res or IgProfMemLive.res are in the list as they should:
891  EndOfJobProfileLink=filter(lambda x: "IgProfMemTotal.res" in x or "IgProfMemLive.res" in x, ProfileLinks)[0] #Assume only one of the two is there, as it should.
892  #Remove it from the list so we can order it:
893  ProfileLinks.remove(EndOfJobProfileLink)
894  #Sort the list in numerical order:
895  ProfileLinks.sort(key=lambda x: int(x.split(".")[-2]))
896  #Prepare regexp to find and replace MemTotal with MemLive for .gz links
897  IgProfMemLive_regexp=re.compile("IgProfMemLive")
898  if IgProfMemLive_regexp.search(EndOfJobProfileLink):
899  MemProfile="IgProf MEM LIVE"
900  else:
901  MemProfile="IgProf MEM TOTAL"
902  #Prepare the table header:
903  INDEX.write("<li>%s"%MemProfile)
904  INDEX.write("<table><tr><td>Profile after event</td><td>Total Memory Size (bytes)</td><td>Total Calls (number)</td><td>Link to gzipped IgProf profile</td></tr>")
905  for link in ProfileLinks:
906  #Build and check the link to the .gz profile that is always called IgProfMemTotal also for MemLive:
907  gzProfile=os.path.join(link.split("/")[-3],link.split("/")[-1])[:-3]+"gz"
908  if IgProfMemLive_regexp.search(gzProfile):
909  gzProfile=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzProfile)
910  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(link.split(".")[-2],open(link,"r").readlines()[6].split()[1],open(link,"r").readlines()[6].split()[2],gzProfile,os.path.basename(gzProfile)))
911  #Put in the end of job one by hand:
912  gzEndOfJobProfileLink=os.path.join(EndOfJobProfileLink.split("/")[-3],EndOfJobProfileLink.split("/")[-1])[:-3]+"gz"
913  if IgProfMemLive_regexp.search(gzEndOfJobProfileLink):
914  gzEndOfJobProfileLink=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzEndOfJobProfileLink)
915  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%("End of job",open(EndOfJobProfileLink,"r").readlines()[6].split()[1],open(EndOfJobProfileLink,"r").readlines()[6].split()[2],gzEndOfJobProfileLink,os.path.basename(gzEndOfJobProfileLink)))
916  #Closing the table and the list item tags
917  INDEX.write("</table>")
918  INDEX.write("</li>")
919  #FIXME:
920  #These numbers are used in the index.html they are not automatically matched to the actual
921  #ones (one should automate this, by looking into the cmsCreateSimPerfTestPyRelVal.log logfile)
922 
923  NumOfEvents={
924  DirName[0] : TimeSizeNumOfEvents,
925  DirName[1] : IgProfNumOfEvents,
926  DirName[2] : IgProfNumOfEvents,
927  DirName[3] : IgProfNumOfEvents,
928  DirName[4] : CallgrindNumOfEvents,
929  DirName[5] : MemcheckNumOfEvents,
930  DirName[6] : TimeSizeNumOfEvents,
931  DirName[7] : IgProfNumOfEvents,
932  DirName[8] : IgProfNumOfEvents,
933  DirName[9] : IgProfNumOfEvents,
934  DirName[10] : CallgrindNumOfEvents,
935  DirName[11] : MemcheckNumOfEvents
936  }
937 
938  Profile=(
939  #These need to match the profile directory names ending within the candle directories
940  "TimingReport",
941  "TimeReport",
942  "SimpleMemReport",
943  "EdmSize",
944  "IgProfperf",
945  "IgProfMemTotal",
946  "IgProfMemLive",
947  "IgProfMemAnalyse",
948  "valgrind",
949  "memcheck_valgrind"
950  )
951  IgProfMemAnalyseOut=( #This is the special case of IgProfMemAnalyse
952  "doBeginJob_output.html",
953  "doEvent_output.html",
954  "mem_live.html",
955  "mem_total.html"
956  )
957  memcheck_valgrindOut=( #This is the special case of Valgrind MemCheck (published via Giovanni's script)
958  "beginjob.html",
959  "edproduce.html",
960  "esproduce.html"
961  )
962  OutputHtml={ #These are the filenames to be linked in the candle html page for each profile
963  Profile[0] : "*TimingReport.html", #The wildcard spares the need to know the candle name
964  Profile[1] : "TimeReport.html", #This is always the same (for all candles)
965  Profile[2] : "*.html", #This is supposed to be *SimpleMemoryCheck.html, but there is a bug in cmsRelvalreport.py and it is called TimingReport.html!
966  Profile[3] : "objects_pp.html", #This is one of 4 objects_*.html files, it's indifferent which one to pick, just need consistency
967  Profile[4] : "overall.html", #This is always the same (for all candles)
968  Profile[5] : "overall.html", #This is always the same (for all candles)
969  Profile[6] : "overall.html", #This is always the same (for all candles)
970  Profile[7] : "doBeginJob_output.html", #This is complicated... there are 3 html to link... (see IgProf MemAnalyse below)
971  Profile[8] : "overall.html", #This is always the same (for all candles)
972  Profile[9] : "beginjob.html" #This is complicated there are 3 html to link here too... (see Valgrind MemCheck below)
973  }
974 
975 
976  candnreg = re.compile("CandleName")
977  candhreg = re.compile("CandlesHere")
978  try:
979  CAND = open(candlHTML,"w")
980  for line in open(tmplfile):
981  if candhreg.search(line):
982  CAND.write('<div id="content">')
983  CAND.write("<h2>")
984  CAND.write(CurrentCandle)
985  CAND.write("</h2>\n")
986 
987  if _verbose:
988  print "Producing candles html: ", CurrentCandle
989 
990  for CurDir in DirName:
991 
992  LocalPath = os.path.join(repdir,"%s_%s" % (CurrentCandle,CurDir))
993  LocalDirname = os.path.basename(LocalPath)
994 
995  if not prevrev == "":
996 
997  profs = []
998  #FIXME!
999  #Check what this was used for.. it now has different DirName list since we added IgProf_Perf and IgProf_Mem on top of IgProf alone (still there)
1000  if CurDir == DirName[0] or CurDir == DirName[6]: #TimeSize tests (with and without PU)
1001  profs = Profile[0:4]
1002  elif CurDir == DirName[1] or CurDir == DirName[7]: #IgProf tests (with and without PU)
1003  profs = Profile[4:8]
1004  elif CurDir == DirName[2] or CurDir == DirName[8]: #IgProf Perf tests (with and without PU)
1005  profs =[Profile[4]] #Keeping format to a list...
1006  elif CurDir == DirName[3] or CurDir == DirName[9]: #IgProf Mem tests (with and without PU)
1007  profs =Profile[5:8] #Keeping format to a list...
1008  elif CurDir == DirName[4] or CurDir == DirName[10]: #Callgrind tests (with and without PU)
1009  profs = [Profile[8]] #Keeping format to a list...
1010  elif CurDir == DirName[5] or CurDir == DirName[11]: #Memcheck tests (with and without PU)
1011  profs = [Profile[9]] #Keeping format to a list...
1012  #This could be optimized, but for now just comment the code:
1013  #This for cycle takes care of the case in which there are regression reports to link to the html:
1014  for prof in profs:
1015  if _verbose:
1016  print "Scanning for profile information for: ", prof
1017 
1018  printed = False
1019  fullprof = (CurrentCandle,prof)
1020  outd = ""
1021  nologext = ""
1022  if prof == "TimingReport":
1023  timeReports = glob.glob(os.path.join(LocalPath,"%s_*_%s.log" % (CandFname[CurrentCandle],prof)))
1024  if len(timeReports) > 0:
1025  if not printed:
1026  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1027  printed = True
1028  for log in timeReports:
1029  reportName = os.path.basename(log)
1030  (nologext, outd) = getOutputNames(LocalDirname,reportName)
1031  CAND.write("<h4>%s</h4>\n" % reportName)
1032  htmNames = ["changes.png"]
1033  otherNames = ["graphs.png" , "histos.png"]
1034  regressHTML= "%s-regress.html" % nologext
1035  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,outd,x),otherNames)))
1036  html = ""
1037  if pathsExist:
1038  html = "<p>Performance graphs and histograms superimposed for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1039  else:
1040  html = "<p>No change in performance graph available</p>\n"
1041  regressHTML="%s/%s" % (WebArea,regressHTML)
1042 
1043  for x in htmNames:
1044  abspath = os.path.join(repdir,outd,x)
1045  if os.path.exists(abspath):
1046  html += "<p><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></p>\n" % (outd,x,outd,x)
1047  else:
1048  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1049 
1050  createRegressHTML(regressHTML,repdir,outd,CurrentCandle,otherNames)
1051  CAND.write(html)
1052  CAND.write("\n</tr></table>")
1053 
1054 
1055  elif prof == "SimpleMemReport":
1056  simMemReports = glob.glob(os.path.join(LocalPath,"%s_*_%s" % (CandFname[CurrentCandle],prof)))
1057  simMemReports = map(lambda x: (CandFname[CurrentCandle],prof,x), simMemReports )
1058  simMemReports.sort(cmp=dirname_cmp)
1059  simMemReports = map(lambda x: x[2], simMemReports )
1060  if len(simMemReports) > 0:
1061  if not printed:
1062  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1063  printed = True
1064  for adir in simMemReports:
1065  reportName = os.path.basename(adir)
1066  CAND.write("<h4>%s</h4>\n" % reportName)
1067  htmNames = ["vsize_change.png", "rss_change.png"]
1068  otherNames = ["vsize_graphs.png","rss_graphs.png"]
1069  nologext = reportName
1070  outd = reportName
1071  regressHTML= "%s-regress.html" % nologext
1072  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,LocalDirname,outd,x),otherNames)))
1073  html = ""
1074  if pathsExist:
1075  html = "<p>Superimposed memory performance graphs for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1076  else:
1077  html = "<p>No change in performance graph available</p>\n"
1078  regressHTML="%s/%s" % (WebArea,regressHTML)
1079 
1080  for x in htmNames:
1081  abspath = os.path.join(repdir,LocalDirname,outd,x)
1082  if os.path.exists(abspath):
1083  html += "<p><a href=\"./%s/%s/%s\"><img src=\"./%s/%s/%s\" /></a></p>\n" % (LocalDirname,outd,x,LocalDirname,outd,x)
1084  else:
1085  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1086 
1087  createRegressHTML(regressHTML,repdir,"%s/%s" % (LocalDirname,outd),CurrentCandle,otherNames)
1088  CAND.write(html)
1089  CAND.write("\n</tr></table>")
1090 
1091  elif prof == "EdmSize" or prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof== "IgProfperf" or prof == "Callgrind":
1092  regresPath = os.path.join(LocalPath,"%s_*_%s_regression" % (CandFname[CurrentCandle],prof))
1093  regresses = glob.glob(regresPath)
1094  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s_regression" % (CandFname[CurrentCandle],prof))
1095  if len(regresses) > 0:
1096  if not printed:
1097  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1098  printed = True
1099  regresses = map(lambda x: (CandFname[CurrentCandle],prof,x),regresses)
1100  regresses.sort(cmp=reg_dirname_cmp)
1101  regresses = map(lambda x: x[2],regresses)
1102  for rep in regresses:
1103  base = os.path.basename(rep)
1104  found = stepreg.search(base)
1105  step = "Unknown-step"
1106  if found:
1107  step = found.groups()[0]
1108  htmlpage = ""
1109  if prof == "IgProfMemLive" or prof == "IgProfMemTotal" or prof== "IgProfperf" or prof == "Callgrind":
1110  htmlpage = "overall.html"
1111  else:
1112  htmlpage = "objects_pp.html"
1113  CAND.write("<a href=\"%s/%s/%s\">%s %s regression report</a><br/>\n" % (LocalDirname,base,htmlpage,prof,step))
1114  #Here we go back to the "non-regression" reports listing
1115  CandleLogFiles = []
1116  if os.path.exists(LocalPath):
1117  thedir = os.listdir(LocalPath)
1118  CandleLogFiles = filter(lambda x: (x.endswith(".log") or x.endswith("EdmSize")) and not os.path.isdir(x) and os.path.exists(x), map(lambda x: os.path.abspath(os.path.join(LocalPath,x)),thedir))
1119 
1120  if (len(CandleLogFiles)>0):
1121 
1122  syscp(CandleLogFiles,WebArea + "/")
1123  base = os.path.basename(LocalPath)
1124  lfileshtml = ""
1125 
1126  for cand in CandleLogFiles:
1127  cand = os.path.basename(cand)
1128  if _verbose:
1129  print "Found %s in %s\n" % (cand,LocalPath)
1130 
1131  if not "EdmSize" in cand:
1132  lfileshtml += "<a href=\"./%s/%s\">%s </a><br/>" % (base,cand,cand)
1133 
1134  CAND.write("<p><strong>Logfiles for %s</strong></p>\n" % CurDir)
1135  CAND.write(lfileshtml)
1136 
1137  PrintedOnce = False
1138  for CurrentProfile in Profile:
1139  #print Steps
1140  for step in Steps: #Using Steps here that is Step+ProductionSteps!
1141  #print step
1142  #print "DEBUG:%s,%s,%s,%s,%s,%s"%(repdir,CurrentCandle,CurDir,step,CurrentProfile,OutputHtml[CurrentProfile])
1143  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1144  CurDir,
1145  step,
1146  CurrentProfile,
1147  OutputHtml[CurrentProfile])
1148  #if ProfileReportLink:
1149  # print ProfileReportLink
1150  isProfinLink = False
1151  if len (ProfileReportLink) > 0:
1152  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1153 
1154  if isProfinLink:
1155  #It could also not be there
1156 
1157  if (PrintedOnce==False):
1158  #Making sure it's printed only once per directory (TimeSize, IgProf, Valgrind) each can have multiple profiles
1159 
1160  #This is the "title" of a series of profiles, (TimeSize, IgProf, Valgrind)
1161  CAND.write("<p><strong>%s</strong></p>\n" % CurDir)
1162  CAND.write("<ul>\n")
1163  PrintedOnce=True
1164 
1165  #Special cases first (IgProf MemAnalyse and Valgrind MemCheck)
1166  #Add among the special cases any IgProfMem (5,6,7) since now we added the dumping:
1167  if (CurrentProfile == Profile[5] or CurrentProfile == Profile[6]):
1168  for prolink in ProfileReportLink:
1169  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1170  for igprofmem in ["*IgProfMemTotal*.res","*IgProfMemLive*.res"]:
1171  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1172  CurDir,
1173  step,
1174  CurrentProfile,
1175  igprofmem)
1176  isProfinLink = False
1177  if len (ProfileReportLink) > 0:
1178  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1179  if isProfinLink :#It could also not be there
1180  #for prolink in ProfileReportLink:
1181  IgProfDumpsTable(CAND,ProfileReportLink,step)
1182  # _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=os.path.basename(prolink))
1183 
1184  elif (CurrentProfile == Profile[7]):
1185  for igprof in IgProfMemAnalyseOut:
1186  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1187  CurDir,
1188  step,
1189  CurrentProfile,
1190  igprof)
1191  isProfinLink = False
1192  if len (ProfileReportLink) > 0:
1193  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1194  if isProfinLink :#It could also not be there
1195  for prolink in ProfileReportLink:
1196  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=igprof)
1197 
1198 
1199  elif (CurrentProfile == Profile[9]):
1200 
1201  for memprof in memcheck_valgrindOut:
1202  #print memprof
1203  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1204  CurDir,
1205  step,
1206  CurrentProfile,
1207  memprof
1208  )
1209  isProfinLink = False
1210  if len (ProfileReportLink) > 0:
1211  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1212  if isProfinLink :#It could also not be there
1213  for prolink in ProfileReportLink:
1214  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=memprof)
1215 
1216  else:
1217  for prolink in ProfileReportLink:
1218  if "regression" not in prolink: #To avoid duplication of links to regression reports!
1219  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1220  #print "Step is %s, CurrentProfile is %s and ProfileReportLink is %s and prolink is %s"%(step,CurrentProfile,ProfileReportLink,prolink)
1221 
1222 
1223  if PrintedOnce:
1224  CAND.write("</ul>\n")
1225  PrintedOnce=False
1226 
1227  CAND.write("<hr />")
1228  CAND.write("<br />\n")
1229  CAND.write("</div>\n")
1230  elif candnreg.search(line):
1231  CAND.write(CurrentCandle)
1232  else:
1233  CAND.write(line)
1234 
1235  CAND.close()
1236  except IOError as detail:
1237  print "ERROR: Could not write candle html %s because %s" % (os.path.basename(candlHTML),detail)
1238 
1239 
1240 def populateFromTupleRoot(tupname,repdir,rootfile,pureg):
1241  table = Table()
1242  for cand in Candles:
1243  fname = CandFname[cand]
1244  globpath = os.path.join(repdir,"%s_TimeSize" % cand,"%s_*_TimingReport" % fname)
1245  stepDirs = glob.glob(globpath)
1246  stepDirs = map(lambda x: (fname,x), stepDirs)
1247  stepDirs.sort(cmp=timerep_cmp)
1248  stepDirs = map(lambda x: x[1], stepDirs)
1249  stepreg = re.compile("%s_(.*)_TimingReport" % fname)
1250  createNewRow = True
1251  curRow = None
1252  createPURow = True
1253  puRow = None
1254  for stepdir in stepDirs:
1255  base = os.path.basename(stepdir)
1256  found = stepreg.search(base)
1257  step = "Unknown-step"
1258  if found:
1259  step = found.groups()[0]
1260  realstep = "Unknown-step"
1261  if "PILEUP" in step:
1262  found = pureg.search(step)
1263  if found:
1264  realstep = found.groups()[0]
1265  if createPURow:
1266  createPURow = False
1267  puRow = Row(table)
1268  rootf = os.path.join(stepdir,rootfile)
1269 
1270  if os.path.exists(rootf):
1271  f = ROOT.TFile(rootf)
1272 
1273  cpu_time_tree = ROOT.TTree()
1274  f.GetObject("cpu_time_tuple;1",cpu_time_tree)
1275  if cpu_time_tree:
1276  if cpu_time_tree.InheritsFrom("TTree"):
1277  data1 = None
1278  data2 = None
1279  for t in cpu_time_tree:
1280  data1 = t.total1
1281  data2 = t.total2
1282  if data1 and data2:
1283  if createNewRow:
1284  createNewRow = False
1285  curRow = table.newRow(cand)
1286  data_tuple = (data1,data2)
1287 
1288  if "PILEUP" in step:
1289  puRow.addEntry(realstep,data_tuple)
1290  else:
1291  if createNewRow:
1292  createNewRow = False
1293  curRow = table.newRow(cand)
1294 
1295  curRow.addEntry(step,data_tuple)
1296  f.Close()
1297  if puRow == None:
1298  pass
1299  else:
1300  table.addRow(puRow,"%s PILEUP" %cand)
1301  return table
1302 
1303 
1304 def createHTMLtab(INDEX,table_dict,ordered_keys,header,caption,name,mode=0):
1305  cols = len(ordered_keys)
1306  totcols = (cols * 3) + 1
1307  innercol = 3
1308  colspan = totcols - 1
1309  labels = []
1310  if mode == 1:
1311  labels = ["fs1","fs2","&#x0394;"]
1312  elif mode == 2 or mode == 3:
1313  colspan = cols
1314  innercol = 1
1315  else:
1316  labels = ["t1" ,"t2" ,"&#x0394;"]
1317 
1318 
1319  INDEX.write("<h3>%s</h3>\n" % header)
1320  INDEX.write("<table>\n")
1321  INDEX.write("<caption>%s</caption>\n" % caption)
1322  INDEX.write("<thead><tr><th></th><th colspan=\"%s\" scope=\"colgroup\">%s</th></tr></thead>" % (colspan,name))
1323  INDEX.write("<tbody>\n")
1324  for key in ordered_keys:
1325  INDEX.write("<tr>")
1326  if key == None:
1327  INDEX.write("<th></th>")
1328  else:
1329  INDEX.write("<td scope=\"row\">")
1330  INDEX.write(key)
1331  INDEX.write("</td>")
1332  for col in table_dict[None]:
1333  if key == None:
1334  INDEX.write("<th colspan=\"%s\" scope=\"col\">" % innercol)
1335  INDEX.write(col)
1336  INDEX.write("</th>")
1337  else:
1338  rowdict = table_dict[key].getRowDict()
1339  if col in rowdict:
1340  if mode == 2:
1341  dat = prettySize(rowdict[col])
1342  INDEX.write("<td>")
1343  INDEX.write("%s" % dat)
1344  INDEX.write("</td>")
1345 
1346  elif mode == 3:
1347  dat = rowdict[col]
1348  INDEX.write("<td>")
1349  INDEX.write("%6.2f" % dat)
1350  INDEX.write("</td>")
1351  else:
1352  (data1, data2) = rowdict[col]
1353  diff = data2 - data1
1354 
1355  if mode == 1:
1356  diff = prettySize(diff)
1357  data1 = prettySize(data1)
1358  data2 = prettySize(data2)
1359 
1360  seq = [ data1, data2, diff ]
1361  for dat in seq:
1362  INDEX.write("<td id=\"data\">")
1363  if mode == 1:
1364  INDEX.write("%s" % dat) # %s if
1365  else:
1366  INDEX.write("%6.2f" % dat) # %s if
1367 
1368  INDEX.write("</td>")
1369  else:
1370  if mode == 2 or mode == 3:
1371  INDEX.write("<td>")
1372  INDEX.write("N/A")
1373  INDEX.write("</td>")
1374  else:
1375  for i in range(3):
1376  INDEX.write("<td>")
1377  INDEX.write("N/A")
1378  INDEX.write("</td>")
1379  INDEX.write("</tr>\n")
1380  # write an additional row if this row is the header row
1381  # we need to describe the sub columns
1382  if not (mode == 2 or mode == 3 ):
1383  if key == None:
1384  INDEX.write("<tr>")
1385  INDEX.write("<th>Candles</th>")
1386  for col in table_dict[None]:
1387  INDEX.write("<th>%s</th>" % labels[0])
1388  INDEX.write("<th>%s</th>" % labels[1])
1389  INDEX.write("<th>%s</th>" % labels[2])
1390  INDEX.write("</tr>\n")
1391  INDEX.write("</tbody></table>\n")
1392 
1393  INDEX.write("<br />")
1394 
1395 def stageIgProfReports(remotedir,arch,version):
1396  '''Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.'''
1397  #FIXME: can eliminate this part if using tar pipes... was done with rsynch in mind
1398  #Compose command to create remote dir:
1399  if ":" in remotedir: #Remote host local directory case
1400  (host,dir)=remotedir.split(":")
1401  mkdir_cmd="ssh %s (mkdir %s;mkdir %s/%s)"%(host,dir,dir,arch)
1402  else: #AFS or local case
1403  mkdir_cmd="mkdir %s;mkdir %s/%s"%(remotedir,remotedir,arch)
1404 
1405  #Create remote dir:
1406  try:
1407  print mkdir_cmd
1408  os.system(mkdir_cmd)
1409  print "Successfully created publication directory"
1410  except:
1411  print "Issues with publication directory existence/creation!"
1412 
1413  #Copy files over to remote dir
1414  #replacing rsync with tar pipes since it can hang on AFS (Andreas' experience):
1415  #rsync_cmd="rsync -avz *_IgProf_*/*.sql3 %s/%s/%s"%(remotedir,arch,version)
1416  if ":" in remotedir:
1417  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | ssh %s "cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*"'%(host,dir,arch,version,version)
1418  else:
1419  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | (cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*)'%(remotedir,arch,version,version)
1420  try:
1421  # print rsync_cmd
1422  # os.system(rsync_cmd)
1423  print tarpipe_cmd
1424  os.system(tarpipe_cmd)
1425  print "Successfully copied IgProf reports to %s"%remotedir
1426  except:
1427  print "Issues with rsyncing to the remote directory %s!"%remotedir
1428 
1429  #Make sure permissions are set for group to be able to write:
1430  if ":" in remotedir: #Remote host local directory case
1431  chmod_cmd="ssh %s chmod -R 775 %s/%s"%(host,dir,arch)
1432  else:
1433  chmod_cmd="chmod -R 775 %s/%s"%(remotedir,arch)
1434  try:
1435  print chmod_cmd
1436  os.system(chmod_cmd)
1437  print "Successfully set permissions for IgProf reports directory %s"%remotedir
1438  except:
1439  print "(Potential) issues with chmoding the remote directory %s!"%remotedir
1440 
1441  return #Later, report here something like the web link to the reports in igprof-navigator...
1442 
1443 
1444 #####################
1445 #
1446 # Create web report index and create HTML file for each candle
1447 #
1448 def createWebReports(WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
1449 
1450  #Some nomenclature
1451 
1452  Candle = Candles #These need to match the directory names in the work area
1453 
1454  CmsDriverCandle = CandFname #{ #These need to match the cmsDriver.py output filenames
1455 
1456 
1457  #Produce a "small" index.html file to navigate the html reports/logs etc
1458  IndexFile="%s/index.html" % WebArea
1459  TemplateHtml="%s/doc/index.html" % BASE_PERFORMANCE
1460 
1461  cmsverreg = re.compile("CMSSW_VERSION")
1462  hostreg = re.compile("HOST")
1463  lpathreg = re.compile("LocalPath")
1464  fsizereg = re.compile("FSizeTable")
1465  cpureg = re.compile("CPUTable")
1466  proddreg = re.compile("ProductionDate")
1467  logfreg = re.compile("LogfileLinks")
1468  dirbreg = re.compile("DirectoryBrowsing")
1469  pubdreg = re.compile("PublicationDate")
1470  candhreg = re.compile("CandlesHere")
1471  #Loop line by line to build our index.html based on the template one
1472  #Copy the perf_style.css file from Validation/Performance/doc
1473 
1474  CandlTmpltHTML="%s/doc/candle.html" % BASE_PERFORMANCE
1475  if _verbose:
1476  print "Copying %s/doc/perf_style.css style file to %s/." % (BASE_PERFORMANCE,WebArea)
1477  print "Template used: %s" % TemplateHtml
1478 
1479  syscp((BASE_PERFORMANCE + "/doc/perf_style.css"),WebArea + "/.")
1480  pureg = re.compile("(.*)_PILEUP")
1481  try:
1482  INDEX = open(IndexFile,"w")
1483  for NewFileLine in open(TemplateHtml) :
1484  if cmsverreg.search(NewFileLine):
1485  if prevrev == "":
1486  INDEX.write("Performance Reports for %s\n" % CMSSW_VERSION)
1487  else:
1488  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1489  globs = glob.glob(globpath)
1490  if len(globs) < 1:
1491  pass
1492  else:
1493  latestreg = re.compile("REGRESSION.%s.vs.(.*)" % prevrev)
1494  found = latestreg.search(os.path.basename(globs[0]))
1495  if found:
1496  latestrel = found.groups()[0]
1497  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,latestrel))
1498  else:
1499  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,CMSSW_VERSION))
1500  elif hostreg.search(NewFileLine):
1501  INDEX.write(HOST + "\n")
1502  #File Size Summary Table
1503  elif fsizereg.search(NewFileLine):
1504  #Case of NO-REGRESSION:
1505  if prevrev == "":
1506  fsize_tab = Table()
1507 
1508  for cand in Candles:
1509  fname = CandFname[cand]
1510  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1511  rootfiles = glob.glob(globpath)
1512  rootfiles = map(lambda x: (fname,x), rootfiles)
1513  rootfiles.sort(cmp=rootfile_cmp)
1514  rootfiles = map(lambda x: x[1], rootfiles)
1515  stepreg = re.compile("%s_(.*).root" % fname)
1516  createNewRow = True
1517  curRow = None
1518  createPURow = True
1519  puRow = None
1520  for rootf in rootfiles:
1521  base = os.path.basename(rootf)
1522  found = stepreg.search(base)
1523  step = "Unknown-step"
1524  if found:
1525  step = found.groups()[0]
1526  realstep = "Unknown-step"
1527  if "PILEUP" in step:
1528  found = pureg.search(step)
1529  if found:
1530  realstep = found.groups()[0]
1531  if createPURow:
1532  createPURow = False
1533  puRow = Row(fsize_tab)
1534  try:
1535  statinfo = os.stat(rootf)
1536  fsize = statinfo.st_size
1537  if createNewRow:
1538  createNewRow = False
1539  curRow = fsize_tab.newRow(cand)
1540 
1541  if "PILEUP" in step:
1542  puRow.addEntry(realstep,fsize)
1543  else:
1544  if createNewRow:
1545  createNewRow = False
1546  curRow = fsize_tab.newRow(cand)
1547 
1548  curRow.addEntry(step,fsize)
1549  except IOError as detail:
1550  print detail
1551  except OSError as detail:
1552  print detail
1553  if puRow == None:
1554  pass
1555  else:
1556  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1557  (ordered_keys,table_dict) = fsize_tab.getTable(1)
1558  cols = len(ordered_keys)
1559 
1560  if len(table_dict) > 1 and cols > 0:
1561  createHTMLtab(INDEX,table_dict,ordered_keys,
1562  "Release ROOT file sizes",
1563  "Table showing current release ROOT filesizes in (k/M/G) bytes.",
1564  "Filesizes",2)
1565  #Case of REGRESSION vs previous release:
1566  else:
1567  try:
1568  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1569  globs = glob.glob(globpath)
1570  if len(globs) < 1:
1571  raise IOError(2,globpath,"File does not exist","File does not exist")
1572  idfile = open(globs[0])
1573  oldpath = ""
1574  for line in idfile:
1575  oldpath = line
1576  oldpath = oldpath.strip()
1577  #print "##########TABLE DEBUG :oldpath is %s"%oldpath
1578  fsize_tab = Table()
1579 
1580  for cand in Candles:
1581  fname = CandFname[cand]
1582  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1583  rootfiles = glob.glob(globpath)
1584  rootfiles = map(lambda x: (fname,x), rootfiles)
1585  rootfiles.sort(cmp=rootfile_cmp)
1586  rootfiles = map(lambda x: x[1], rootfiles)
1587  stepreg = re.compile("%s_(.*).root" % fname)
1588  createNewRow = True
1589  curRow = None
1590  createPURow = True
1591  puRow = None
1592  for rootf in rootfiles:
1593  base = os.path.basename(rootf)
1594  found = stepreg.search(base)
1595  step = "Unknown-step"
1596  if found:
1597  step = found.groups()[0]
1598 
1599  realstep = "Unknown-step"
1600  if "PILEUP" in step:
1601  found = pureg.search(step)
1602  if found:
1603  realstep = found.groups()[0]
1604  if createPURow:
1605  createPURow = False
1606  puRow = Row(fsize_tab)
1607  try:
1608  statinfo = os.stat(rootf)
1609  fsize2 = statinfo.st_size
1610  oldfile = os.path.join(oldpath,"%s_TimeSize" % cand,base)
1611  if "PILEUP" in step:
1612  oldfile = os.path.join(oldpath,"%s_PU_TimeSize" % cand,base)
1613  fsize1 = 0
1614 
1615  if os.path.exists(oldfile):
1616  statinfo = os.stat(oldfile)
1617  fsize1 = statinfo.st_size
1618  else:
1619  print "######DID NOT FIND Previous file (needed for the filesize table): %s"%oldfile
1620 
1621  if createNewRow:
1622  createNewRow = False
1623  curRow = fsize_tab.newRow(cand)
1624 
1625  data_tuple = (fsize1,fsize2)
1626  if "PILEUP" in step:
1627  puRow.addEntry(realstep,data_tuple)
1628  else:
1629  if createNewRow:
1630  createNewRow = False
1631  curRow = fsize_tab.newRow(cand)
1632 
1633  curRow.addEntry(step,data_tuple)
1634  except IOError as detail:
1635  print detail
1636  except OSError as detail:
1637  print detail
1638  if puRow == None:
1639  pass
1640  else:
1641  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1642 
1643  (ordered_keys,table_dict) = fsize_tab.getTable()
1644  cols = len(ordered_keys)
1645 
1646  if len(table_dict) > 1 and cols > 0:
1647  createHTMLtab(INDEX,table_dict,ordered_keys,
1648  "Release ROOT file sizes",
1649  "Table showing previous release ROOT filesizes, fs1, latest sizes, fs2, and the difference between them &#x0394; in (k/M/G) bytes.",
1650  "Filesizes",1)
1651  except IOError as detail:
1652  print detail
1653  except OSError as detail:
1654  print detail
1655  #CPU Time Summary Table
1656  elif cpureg.search(NewFileLine):
1657  #Case of NO REGRESSION
1658  if prevrev == "":
1659  time_tab = Table()
1660 
1661  for cand in Candles:
1662  fname = CandFname[cand]
1663  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*_TimingReport.log" % fname)
1664  logfiles = glob.glob(globpath)
1665  logfiles = map(lambda x: (fname,x), logfiles)
1666  logfiles.sort(cmp=logrep_cmp)
1667  logfiles = map(lambda x: x[1], logfiles)
1668 
1669  stepreg = re.compile("%s_(.*)_TimingReport.log" % fname)
1670  createNewRow = True
1671  curRow = None
1672  createPURow = True
1673  puRow = None
1674  for log in logfiles:
1675  base = os.path.basename(log)
1676  found = stepreg.search(base)
1677  step = "Unknown-step"
1678  if found:
1679  step = found.groups()[0]
1680 
1681  realstep = "Unknown-step"
1682  if "PILEUP" in step:
1683  found = pureg.search(step)
1684  if found:
1685  realstep = found.groups()[0]
1686  if createPURow:
1687  createPURow = False
1688  puRow = Row(time_tab)
1689 
1690  data = cpr.getTimingLogData(log)
1691  mean = 0
1692  i = 0
1693  for evtnum, time in data:
1694  mean += time
1695  i += 1
1696  try:
1697  mean = mean / float(i)
1698  except ZeroDivisionError as detail:
1699  print "WARNING: Could not calculate mean CPU time from log because no events could be parsed", log
1700 
1701  if "PILEUP" in step:
1702  puRow.addEntry(realstep,mean)
1703  else:
1704  if createNewRow:
1705  createNewRow = False
1706  curRow = time_tab.newRow(cand)
1707 
1708  curRow.addEntry(step,mean)
1709  if puRow == None:
1710  pass
1711  else:
1712  time_tab.addRow(puRow,"%s PILEUP" %cand)
1713 
1714  (ordered_keys,table_dict) = time_tab.getTable(1)
1715  cols = len(ordered_keys)
1716 
1717  if len(table_dict) > 1 and cols > 0:
1718  createHTMLtab(INDEX,table_dict,ordered_keys,
1719  "Release CPU times",
1720  "Table showing current release CPU times in secs.",
1721  "CPU Times (s)",3)
1722  #Case of REGRESSION (CPU Time Summary Table)
1723  else:
1724 
1725 
1726  ####
1727  #
1728  # Create the table data structure
1729  #
1730  cpu_time_tab = populateFromTupleRoot("cpu_time_tuple",repdir,"timing-regress.root",pureg)
1731 
1732 
1733  ###########
1734  #
1735  # Create HTML table from table data structure
1736  #
1737 
1738  (ordered_keys,table_dict) = cpu_time_tab.getTable()
1739 
1740  cols = len(ordered_keys)
1741  if len(table_dict) > 1 and cols > 0:
1742  createHTMLtab(INDEX,table_dict,ordered_keys,
1743  "Release CPU times",
1744  "Table showing previous release CPU times, t1, latest times, t2, and the difference between them &#x0394; in secs.",
1745  "CPU Times (s)")
1746 
1747 
1748 
1749  elif lpathreg.search(NewFileLine):
1750  INDEX.write(repdir + "\n")
1751  elif proddreg.search(NewFileLine):
1752  INDEX.write(ExecutionDate + "\n")
1753  elif logfreg.search(NewFileLine):
1754  INDEX.write("<br />\n")
1755  for log in LogFiles:
1756  log = os.path.basename(log)
1757  if _verbose:
1758  print "linking log file %s" % log
1759  INDEX.write("<a href=\"./%s\"> %s </a>" % (log,log))
1760  INDEX.write("<br />\n")
1761  #Add the cmsScimark results here:
1762  INDEX.write("Results for cmsScimark2 benchmark (running on the other cores) available at:\n")
1763  for cmssci in cmsScimarkResults:
1764  scimarkdirs=cmssci.split("/")
1765  localdir=scimarkdirs[-2]
1766  #print localdir
1767  cmssci = os.path.basename(cmssci)
1768  relativelink=os.path.join(localdir,cmssci)
1769  #print relativelink
1770  INDEX.write("<a href=\"%s\"> %s </a>" % (relativelink,cmssci))
1771  INDEX.write("<br />\n")
1772 
1773 
1774  elif dirbreg.search(NewFileLine):
1775  #Create a subdirectory DirectoryBrowsing to circumvent the fact the dir is not browsable if there is an index.html in it.
1776  #Bug! This does not work since it points to index.html automatically!
1777  #os.symlink("./","%s/DirectoryBrowsing" % WebArea)
1778  #Actually all the following is done later...
1779  #Create a physical directory first
1780  #os.mkdir("%s/DirectoryBrowsing" % WebArea)
1781  #Then will populate it with symbolic links(once they have been copied there!) from all WebArea files, except index.html, see down towards the end!
1782  INDEX.write("Click <a href=\"./DirectoryBrowsing/\">here</a> to browse the directory containing all results (except the root files)\n")
1783 
1784  elif pubdreg.search(NewFileLine):
1785  INDEX.write(date + "\n")
1786  elif candhreg.search(NewFileLine):
1787  for acandle in Candle:
1788  globpath = os.path.join(repdir,"%s_*" % acandle)
1789  globs = glob.glob(globpath)
1790  if len(globs) > 0:
1791  candlHTML = "%s.html" % acandle
1792  INDEX.write("<a href=\"./%s\"> %s </a>" % (candlHTML,acandle))
1793  INDEX.write("<br />\n")
1794 
1795  candlHTML=os.path.join(WebArea,candlHTML)
1796  createCandlHTML(CandlTmpltHTML,candlHTML,acandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
1797  else:
1798  INDEX.write(NewFileLine)
1799 
1800  #End of while loop on template html file
1801  INDEX.close()
1802  except IOError as detail:
1803  print "Error: Could not create index Html file for some reason, check position. Details : %s" % detail
1804 
1805 ########################
1806 #
1807 # Grab dirs that end in strings defined in DirName
1808 #
1809 def getDirnameDirs(repdir,WebArea):
1810  Dir = os.listdir(repdir)
1811  def _containsDirName(elem):
1812  return reduce(lambda x,y: x or y,map(lambda x: x in elem, DirName))
1813  def _print4Lambda(elem,WebArea):
1814  if _verbose:
1815  print "Copying %s to %s\n" % (elem,WebArea)
1816 
1817  dirstocp = filter(lambda x: _containsDirName(x),map(lambda x: repdir + x,Dir))
1818  map(lambda x: _print4Lambda(x,WebArea),dirstocp)
1819  syscp(dirstocp,WebArea + "/")
1820  os.mkdir("%s/DirectoryBrowsing" % WebArea)
1821  for file in os.listdir(WebArea):
1822  if file != "index.html": #Potential maintenance issue if the index.html changes name to something the server automatically displays when pointing to the directory...
1823  #Use relative path ".." instead of WebArea to avoid problems when copying stuff to a remote server!
1824  os.symlink("%s/%s"%("..",file),"%s/DirectoryBrowsing/%s" % (WebArea,file))
1825 
1826 #######################
1827 #
1828 # Upload stage to remote location
1829 def syncToRemoteLoc(stage,drive,path,port):
1830  stage = addtrailingslash(stage)
1831  cmd = "rsync -avz"
1832  # We must, MUST, do os.path.normpath otherwise rsync will dump the files in the directory
1833  # we specify on the remote server, rather than creating the CMS_VERSION directory
1834  #--rsh=\"ssh -l relval\"
1835  args = "--port=%s %s %s:%s" % (port,os.path.normpath(stage),drive,path)
1836  retval = -1
1837  if _dryrun:
1838  print cmd + " --dry-run " + args
1839  retval = os.system(cmd + " --dry-run " + args )
1840  else:
1841  print cmd+" "+args
1842  retval = os.system(cmd + " " + args)
1843  return retval
1844 
1845 ################
1846 #
1847 # Delete tmp dir if we used it
1848 def delTmpDir():
1849  if os.path.exists(TMP_DIR) and IS_TMP:
1850  os.system("rm -Rf " + TMP_DIR)
1851 
1852 #####################
1853 #
1854 # Some functions used for copying
1855 
1856 def getRelativeDir(parent,child,keepTop=True):
1857  def _walkpath(path):
1858  dirs = []
1859  while True:
1860  head , tail = os.path.split(path)
1861  if tail == "":
1862  break
1863  dirs.append(tail)
1864  path = head
1865  for i in range(len(dirs)-1,-1,-1):
1866  adir = dirs[i]
1867  yield adir
1868  return
1869  pwalk = _walkpath(parent)
1870  n = 0
1871  try:
1872  while True:
1873  next(pwalk)
1874  n += 1
1875  except StopIteration:
1876  pass
1877 
1878  if keepTop:
1879  n = n - 1
1880 
1881  cwalk = _walkpath(child)
1882  try:
1883  #prewalk
1884  for x in range(n):
1885  next(cwalk)
1886  except StopIteration:
1887  print "ERROR: Unable to determine relative dir"
1888  raise ReldirExcept
1889 
1890  relpath = ""
1891  try:
1892  while True:
1893  relpath=os.path.join(relpath,next(cwalk))
1894  except StopIteration:
1895  pass
1896  return relpath
1897 
1898 def docopy(src,dest):
1899  try:
1900  copy2(src,dest)
1901  except OSError as detail:
1902  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1903  except IOError as detail:
1904  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1905  else:
1906  if _verbose:
1907  print "Copied %s to %s" % (src,dest)
1908 
1909 def copytree4(src,dest,keepTop=True):
1910  def _getNewLocation(source,child,dst,keepTop=keepTop):
1911  place = getRelativeDir(source,child,keepTop=keepTop)
1912  return os.path.join(dst,place)
1913  def _copyFilter(source,dst,curdir,fsnodes,filter,dirs=False):
1914  for node in fsnodes:
1915  dontFilter = True
1916  filterExist = not len(filter) == 0
1917  if filterExist:
1918  dontFilter = not reduce(lambda x,y: x or y,map(lambda x: fnmatch.fnmatch(node,x),filter))
1919  if dontFilter:
1920  node = os.path.join(curdir,node) # convert to absolute path
1921  try:
1922  newnode = _getNewLocation(source,node,dst)
1923  if dirs:
1924  os.mkdir(newnode)
1925  else:
1926  copy2(node,newnode)
1927  except IOError as detail:
1928  print "WARNING: Could not copy %s to %s because %s" % (node,newnode,detail)
1929  except OSError as detail:
1930  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1931  except ReldirExcept:
1932  print "WARNING: Could not determine new location for source %s into destination %s" % (source,dst)
1933  else:
1934  if len(filter) > 0:
1935  try:
1936  match = fnmatch.fnmatch(node,filter[0])
1937  assert not match
1938  except AssertionError as detail:
1939  print node, filter[0], match
1940  raise RuntimeError
1941  if _verbose:
1942  if "root" in node:
1943  print "Filter %s Copied %s to %s" % (dontFilter,node,newnode)
1944  print "fnmatch %s" % fnmatch.fnmatch(node,cpFileFilter[0])
1945 
1946  gen = os.walk(src)
1947  try:
1948  newloc = _getNewLocation(src,src,dest)
1949 
1950  os.mkdir(newloc)
1951  try:
1952  while True:
1953  step = next(gen)
1954  curdir = step[0]
1955  dirs = step[1]
1956  files = step[2]
1957 
1958  _copyFilter(src,dest,curdir,dirs,cpDirFilter,dirs=True)
1959  _copyFilter(src,dest,curdir,files,cpFileFilter)
1960 
1961  except StopIteration:
1962  pass
1963  except IOError as detail:
1964  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1965  except OSError as detail:
1966  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1967  except ReldirExcept:
1968  print "WARNING: Could not determine the new location for source %s into destination %s" % (src,dest)
1969 
1970 def syscp(srcs,dest):
1971  if type(srcs) == type(""):
1972  if os.path.exists(srcs):
1973  if os.path.isdir(srcs):
1974  copytree4(srcs,dest)
1975  else:
1976  docopy(srcs,dest)
1977  else:
1978  print "ERROR: file to be copied %s does not exist" % foo
1979  else:
1980  for src in srcs:
1981  if os.path.exists(src):
1982  if os.path.isdir(src):
1983  #copy tree
1984  copytree4(src,dest)
1985  else:
1986  docopy(src,dest)
1987  else:
1988  print "ERROR: file to be copied %s does not exist" % foo
1989 
1990 def print_header():
1991  print "%s\n" % PROG_NAME
1992 
1993 if __name__ == "__main__":
1994  main()
cmsPerfPublish.rootfile_cmp
def rootfile_cmp(x, y)
Definition: cmsPerfPublish.py:716
cmsPerfPublish.optionparse
def optionparse()
Option parser.
Definition: cmsPerfPublish.py:332
cmsPerfPublish.getcmd
def getcmd(command)
Definition: cmsPerfPublish.py:84
cmsPerfPublish.docopy
def docopy(src, dest)
Definition: cmsPerfPublish.py:1898
cmsPerfPublish.timerep_cmp
def timerep_cmp(x, y)
Definition: cmsPerfPublish.py:764
cmsPerfPublish.getcmdBasic
def getcmdBasic(cmd)
Definition: cmsPerfPublish.py:79
cmsPerfPublish.Table.addRow
def addRow(self, row, name)
Definition: cmsPerfPublish.py:179
cmsPerfPublish.syscp
def syscp(srcs, dest)
Definition: cmsPerfPublish.py:1970
cmsPerfPublish.prettySize
def prettySize(size)
Definition: cmsPerfPublish.py:91
cmsPerfPublish.dirname_cmp
def dirname_cmp(x, y)
Definition: cmsPerfPublish.py:728
GetRecoTauVFromDQM_MC_cff.next
next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
cmsPerfPublish.getArchVersionFromLog
def getArchVersionFromLog(logfile)
Definition: cmsPerfPublish.py:476
cmsPerfPublish.stageIgProfReports
def stageIgProfReports(remotedir, arch, version)
Definition: cmsPerfPublish.py:1395
cmsPerfPublish.reg_dirname_cmp
def reg_dirname_cmp(x, y)
Definition: cmsPerfPublish.py:740
cmsPerfPublish.copytree4
def copytree4(src, dest, keepTop=True)
Definition: cmsPerfPublish.py:1909
cmsPerfPublish.Table._newCol
def _newCol(self, name)
Definition: cmsPerfPublish.py:133
cmsPerfPublish.get_environ
def get_environ()
Get require environment variables.
Definition: cmsPerfPublish.py:302
cmsPerfPublish.syncToRemoteLoc
def syncToRemoteLoc(stage, drive, path, port)
Upload stage to remote location.
Definition: cmsPerfPublish.py:1829
cmsPerfPublish.scanReportArea
def scanReportArea(repdir)
Scan report area for required things.
Definition: cmsPerfPublish.py:642
cmsPerfPublish.getOutputNames
def getOutputNames(base, reportName)
Definition: cmsPerfPublish.py:708
cmsPerfPublish.createRegressHTML
def createRegressHTML(reghtml, repdir, outd, CurrentCandle, htmNames)
Definition: cmsPerfPublish.py:683
cmsPerfPublish.addtrailingslash
def addtrailingslash(adir)
Definition: cmsPerfPublish.py:70
cmsPerfPublish.getStageRepDirs
def getStageRepDirs(options, args)
Determine locations of staging and report dirs.
Definition: cmsPerfPublish.py:495
digi_MixPreMix_cfi.strip
strip
Definition: digi_MixPreMix_cfi.py:15
cmsPerfPublish.logrep_cmp
def logrep_cmp(x, y)
Definition: cmsPerfPublish.py:752
genParticles_cff.map
map
Definition: genParticles_cff.py:11
cmsPerfPublish.createCandlHTML
def createCandlHTML(tmplfile, candlHTML, CurrentCandle, WebArea, repdir, ExecutionDate, LogFiles, cmsScimarkResults, date, prevrev)
Create HTML pages for candles.
Definition: cmsPerfPublish.py:846
cmsPerfPublish.Table.getTable
def getTable(self, mode=0)
Definition: cmsPerfPublish.py:151
cmsPerfPublish.Table.newRow
def newRow(self, name)
Definition: cmsPerfPublish.py:142
cmsPerfPublish.getNumOfEventsFromLog
def getNumOfEventsFromLog(logfile)
Definition: cmsPerfPublish.py:451
ALCARECOTkAlBeamHalo_cff.filter
filter
Definition: ALCARECOTkAlBeamHalo_cff.py:26
reco::return
return(e1-e2)*(e1-e2)+dp *dp
cmsPerfPublish.Row.addEntry
def addEntry(self, colname, value)
Definition: cmsPerfPublish.py:114
cmsPerfPublish.createHTMLtab
def createHTMLtab(INDEX, table_dict, ordered_keys, header, caption, name, mode=0)
Definition: cmsPerfPublish.py:1304
main
Definition: main.py:1
cmsPerfPublish.delTmpDir
def delTmpDir()
Delete tmp dir if we used it.
Definition: cmsPerfPublish.py:1848
cmsPerfPublish.createWebReports
def createWebReports(WebArea, repdir, ExecutionDate, LogFiles, cmsScimarkResults, date, prevrev)
Create web report index and create HTML file for each candle.
Definition: cmsPerfPublish.py:1448
cmsPerfPublish.step_cmp
def step_cmp(x, y)
Definition: cmsPerfPublish.py:776
cmsPerfPublish.fail
def fail(errstr="")
Definition: cmsPerfPublish.py:65
cmsPerfPublish.Row.__init__
def __init__(self, table)
Definition: cmsPerfPublish.py:107
resolutioncreator_cfi.object
object
Definition: resolutioncreator_cfi.py:4
cmsPerfPublish.populateFromTupleRoot
def populateFromTupleRoot(tupname, repdir, rootfile, pureg)
Definition: cmsPerfPublish.py:1240
cmsPerfPublish.getRelativeDir
def getRelativeDir(parent, child, keepTop=True)
Some functions used for copying.
Definition: cmsPerfPublish.py:1856
cmsPerfPublish.getDirnameDirs
def getDirnameDirs(repdir, WebArea)
Grab dirs that end in strings defined in DirName.
Definition: cmsPerfPublish.py:1809
Generated for CMSSW Reference Manual by doxygen 1.8.11