CMS 3D CMS Logo
cmsPerfPublish.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #G.Benelli Jan 22 2007, J. Nicolson 12 Aug 2008
3 #A little script to move Simulation Performance Suite
4 #relevant html and log files into our public area
5 #/afs/cern.ch/cms/sdt/web/performance/simulation/
6 #Set here the standard number of events (could become an option... or could be read from the log...)
7 
8 ###############################
9 #
10 # Warning!!! We should use copytree() instead of our self defined copytree4
11 # However, copytree does not use ignore patterns (for filtering files)
12 # before python v2.6, when we upgrade to python 2.6 we should use this
13 # functionality.
14 from __future__ import print_function
15 from builtins import range
16 import tempfile as tmp
17 import optparse as opt
18 import cmsPerfRegress as cpr
19 import re, os, sys, time, glob, socket, fnmatch
20 from shutil import copy2, copystat
21 from stat import *
22 from cmsPerfCommons import CandFname, Step, ProductionSteps, Candles
23 import ROOT
24 #Switching from os.popen4 to subprocess.Popen for Python 2.6 (340_pre5 onwards):
25 import subprocess
26 from functools import reduce
27 
28 PROG_NAME = os.path.basename(sys.argv[0])
29 DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal"
30 DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation"
31 TMP_DIR = ""
32 cpFileFilter = ( "*.root", ) # Unix pattern matching not regex
33 cpDirFilter = ( ) # Unix pattern matching not regex
34 
35 TimeSizeNumOfEvents = -9999
36 IgProfNumOfEvents = -9999
37 CallgrindNumOfEvents = -9999
38 MemcheckNumOfEvents = -9999
39 
40 DirName=( #These need to match the candle directory names ending (depending on the type of profiling)
41  "TimeSize",
42  "IgProf",
43  "IgProf_Perf",
44  "IgProf_Mem",
45  "Callgrind",
46  "Memcheck",
47  #Adding the extra PU directories:
48  "PU_TimeSize",
49  "PU_IgProf",
50  "PU_IgProf_Perf",
51  "PU_IgProf_Mem",
52  "PU_Callgrind",
53  "PU_Memcheck"
54  )
55 #Defining Steps as a union of Step and ProductionSteps:
56 Steps=set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"]) #Adding GEN,FASTSIM too by hand.
57 print(Steps)
58 
59 ##################
60 #
61 # Small functions
62 #
63 
64 class ReldirExcept(Exception):
65  "Relative directory could not be determined"
66 
67 def fail(errstr=""):
68  print(errstr)
69  delTmpDir()
70  sys.exit()
71 
72 def addtrailingslash(adir):
73  trail = re.compile("/$")
74  if os.path.isdir(adir) and not trail.search(adir):
75  adir = adir + "/"
76  return adir
77 
78 def getDate():
79  return time.ctime()
80 
81 def getcmdBasic(cmd):
82  #Obsolete popen4-> subprocess.Popen
83  #return os.popen4(cmd)[1].read().strip()
84  return subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
85 
86 def getcmd(command):
87  if _debug > 2:
88  print(command)
89  #Obsolete popen4-> subprocess.Popen
90  #return os.popen4(command)[1].read().strip()
91  return subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
92 
93 def prettySize(size):
94  nega = size < 0
95  if nega:
96  size = -size
97  suffixes = [("B",2**10), ("k",2**20), ("M",2**30), ("G",2**40), ("T",2**50)]
98  for suf, lim in suffixes:
99  if size > lim:
100  continue
101  else:
102  if nega:
103  return "-" + round(size/float(lim/2**10),2).__str__() + suf
104  else:
105  return round(size/float(lim/2**10),2).__str__()+suf
106 
107 class Row(object):
108 
109  def __init__(self,table):
110  self.table = table
111  self.coldata = {}
112 
113  def __str__(self):
114  return str(self.coldata)
115 
116  def addEntry(self,colname,value):
117  self.table._newCol(colname)
118  self.coldata[colname] = value
119 
120  def getRowDict(self):
121  return self.coldata
122 
123 class Table(object):
124 
125  def __init__(self):
126  self.colNames = []
127  self.keys = [None]
128  self.rows = {None: self.colNames}
129 
130  def __str__(self):
131  out = self.keys
132  out += "\n" + self.rows
133  return out
134 
135  def _newCol(self,name):
136  if name in self.colNames:
137  pass
138  else:
139  self.colNames.append(name)
140 
141  def getCols(self):
142  return self.colNames
143 
144  def newRow(self,name):
145  if name in self.rows.keys():
146  pass
147  else:
148  self.keys.append(name)
149  self.rows[name] = Row(self)
150 
151  return self.rows[name]
152 
153  def getTable(self,mode=0):
154  name = "Total"
155 
156  for key in self.keys:
157  if key == None:
158  pass
159  else:
160  total1 = 0
161  total2 = 0
162  rowobj = self.rows[key]
163  rowdict = rowobj.getRowDict()
164  for col in self.colNames:
165  if col == None:
166  pass
167  elif col in rowdict and not col == name:
168  if mode == 1:
169  total1 += rowdict[col]
170  else:
171  (step_tot1, step_tot2) = rowdict[col]
172  total1 += step_tot1
173  total2 += step_tot2
174  if mode == 1:
175  rowobj.addEntry(name,total1)
176  else:
177  rowobj.addEntry(name,(total1,total2))
178 
179  return (self.keys, self.rows)
180 
181  def addRow(self,row,name):
182  if name in self.rows.keys():
183  pass
184  else:
185  self.keys.append(name)
186  self.rows[name] = row
187 
188  def transpose(self):
189  transp = Table()
190  for col in self.colnames:
191  rowobj = transp.newRow(col)
192  for key in self.keys:
193  if key == None:
194  pass
195  else:
196  row_dict = self.rows[key].getRowDict()
197  if key in row_dict:
198  rowobj.addEntry(key,row_dict[col])
199  return transp
200 
201 ######################
202 #
203 # Main
204 #
205 def main():
206  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
207  #Bad design... why is this function defined here?
208  #Either no function, or define somewhere else.
209  def _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage):
210  """Use function syscp to copy LogFiles and cmsScimarkDir over to staging area"""
211  if _verbose:
212  print("Copying the logfiles to %s/." % stage)
213  print("Copying the cmsScimark2 results to the %s/." % stage)
214 
215  syscp(LogFiles , stage + "/")
216  syscp(cmsScimarkDir, stage + "/")
217 
218  #Same comment as above about the opportunity of this function definition here.
219  def _createLogFile(LogFile,date,LocalPath,ShowTagsResult):
220  """Creating a small logfile with basic publication script running information (never used really by other scripts in the suite)."""
221  try:
222  LOG = open(LogFile,"w")
223  if _verbose:
224  print("Writing Production Host, Location, Release and Tags information in %s" % LogFile)
225  LOG.write("These performance tests were executed on host %s and published on %s" % (HOST,date))
226  LOG.write("They were run in %s" % LocalPath)
227  LOG.write("Results of showtags -r in the local release:\n%s" % ShowTagsResult)
228  LOG.close()
229  except IOError as detail:
230  print("WARNING: Can't create log file")
231  print(detail)
232 
233  # Print Program header
234  print_header()
235 
236  # Get environment variables
237  #FIXME: should check into this and make sure the proper variables for the tests being published are read from logfile (case of deprecated releases...)
238  print("\n Getting Environment variables...")
239  (LocalPath, ShowTagsResult) = get_environ()
240 
241  # Parse options
242  (options,args) = optionparse()
243 
244  # Determine program parameters and input/staging locations
245  print("\n Determining locations for input and staging...")
246  (drive,path,remote,stage,port,repdir,prevrev,igprof_remotedir) = getStageRepDirs(options,args)
247 
248  #Get the number of events for each test from logfile:
249  print("\n Getting the number of events for each test...")
250  #Let's do a quick implementation of something that looks at the logfile:
251  cmsPerfSuiteLogfile="%s/cmsPerfSuite.log"%repdir
252 
253  if os.path.exists(cmsPerfSuiteLogfile):
254  try:
255  (TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents)=getNumOfEventsFromLog(cmsPerfSuiteLogfile)
256  #Get the CMSSW version and SCRAM architecture from log (need these for the IgProf new publishing with igprof-navigator)
257  (CMSSW_arch,CMSSW_version)=getArchVersionFromLog(cmsPerfSuiteLogfile)
258  #For now keep the dangerous default? Better set it to a negative number...
259  except:
260  print("There was an issue in reading out the number of events for the various tests or the architecture/CMSSW version using the standard logfile %s"%cmsPerfSuiteLogFile)
261  print("Check that the format was not changed: this scripts relies on the initial perfsuite arguments to be dumped in the logfile one per line!")
262  print("For now taking the default values for all tests (0)!")
263 
264  print("\n Scan report directory...")
265  # Retrieve some directories and information about them
266  (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir) = scanReportArea(repdir)
267  print("cmsScimarkResults are %s"%cmsScimarkResults)
268  print("\n Copy report files to staging directory...")
269  # Copy reports to staging area
270  _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage)
271 
272  print("\n Creating log file...")
273  # Produce a small logfile with basic info on the Production area
274  _createLogFile("%s/ProductionLog.txt" % stage,date,repdir,ShowTagsResult)
275 
276  #Check if there are IgProf tests:
277  for dirname in os.listdir(repdir):
278  if "IgProf" in dirname:
279  print("\n Handling IgProf reports...")
280  # add a function to handle the IgProf reports
281  stageIgProfReports(igprof_remotedir,CMSSW_arch,CMSSW_version)
282 
283  print("\n Creating HTML files...")
284  # create HTML files
285  createWebReports(stage,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
286 
287  print("\n Copy profiling logs to staging directory...")
288  # Copy over profiling logs...
289  getDirnameDirs(repdir,stage)
290 
291  # Send files to remote location
292  if remote:
293  print("\n Uploading web report to remote location...")
294  syncToRemoteLoc(stage,drive,path,port)
295  print("\n Finished uploading! Now removing staging directory...")
296  delTmpDir()
297 
298  print("\n Finished!!!")
299 
300 ##########################
301 #
302 # Get require environment variables
303 #
304 def get_environ():
305  global CMSSW_VERSION, CMSSW_RELEASE_BASE, CMSSW_BASE, HOST, USER, BASE_PERFORMANCE, CMSSW_WORK
306  global DEF_RELVAL, DEF_SIMUL
307 
308  try:
309  CMSSW_VERSION=os.environ['CMSSW_VERSION']
310  CMSSW_RELEASE_BASE=os.environ['CMSSW_RELEASE_BASE']
311  CMSSW_BASE=os.environ['CMSSW_BASE']
312  HOST=os.environ['HOST']
313  USER=os.environ['USER']
314  CMSSW_WORK = os.path.join(CMSSW_BASE,"work/Results")
315  except KeyError as detail:
316  fail("ERROR: Could not retrieve some necessary environment variables. Have you ran scramv1 runtime -csh yet?. Details: %s" % detail)
317 
318  LocalPath=getcmdBasic("pwd")
319  ShowTagsResult=getcmdBasic("showtags -r")
320 
321  #Adding a check for a local version of the packages
322  PerformancePkg="%s/src/Validation/Performance" % CMSSW_BASE
323  if (os.path.exists(PerformancePkg)):
324  BASE_PERFORMANCE=PerformancePkg
325  print("**Using LOCAL version of Validation/Performance instead of the RELEASE version**")
326  else:
327  BASE_PERFORMANCE="%s/src/Validation/Performance" % CMSSW_RELEASE_BASE
328 
329  return (LocalPath,ShowTagsResult)
330 
331 ###############
332 #Option parser
333 #
334 def optionparse():
335  global PROG_NAME, _debug, _dryrun, _verbose
336 
337  parser = opt.OptionParser(usage=("""%s [HOST:]DEST_PATH [Options]
338 
339  Arguments:
340  [HOST:]DEST_PATH - This is where the report should be published, you can specify a local path or a directory on a remote machine (HOST is optional)
341 
342  Examples:
343  Publish report to default local directory
344  ./%s
345  Publish report to \"/some/local/dir\"
346  ./%s /some/local/dir
347  Publish report to \"/some/other/dir\" remote host \"hal.cern.ch\"
348  ./%s hal.cern.ch:/some/other/dir
349  Publish report to default relval location (this could be remote or local depending on the hardcoded default)
350  ./%s --relval"""
351  % ( PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME)))
352 
353  devel = opt.OptionGroup(parser, "Developer Options",
354  "Caution: use these options at your own risk."
355  "It is believed that some of them bite.\n")
356  parser.add_option(
357  '--relval',
358  action="store_true",
359  dest='relval',
360  help='Use the default RelVal location',
361  #metavar='<STEPS>',
362  )
363 
364  parser.add_option(
365  '-v',
366  '--verbose',
367  action="store_true",
368  dest='verbose',
369  help='output more information',
370  #metavar='<STEPS>',
371  )
372 
373  parser.add_option(
374  '--simul',
375  action="store_true",
376  dest='simulation',
377  help='Use the default simulation location',
378  #metavar='<STEPS>',
379  )
380 
381  parser.add_option(
382  '--prev',
383  type="string",
384  dest='previousrev',
385  help='The override the name of the previous release. Default is the string obtain from the identification file REGRESSION.<prevrel>.vs.<newrel>',
386  metavar='<NAME>',
387  default="",
388  )
389 
390  parser.add_option(
391  '--input',
392  type="string",
393  dest='repdir',
394  help='The location of the report files to be published',
395  metavar='<DIR>'
396  )
397 
398  parser.add_option(
399  '-p',
400  '--port',
401  type='int',
402  dest='port',
403  help='Use a particular port number to rsync material to a remote server',
404  metavar='<PORT>'
405  )
406  parser.add_option(
407  '--igprof',
408  type='string',
409  dest='ig_remotedir',
410  default='IgProfData', #Reverting to local default publication for now#'/afs/cern.ch/cms/sdt/web/qa/igprof-testbed/data', #For now going straight into AFS... later implement security via local disk on cmsperfvm and cron job there:
411  #default='cmsperfvm:/data/projects/conf/PerfSuiteDB/IgProfData', #This should not change often! In this virtual machine a cron job will run to move stuff to AFS.
412  help='Specify an AFS or host:mydir remote directory instead of default one',
413  metavar='<IGPROF REMOTE DIRECTORY>'
414  )
415 
416  devel.add_option(
417  '-d',
418  '--debug',
419  type='int',
420  dest='debug',
421  help='Show debug output',
422  #metavar='DEBUG',
423  )
424 
425  devel.add_option(
426  '--dry-run',
427  action="store_true",
428  dest='dryrun',
429  help='Do not send files to remote server, but run everything else',
430  #metavar='DEBUG',
431  )
432 
433  repdirdef = os.getcwd()
434  parser.set_defaults(debug=0,simulation=False,relval=False,port=873,pretend=False,repdir=repdirdef,verbose=False)
435  parser.add_option_group(devel)
436 
437  (options, args) = parser.parse_args()
438 
439  _debug = options.debug
440  _dryrun = options.dryrun
441  _verbose = options.verbose
442 
443  numofargs = len(args)
444 
445  if (options.simulation and options.relval) or ((options.simulation or options.relval) and numofargs >= 1):
446  parser.error("You can not specify simulation and relval together. Neither can you specify simulation or relval AND a path")
447  sys.exit()
448 
449  return (options, args)
450 
451 
452 #A function to get the number of events for each test from the logfile:
453 def getNumOfEventsFromLog(logfile):
454  '''A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!'''
455  log=open(logfile,"r")
456  TimeSizeEvents=0
457  IgProfEvents=0
458  CallgrindEvents=0
459  MemcheckEvents=0
460  for line in log:
461  #FIXME:
462  #For robustness could read this from the Launching the X tests (.....) with N events each and keep that format decent for parsing.
463  #One more place where XML would seem a better choice to extract information (Timing of the performance suite in general is also, but publishing and harvesting some other info as well).
464  if 'TimeSizeEvents' in line and not TimeSizeEvents:
465  lineitems=line.split()
466  TimeSizeEvents=lineitems[lineitems.index('TimeSizeEvents')+1]
467  if 'IgProfEvents' in line and not IgProfEvents:
468  lineitems=line.split()
469  IgProfEvents=lineitems[lineitems.index('IgProfEvents')+1]
470  if 'CallgrindEvents' in line and not CallgrindEvents:
471  lineitems=line.split()
472  CallgrindEvents=lineitems[lineitems.index('CallgrindEvents')+1]
473  if 'MemcheckEvents' in line and not MemcheckEvents:
474  lineitems=line.split()
475  MemcheckEvents=lineitems[lineitems.index('MemcheckEvents')+1]
476  return (TimeSizeEvents,IgProfEvents,CallgrindEvents,MemcheckEvents)
477 
478 def getArchVersionFromLog(logfile):
479  '''Another very fragile function to get the architecture and the CMSSW version parsing the logfile...'''
480  log=open(logfile,"r")
481  arch=re.compile("^Current Architecture is")
482  version=re.compile("^Current CMSSW version is")
483  CMSSW_arch="UNKNOWN_ARCH"
484  CMSSW_version="UNKNOWN_VERSION"
485  for line in log:
486  if arch.search(line):
487  CMSSW_arch=line.split()[3]
488  if version.search(line):
489  CMSSW_version=line.split()[4]
490  return(CMSSW_arch,CMSSW_version)
491 
492 
493 #####################
494 #
495 # Determine locations of staging and report dirs
496 #
497 def getStageRepDirs(options,args):
498  global TMP_DIR, IS_TMP, DEF_LOCAL, CMSSW_VERSION
499  DEF_LOCAL = CMSSW_WORK
500  numofargs = len(args)
501 
502  repdir = os.path.abspath(options.repdir)
503  repdir = addtrailingslash(repdir)
504 
505  if not os.path.exists(repdir):
506  fail("ERROR: The specified report directory %s to retrieve report information from does not exist, exiting" % repdir)
507 
508  previousrev = options.previousrev
509  if previousrev == "":
510  regressfiles = glob.glob("%s/REGRESSION.*.vs.*" % repdir)
511  if not len(regressfiles) == 0:
512  regressID = regressfiles[0]
513  base = os.path.basename(regressID)
514  split = base.split(".")
515  previousrev = split[1]
516  currentrel = split[3]
517  print("Regression Identification file exists, renaming report title for regression report. Old ver: %s" % previousrev)
518  else:
519  print("No regression ID file exists and previous release name was not specified. Producing normal report.")
520  else:
521  print("Previous release name was specified, renaming report title for regression report. Old ver %s" % previousrev)
522 
523  uri = ""
524  defaultlocal = False
525  if options.simulation:
526  uri = DEF_SIMUL
527  elif options.relval:
528  uri = DEF_RELVAL
529  elif numofargs >= 1:
530  uri = args[0] # Add directory CMSSW_VERSION later in temp! Not now, otherwise we get into a mess if this is a remote dir
531  else:
532  defaultlocal = True
533  uri = DEF_LOCAL
534 
535  ####
536  #
537  # Determine if location is remote
538  #
539  # Try not to re-arrange we don't want to assume that default locations are not remote
540  #
541  ####
542 
543  drive = ""
544  path = ""
545  if ":" in uri:
546  drive, path = uri.split(":",1)
547  else:
548  path = uri
549 
550  if drive == "":
551  path = os.path.abspath(path)
552  remote = not drive == ""
553 
554  if remote:
555  unResolved = True
556  try:
557  socket.getaddrinfo(drive,53)
558  unResolved = False
559  except socket.gaierror:
560  unResolved = True
561 
562  # try see if it's an ipaddress
563  if unResolved:
564  try:
565  socket.gethostbyaddr(drive)
566  unResolved = False
567  except socket.gaierror:
568  unResolved = True
569  if unResolved:
570  print("ERROR: Can not determine your hostname or ipv{4,6} address %s" % drive)
571  if not (_dryrun or _test):
572  fail("exiting...")
573 
574  if (not remote) and (not options.port == 873) :
575  print("WARNING: Can not use a port if not performing a remote copy, ignoring")
576  port = options.port
577 
578  ###
579  #
580  # Determine Staging Area
581  #
582 
583  StagingArea=""
584  localExists = os.path.exists("%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
585 
586  if remote:
587  #Cannot use this since the /tmp is limited to 2GB on lxbuild machines!
588  #TMP_DIR=tmp.mkdtemp(prefix="/tmp/%s" % PROG_NAME)
589  TMP_DIR=tmp.mkdtemp(prefix="/build/%s" % PROG_NAME)
590  StagingArea = TMP_DIR
591  #Local but dir already exists
592  elif defaultlocal and localExists:
593  TMP_DIR=tmp.mkdtemp(prefix="%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
594  StagingArea = TMP_DIR
595  print("WARNING: %s already exists, creating a temporary staging area %s" % (CMSSW_WORK,TMP_DIR))
596  #Local cases
597  elif defaultlocal:
598  StagingArea = CMSSW_WORK
599  try:
600  os.mkdir(os.path.join(CMSSW_BASE,"work"))
601  os.mkdir(os.path.join(CMSSW_BASE,"work","Results"))
602  except OSError:
603  pass
604  print("**User did not specify location of results, staging in default %s**" % StagingArea)
605  else:
606  print("**User chose to publish results in a local directory**")
607  StagingArea = path
608  if not os.path.exists(path):
609  try:
610  os.mkdir("%s" % path)
611  except OSError as detail:
612  if detail.errno == 13:
613  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
614  elif detail.errno == 17:
615  #If the directory already exists just carry on
616  pass
617  else:
618  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
619 
620  IS_TMP = not TMP_DIR == ""
621  ######
622  #
623  # create Version dir
624  # This is why when we create a Tmp dir we get the version(tmpname)/version
625  # structure. We should remove this if we don't want it but i dont think it matters
626  StagingArea="%s/%s" % (StagingArea,CMSSW_VERSION)
627  try:
628  os.mkdir("%s" % StagingArea)
629  except OSError as detail:
630  if detail.errno == 13:
631  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
632  elif detail.errno == 17:
633  #If the directory already exists just carry on
634  pass
635  else:
636  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
637 
638  return (drive,path,remote,StagingArea,port,repdir,previousrev,options.ig_remotedir)
639 
640 ####################
641 #
642 # Scan report area for required things
643 #
644 def scanReportArea(repdir):
645  """Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results.
646  It returns Execution date (completion), current date, list of logfiles and cmsScimark results"""
647  date=getDate()
648  LogFiles = glob.glob(repdir + "cms*.log")
649  if _verbose:
650  print("Found the following log files:")
651  print(LogFiles)
652 
653  cmsScimarkDir = glob.glob(repdir + "cmsScimarkResults_*")
654  if _verbose:
655  print("Found the following cmsScimark2 results directories:")
656  print(cmsScimarkDir)
657 
658  cmsScimarkResults = []
659  for adir in cmsScimarkDir:
660  htmlfiles = glob.glob(adir + "/*.html")
661  #FIXME:
662  #Really unnecessary use of map in my opinion
663  #Could do with cmsScimarkResults.extend(htmlfiles)
664  map(cmsScimarkResults.append,htmlfiles)
665 
666  ExecutionDateLast = ""
667  ExecutionDate = ""
668  ExecutionDateSec=0
669  cmsreg = re.compile("^cmsPerfSuite")
670  for logf in LogFiles:
671  if cmsreg.search(logf):
672  ExecutionDateLastSec = os.stat(logf)[ST_CTIME]
673  ExecutionDateLast = os.stat(logf)[ST_MTIME]
674  if _verbose:
675  print("Execution (completion) date for %s was: %s" % (logf,ExecutionDateLast))
676  if (ExecutionDateLastSec > ExecutionDateSec):
677  ExecutionDateSec = ExecutionDateLastSec
678  ExecutionDate = ExecutionDateLast
679 
680  if ExecutionDate == "":
681  ExecutionDate = ExecutionDateLast
682 
683  return (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir)
684 
685 def createRegressHTML(reghtml,repdir,outd,CurrentCandle,htmNames):
686  RegressTmplHTML="%s/doc/regress.html" % (BASE_PERFORMANCE)
687  candnreg = re.compile("CandleName")
688  candhreg = re.compile("CandlesHere")
689  try:
690  REGR = open(reghtml,"w")
691  for line in open(RegressTmplHTML):
692  if candhreg.search(line):
693  html = "<table>"
694 
695  for x in htmNames:
696  abspath = os.path.join(repdir,outd)
697  if os.path.exists(abspath):
698  html += "<tr><td><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></td></tr>\n" % (outd,x,outd,x)
699  else:
700  html += "<tr><td> %s does not exist probably because the log file for the previous release was missing</td></tr>" % (abspath)
701  html += "</table>"
702  REGR.write(html)
703  elif candnreg.search(line):
704  REGR.write(CurrentCandle)
705  else:
706  REGR.write(line)
707  except IOError as detail:
708  print("ERROR: Could not write regression html %s because %s" % (os.path.basename(reghtml),detail))
709 
710 def getOutputNames(base,reportName):
711  logreg = re.compile("(.*)\.log$")
712  matches = logreg.search(reportName)
713  logdir = logreg.sub(matches.groups()[0],reportName)
714  outd = os.path.join(base,logdir)
715  nologext = matches.groups()[0]
716  return (nologext,outd)
717 
718 def rootfile_cmp(x,y):
719  (fname,x) = x
720  (fname,y) = y
721  x = os.path.basename(x)
722  y = os.path.basename(y)
723  stepreg = re.compile("%s_(..*)\.root" % fname)
724  if stepreg.search(x):
725  x = stepreg.search(x).groups()[0]
726  if stepreg.search(y):
727  y = stepreg.search(y).groups()[0]
728  return step_cmp(x,y)
729 
730 def dirname_cmp(x,y):
731  (candle,prof,x) = x
732  (candle,prof,y) = y
733  x = os.path.basename(x)
734  y = os.path.basename(y)
735  stepreg = re.compile("%s_(..*)_%s" % (candle,prof))
736  if stepreg.search(x):
737  x = stepreg.search(x).groups()[0]
738  if stepreg.search(y):
739  y = stepreg.search(y).groups()[0]
740  return step_cmp(x,y)
741 
742 def reg_dirname_cmp(x,y):
743  (candle,prof,x) = x
744  (candle,prof,y) = y
745  x = os.path.basename(x)
746  y = os.path.basename(y)
747  stepreg = re.compile("%s_(..*)_%s_regression" % (candle,prof))
748  if stepreg.search(x):
749  x = stepreg.search(x).groups()[0]
750  if stepreg.search(y):
751  y = stepreg.search(y).groups()[0]
752  return step_cmp(x,y)
753 
754 def logrep_cmp(x,y):
755  (fname,x) = x
756  (fname,y) = y
757  x = os.path.basename(x)
758  y = os.path.basename(y)
759  stepreg = re.compile("%s_(..*)_TimingReport.log" % fname)
760  if stepreg.search(x):
761  x = stepreg.search(x).groups()[0]
762  if stepreg.search(y):
763  y = stepreg.search(y).groups()[0]
764  return step_cmp(x,y)
765 
766 def timerep_cmp(x,y):
767  (fname,x) = x
768  (fname,y) = y
769  x = os.path.basename(x)
770  y = os.path.basename(y)
771  stepreg = re.compile("%s_(..*)_TimingReport" % fname)
772  if stepreg.search(x):
773  x = stepreg.search(x).groups()[0]
774  if stepreg.search(y):
775  y = stepreg.search(y).groups()[0]
776  return step_cmp(x,y)
777 
778 def step_cmp(x,y):
779  xstr = x
780  ystr = y
781  x_idx = -1
782  y_idx = -1
783  bestx_idx = -1
784  besty_idx = -1
785  sndbst_x = -1
786  sndbst_y = -1
787  last_x = -1
788  last_y = -1
789  for i in range(len(Step)):
790  stepreg = re.compile("^%s.*" % Step[i])
791  # fallback
792  if Step[i] in xstr and sndbst_x == -1:
793  last_x = i
794  if Step[i] in ystr and sndbst_y == -1:
795  last_y = i
796  # second best
797  if xstr in Step[i] and bestx_idx == -1:
798  sndbst_x = i
799  if ystr in Step[i] and besty_idx == -1:
800  sndbst_y = i
801  # next best
802  if stepreg.search(xstr) and x_idx == -1:
803  bestx_idx = i # If an exact match has not been found but something similar has, set x best index
804  if stepreg.search(ystr) and y_idx == -1:
805  besty_idx = i # If an exact match has not been found but something similar has, set y best index
806  # actual
807  if Step[i] == xstr and x_idx == -1:
808  x_idx = i # if an exact match has been found then set x index
809  if Step[i] == ystr and y_idx == -1:
810  y_idx = i # if an exact match has been found then set y index
811  if not ( x_idx == -1 or y_idx == -1):
812  break # if an exact match has been found for both, stop
813 
814  # use best match if we still can't find indices
815  if x_idx == -1:
816  x_idx = bestx_idx
817  if y_idx == -1:
818  y_idx = besty_idx
819 
820  # use second best if we still can't find indices
821  if x_idx == -1:
822  x_idx = sndbst_x
823  if y_idx == -1:
824  y_idx = sndbst_y
825 
826  # use fallback if we still can't find indices
827  if x_idx == -1:
828  x_idx = last_x
829  if y_idx == -1:
830  y_idx = last_y
831 
832  if x_idx == -1 or y_idx == -1:
833  print("WARNING: No valid step names could be found in the logfiles or root filenames being sorted: x: %s y: %s." % (xstr,ystr))
834  print("x", x_idx, "y", y_idx)
835 
836  if x_idx < y_idx:
837  return -1
838  elif x_idx == y_idx:
839  return 0
840  elif y_idx < x_idx:
841  return 1
842 
843 
844 ######################
845 #
846 # Create HTML pages for candles
847 
848 def createCandlHTML(tmplfile,candlHTML,CurrentCandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
849  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
850  def _stripbase(base, astr):
851  basereg = re.compile("^%s/?(.*)" % base)
852  out = astr
853  found = basereg.search(astr)
854  if found:
855  out = found.groups()[0]
856  return out
857 
858  def _getProfileReportLink(repdir,CurrentCandle,CurDir,step,CurrentProfile,Profiler):
859  #FIXME:
860  #Pileup now has it own directory... should add it in the DirName dictionary at the beginning?
861  ProfileTemplate=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step,CurrentProfile),Profiler)
862  #print ProfileTemplate
863  #There was the issue of SIM vs sim (same for DIGI) between the previous RelVal based performance suite and the current.
864  ProfileTemplateLowCaps=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step.lower(),CurrentProfile),Profiler)
865  ProfileReportLink = glob.glob(ProfileTemplate)
866  #Filter out the Pile up directories when step does not contain Pile up
867  #if not ('PILEUP' in step):
868  # print "NPNPNPNP BEFORE %s"%ProfileReportLink
869  # ProfileReportLink=filter(lambda x: "PU" not in x,ProfileReportLink)
870  # print "NPNPNPNP %s"%ProfileReportLink
871  #print ProfileReportLink
872  if len(ProfileReportLink) > 0:
873  #print ProfileReportLink
874  if not reduce(lambda x,y: x or y,map(lambda x: CurrentCandle in x,ProfileReportLink)):# match with caps try low caps
875  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
876  else:
877  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
878  ProfileReportLink = map(lambda x: _stripbase(repdir,x),ProfileReportLink)
879  #print ProfileReportLink
880  return ProfileReportLink
881 
882  def _writeReportLink(INDEX,ProfileReportLink,CurrentProfile,step,NumOfEvents,Profiler=""):
883  if Profiler == "":
884  INDEX.write("<li><a href=\"%s\">%s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,step,NumOfEvents))
885  else:
886  #FIXME: need to fix this: why do we have the number of Memcheck events hardcoded?
887  if CurrentProfile == "memcheck_valgrind":#FIXME:quick and dirty hack to report we have changed memcheck to 5 events now...
888  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,"5"))
889  else:
890  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,NumOfEvents))
891  def IgProfDumpsTable(INDEX,ProfileLinks,step):
892  #See if the end of job profiles IgProfMemTotal.res or IgProfMemLive.res are in the list as they should:
893  EndOfJobProfileLink=filter(lambda x: "IgProfMemTotal.res" in x or "IgProfMemLive.res" in x, ProfileLinks)[0] #Assume only one of the two is there, as it should.
894  #Remove it from the list so we can order it:
895  ProfileLinks.remove(EndOfJobProfileLink)
896  #Sort the list in numerical order:
897  ProfileLinks.sort(key=lambda x: int(x.split(".")[-2]))
898  #Prepare regexp to find and replace MemTotal with MemLive for .gz links
899  IgProfMemLive_regexp=re.compile("IgProfMemLive")
900  if IgProfMemLive_regexp.search(EndOfJobProfileLink):
901  MemProfile="IgProf MEM LIVE"
902  else:
903  MemProfile="IgProf MEM TOTAL"
904  #Prepare the table header:
905  INDEX.write("<li>%s"%MemProfile)
906  INDEX.write("<table><tr><td>Profile after event</td><td>Total Memory Size (bytes)</td><td>Total Calls (number)</td><td>Link to gzipped IgProf profile</td></tr>")
907  for link in ProfileLinks:
908  #Build and check the link to the .gz profile that is always called IgProfMemTotal also for MemLive:
909  gzProfile=os.path.join(link.split("/")[-3],link.split("/")[-1])[:-3]+"gz"
910  if IgProfMemLive_regexp.search(gzProfile):
911  gzProfile=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzProfile)
912  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(link.split(".")[-2],open(link,"r").readlines()[6].split()[1],open(link,"r").readlines()[6].split()[2],gzProfile,os.path.basename(gzProfile)))
913  #Put in the end of job one by hand:
914  gzEndOfJobProfileLink=os.path.join(EndOfJobProfileLink.split("/")[-3],EndOfJobProfileLink.split("/")[-1])[:-3]+"gz"
915  if IgProfMemLive_regexp.search(gzEndOfJobProfileLink):
916  gzEndOfJobProfileLink=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzEndOfJobProfileLink)
917  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%("End of job",open(EndOfJobProfileLink,"r").readlines()[6].split()[1],open(EndOfJobProfileLink,"r").readlines()[6].split()[2],gzEndOfJobProfileLink,os.path.basename(gzEndOfJobProfileLink)))
918  #Closing the table and the list item tags
919  INDEX.write("</table>")
920  INDEX.write("</li>")
921  #FIXME:
922  #These numbers are used in the index.html they are not automatically matched to the actual
923  #ones (one should automate this, by looking into the cmsCreateSimPerfTestPyRelVal.log logfile)
924 
925  NumOfEvents={
926  DirName[0] : TimeSizeNumOfEvents,
927  DirName[1] : IgProfNumOfEvents,
928  DirName[2] : IgProfNumOfEvents,
929  DirName[3] : IgProfNumOfEvents,
930  DirName[4] : CallgrindNumOfEvents,
931  DirName[5] : MemcheckNumOfEvents,
932  DirName[6] : TimeSizeNumOfEvents,
933  DirName[7] : IgProfNumOfEvents,
934  DirName[8] : IgProfNumOfEvents,
935  DirName[9] : IgProfNumOfEvents,
936  DirName[10] : CallgrindNumOfEvents,
937  DirName[11] : MemcheckNumOfEvents
938  }
939 
940  Profile=(
941  #These need to match the profile directory names ending within the candle directories
942  "TimingReport",
943  "TimeReport",
944  "SimpleMemReport",
945  "EdmSize",
946  "IgProfperf",
947  "IgProfMemTotal",
948  "IgProfMemLive",
949  "IgProfMemAnalyse",
950  "valgrind",
951  "memcheck_valgrind"
952  )
953  IgProfMemAnalyseOut=( #This is the special case of IgProfMemAnalyse
954  "doBeginJob_output.html",
955  "doEvent_output.html",
956  "mem_live.html",
957  "mem_total.html"
958  )
959  memcheck_valgrindOut=( #This is the special case of Valgrind MemCheck (published via Giovanni's script)
960  "beginjob.html",
961  "edproduce.html",
962  "esproduce.html"
963  )
964  OutputHtml={ #These are the filenames to be linked in the candle html page for each profile
965  Profile[0] : "*TimingReport.html", #The wildcard spares the need to know the candle name
966  Profile[1] : "TimeReport.html", #This is always the same (for all candles)
967  Profile[2] : "*.html", #This is supposed to be *SimpleMemoryCheck.html, but there is a bug in cmsRelvalreport.py and it is called TimingReport.html!
968  Profile[3] : "objects_pp.html", #This is one of 4 objects_*.html files, it's indifferent which one to pick, just need consistency
969  Profile[4] : "overall.html", #This is always the same (for all candles)
970  Profile[5] : "overall.html", #This is always the same (for all candles)
971  Profile[6] : "overall.html", #This is always the same (for all candles)
972  Profile[7] : "doBeginJob_output.html", #This is complicated... there are 3 html to link... (see IgProf MemAnalyse below)
973  Profile[8] : "overall.html", #This is always the same (for all candles)
974  Profile[9] : "beginjob.html" #This is complicated there are 3 html to link here too... (see Valgrind MemCheck below)
975  }
976 
977 
978  candnreg = re.compile("CandleName")
979  candhreg = re.compile("CandlesHere")
980  try:
981  CAND = open(candlHTML,"w")
982  for line in open(tmplfile):
983  if candhreg.search(line):
984  CAND.write('<div id="content">')
985  CAND.write("<h2>")
986  CAND.write(CurrentCandle)
987  CAND.write("</h2>\n")
988 
989  if _verbose:
990  print("Producing candles html: ", CurrentCandle)
991 
992  for CurDir in DirName:
993 
994  LocalPath = os.path.join(repdir,"%s_%s" % (CurrentCandle,CurDir))
995  LocalDirname = os.path.basename(LocalPath)
996 
997  if not prevrev == "":
998 
999  profs = []
1000  #FIXME!
1001  #Check what this was used for.. it now has different DirName list since we added IgProf_Perf and IgProf_Mem on top of IgProf alone (still there)
1002  if CurDir == DirName[0] or CurDir == DirName[6]: #TimeSize tests (with and without PU)
1003  profs = Profile[0:4]
1004  elif CurDir == DirName[1] or CurDir == DirName[7]: #IgProf tests (with and without PU)
1005  profs = Profile[4:8]
1006  elif CurDir == DirName[2] or CurDir == DirName[8]: #IgProf Perf tests (with and without PU)
1007  profs =[Profile[4]] #Keeping format to a list...
1008  elif CurDir == DirName[3] or CurDir == DirName[9]: #IgProf Mem tests (with and without PU)
1009  profs =Profile[5:8] #Keeping format to a list...
1010  elif CurDir == DirName[4] or CurDir == DirName[10]: #Callgrind tests (with and without PU)
1011  profs = [Profile[8]] #Keeping format to a list...
1012  elif CurDir == DirName[5] or CurDir == DirName[11]: #Memcheck tests (with and without PU)
1013  profs = [Profile[9]] #Keeping format to a list...
1014  #This could be optimized, but for now just comment the code:
1015  #This for cycle takes care of the case in which there are regression reports to link to the html:
1016  for prof in profs:
1017  if _verbose:
1018  print("Scanning for profile information for: ", prof)
1019 
1020  printed = False
1021  fullprof = (CurrentCandle,prof)
1022  outd = ""
1023  nologext = ""
1024  if prof == "TimingReport":
1025  timeReports = glob.glob(os.path.join(LocalPath,"%s_*_%s.log" % (CandFname[CurrentCandle],prof)))
1026  if len(timeReports) > 0:
1027  if not printed:
1028  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1029  printed = True
1030  for log in timeReports:
1031  reportName = os.path.basename(log)
1032  (nologext, outd) = getOutputNames(LocalDirname,reportName)
1033  CAND.write("<h4>%s</h4>\n" % reportName)
1034  htmNames = ["changes.png"]
1035  otherNames = ["graphs.png" , "histos.png"]
1036  regressHTML= "%s-regress.html" % nologext
1037  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,outd,x),otherNames)))
1038  html = ""
1039  if pathsExist:
1040  html = "<p>Performance graphs and histograms superimposed for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1041  else:
1042  html = "<p>No change in performance graph available</p>\n"
1043  regressHTML="%s/%s" % (WebArea,regressHTML)
1044 
1045  for x in htmNames:
1046  abspath = os.path.join(repdir,outd,x)
1047  if os.path.exists(abspath):
1048  html += "<p><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></p>\n" % (outd,x,outd,x)
1049  else:
1050  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1051 
1052  createRegressHTML(regressHTML,repdir,outd,CurrentCandle,otherNames)
1053  CAND.write(html)
1054  CAND.write("\n</tr></table>")
1055 
1056 
1057  elif prof == "SimpleMemReport":
1058  simMemReports = glob.glob(os.path.join(LocalPath,"%s_*_%s" % (CandFname[CurrentCandle],prof)))
1059  simMemReports = map(lambda x: (CandFname[CurrentCandle],prof,x), simMemReports )
1060  simMemReports.sort(cmp=dirname_cmp)
1061  simMemReports = map(lambda x: x[2], simMemReports )
1062  if len(simMemReports) > 0:
1063  if not printed:
1064  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1065  printed = True
1066  for adir in simMemReports:
1067  reportName = os.path.basename(adir)
1068  CAND.write("<h4>%s</h4>\n" % reportName)
1069  htmNames = ["vsize_change.png", "rss_change.png"]
1070  otherNames = ["vsize_graphs.png","rss_graphs.png"]
1071  nologext = reportName
1072  outd = reportName
1073  regressHTML= "%s-regress.html" % nologext
1074  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,LocalDirname,outd,x),otherNames)))
1075  html = ""
1076  if pathsExist:
1077  html = "<p>Superimposed memory performance graphs for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1078  else:
1079  html = "<p>No change in performance graph available</p>\n"
1080  regressHTML="%s/%s" % (WebArea,regressHTML)
1081 
1082  for x in htmNames:
1083  abspath = os.path.join(repdir,LocalDirname,outd,x)
1084  if os.path.exists(abspath):
1085  html += "<p><a href=\"./%s/%s/%s\"><img src=\"./%s/%s/%s\" /></a></p>\n" % (LocalDirname,outd,x,LocalDirname,outd,x)
1086  else:
1087  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1088 
1089  createRegressHTML(regressHTML,repdir,"%s/%s" % (LocalDirname,outd),CurrentCandle,otherNames)
1090  CAND.write(html)
1091  CAND.write("\n</tr></table>")
1092 
1093  elif prof == "EdmSize" or prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof== "IgProfperf" or prof == "Callgrind":
1094  regresPath = os.path.join(LocalPath,"%s_*_%s_regression" % (CandFname[CurrentCandle],prof))
1095  regresses = glob.glob(regresPath)
1096  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s_regression" % (CandFname[CurrentCandle],prof))
1097  if len(regresses) > 0:
1098  if not printed:
1099  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1100  printed = True
1101  regresses = map(lambda x: (CandFname[CurrentCandle],prof,x),regresses)
1102  regresses.sort(cmp=reg_dirname_cmp)
1103  regresses = map(lambda x: x[2],regresses)
1104  for rep in regresses:
1105  base = os.path.basename(rep)
1106  found = stepreg.search(base)
1107  step = "Unknown-step"
1108  if found:
1109  step = found.groups()[0]
1110  htmlpage = ""
1111  if prof == "IgProfMemLive" or prof == "IgProfMemTotal" or prof== "IgProfperf" or prof == "Callgrind":
1112  htmlpage = "overall.html"
1113  else:
1114  htmlpage = "objects_pp.html"
1115  CAND.write("<a href=\"%s/%s/%s\">%s %s regression report</a><br/>\n" % (LocalDirname,base,htmlpage,prof,step))
1116  #Here we go back to the "non-regression" reports listing
1117  CandleLogFiles = []
1118  if os.path.exists(LocalPath):
1119  thedir = os.listdir(LocalPath)
1120  CandleLogFiles = [x for x in map(lambda x: os.path.abspath(os.path.join(LocalPath,x)),thedir) if (x.endswith(".log") or x.endswith("EdmSize")) and not os.path.isdir(x) and os.path.exists(x)]
1121 
1122  if (len(CandleLogFiles)>0):
1123 
1124  syscp(CandleLogFiles,WebArea + "/")
1125  base = os.path.basename(LocalPath)
1126  lfileshtml = ""
1127 
1128  for cand in CandleLogFiles:
1129  cand = os.path.basename(cand)
1130  if _verbose:
1131  print("Found %s in %s\n" % (cand,LocalPath))
1132 
1133  if not "EdmSize" in cand:
1134  lfileshtml += "<a href=\"./%s/%s\">%s </a><br/>" % (base,cand,cand)
1135 
1136  CAND.write("<p><strong>Logfiles for %s</strong></p>\n" % CurDir)
1137  CAND.write(lfileshtml)
1138 
1139  PrintedOnce = False
1140  for CurrentProfile in Profile:
1141  #print Steps
1142  for step in Steps: #Using Steps here that is Step+ProductionSteps!
1143  #print step
1144  #print "DEBUG:%s,%s,%s,%s,%s,%s"%(repdir,CurrentCandle,CurDir,step,CurrentProfile,OutputHtml[CurrentProfile])
1145  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1146  CurDir,
1147  step,
1148  CurrentProfile,
1149  OutputHtml[CurrentProfile])
1150  #if ProfileReportLink:
1151  # print ProfileReportLink
1152  isProfinLink = False
1153  if len (ProfileReportLink) > 0:
1154  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1155 
1156  if isProfinLink:
1157  #It could also not be there
1158 
1159  if (PrintedOnce==False):
1160  #Making sure it's printed only once per directory (TimeSize, IgProf, Valgrind) each can have multiple profiles
1161 
1162  #This is the "title" of a series of profiles, (TimeSize, IgProf, Valgrind)
1163  CAND.write("<p><strong>%s</strong></p>\n" % CurDir)
1164  CAND.write("<ul>\n")
1165  PrintedOnce=True
1166 
1167  #Special cases first (IgProf MemAnalyse and Valgrind MemCheck)
1168  #Add among the special cases any IgProfMem (5,6,7) since now we added the dumping:
1169  if (CurrentProfile == Profile[5] or CurrentProfile == Profile[6]):
1170  for prolink in ProfileReportLink:
1171  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1172  for igprofmem in ["*IgProfMemTotal*.res","*IgProfMemLive*.res"]:
1173  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1174  CurDir,
1175  step,
1176  CurrentProfile,
1177  igprofmem)
1178  isProfinLink = False
1179  if len (ProfileReportLink) > 0:
1180  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1181  if isProfinLink :#It could also not be there
1182  #for prolink in ProfileReportLink:
1183  IgProfDumpsTable(CAND,ProfileReportLink,step)
1184  # _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=os.path.basename(prolink))
1185 
1186  elif (CurrentProfile == Profile[7]):
1187  for igprof in IgProfMemAnalyseOut:
1188  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1189  CurDir,
1190  step,
1191  CurrentProfile,
1192  igprof)
1193  isProfinLink = False
1194  if len (ProfileReportLink) > 0:
1195  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1196  if isProfinLink :#It could also not be there
1197  for prolink in ProfileReportLink:
1198  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=igprof)
1199 
1200 
1201  elif (CurrentProfile == Profile[9]):
1202 
1203  for memprof in memcheck_valgrindOut:
1204  #print memprof
1205  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1206  CurDir,
1207  step,
1208  CurrentProfile,
1209  memprof
1210  )
1211  isProfinLink = False
1212  if len (ProfileReportLink) > 0:
1213  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1214  if isProfinLink :#It could also not be there
1215  for prolink in ProfileReportLink:
1216  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=memprof)
1217 
1218  else:
1219  for prolink in ProfileReportLink:
1220  if "regression" not in prolink: #To avoid duplication of links to regression reports!
1221  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1222  #print "Step is %s, CurrentProfile is %s and ProfileReportLink is %s and prolink is %s"%(step,CurrentProfile,ProfileReportLink,prolink)
1223 
1224 
1225  if PrintedOnce:
1226  CAND.write("</ul>\n")
1227  PrintedOnce=False
1228 
1229  CAND.write("<hr />")
1230  CAND.write("<br />\n")
1231  CAND.write("</div>\n")
1232  elif candnreg.search(line):
1233  CAND.write(CurrentCandle)
1234  else:
1235  CAND.write(line)
1236 
1237  CAND.close()
1238  except IOError as detail:
1239  print("ERROR: Could not write candle html %s because %s" % (os.path.basename(candlHTML),detail))
1240 
1241 
1242 def populateFromTupleRoot(tupname,repdir,rootfile,pureg):
1243  table = Table()
1244  for cand in Candles:
1245  fname = CandFname[cand]
1246  globpath = os.path.join(repdir,"%s_TimeSize" % cand,"%s_*_TimingReport" % fname)
1247  stepDirs = glob.glob(globpath)
1248  stepDirs = map(lambda x: (fname,x), stepDirs)
1249  stepDirs.sort(cmp=timerep_cmp)
1250  stepDirs = map(lambda x: x[1], stepDirs)
1251  stepreg = re.compile("%s_(.*)_TimingReport" % fname)
1252  createNewRow = True
1253  curRow = None
1254  createPURow = True
1255  puRow = None
1256  for stepdir in stepDirs:
1257  base = os.path.basename(stepdir)
1258  found = stepreg.search(base)
1259  step = "Unknown-step"
1260  if found:
1261  step = found.groups()[0]
1262  realstep = "Unknown-step"
1263  if "PILEUP" in step:
1264  found = pureg.search(step)
1265  if found:
1266  realstep = found.groups()[0]
1267  if createPURow:
1268  createPURow = False
1269  puRow = Row(table)
1270  rootf = os.path.join(stepdir,rootfile)
1271 
1272  if os.path.exists(rootf):
1273  f = ROOT.TFile(rootf)
1274 
1275  cpu_time_tree = ROOT.TTree()
1276  f.GetObject("cpu_time_tuple;1",cpu_time_tree)
1277  if cpu_time_tree:
1278  if cpu_time_tree.InheritsFrom("TTree"):
1279  data1 = None
1280  data2 = None
1281  for t in cpu_time_tree:
1282  data1 = t.total1
1283  data2 = t.total2
1284  if data1 and data2:
1285  if createNewRow:
1286  createNewRow = False
1287  curRow = table.newRow(cand)
1288  data_tuple = (data1,data2)
1289 
1290  if "PILEUP" in step:
1291  puRow.addEntry(realstep,data_tuple)
1292  else:
1293  if createNewRow:
1294  createNewRow = False
1295  curRow = table.newRow(cand)
1296 
1297  curRow.addEntry(step,data_tuple)
1298  f.Close()
1299  if puRow == None:
1300  pass
1301  else:
1302  table.addRow(puRow,"%s PILEUP" %cand)
1303  return table
1304 
1305 
1306 def createHTMLtab(INDEX,table_dict,ordered_keys,header,caption,name,mode=0):
1307  cols = len(ordered_keys)
1308  totcols = (cols * 3) + 1
1309  innercol = 3
1310  colspan = totcols - 1
1311  labels = []
1312  if mode == 1:
1313  labels = ["fs1","fs2","&#x0394;"]
1314  elif mode == 2 or mode == 3:
1315  colspan = cols
1316  innercol = 1
1317  else:
1318  labels = ["t1" ,"t2" ,"&#x0394;"]
1319 
1320 
1321  INDEX.write("<h3>%s</h3>\n" % header)
1322  INDEX.write("<table>\n")
1323  INDEX.write("<caption>%s</caption>\n" % caption)
1324  INDEX.write("<thead><tr><th></th><th colspan=\"%s\" scope=\"colgroup\">%s</th></tr></thead>" % (colspan,name))
1325  INDEX.write("<tbody>\n")
1326  for key in ordered_keys:
1327  INDEX.write("<tr>")
1328  if key == None:
1329  INDEX.write("<th></th>")
1330  else:
1331  INDEX.write("<td scope=\"row\">")
1332  INDEX.write(key)
1333  INDEX.write("</td>")
1334  for col in table_dict[None]:
1335  if key == None:
1336  INDEX.write("<th colspan=\"%s\" scope=\"col\">" % innercol)
1337  INDEX.write(col)
1338  INDEX.write("</th>")
1339  else:
1340  rowdict = table_dict[key].getRowDict()
1341  if col in rowdict:
1342  if mode == 2:
1343  dat = prettySize(rowdict[col])
1344  INDEX.write("<td>")
1345  INDEX.write("%s" % dat)
1346  INDEX.write("</td>")
1347 
1348  elif mode == 3:
1349  dat = rowdict[col]
1350  INDEX.write("<td>")
1351  INDEX.write("%6.2f" % dat)
1352  INDEX.write("</td>")
1353  else:
1354  (data1, data2) = rowdict[col]
1355  diff = data2 - data1
1356 
1357  if mode == 1:
1358  diff = prettySize(diff)
1359  data1 = prettySize(data1)
1360  data2 = prettySize(data2)
1361 
1362  seq = [ data1, data2, diff ]
1363  for dat in seq:
1364  INDEX.write("<td id=\"data\">")
1365  if mode == 1:
1366  INDEX.write("%s" % dat) # %s if
1367  else:
1368  INDEX.write("%6.2f" % dat) # %s if
1369 
1370  INDEX.write("</td>")
1371  else:
1372  if mode == 2 or mode == 3:
1373  INDEX.write("<td>")
1374  INDEX.write("N/A")
1375  INDEX.write("</td>")
1376  else:
1377  for i in range(3):
1378  INDEX.write("<td>")
1379  INDEX.write("N/A")
1380  INDEX.write("</td>")
1381  INDEX.write("</tr>\n")
1382  # write an additional row if this row is the header row
1383  # we need to describe the sub columns
1384  if not (mode == 2 or mode == 3 ):
1385  if key == None:
1386  INDEX.write("<tr>")
1387  INDEX.write("<th>Candles</th>")
1388  for col in table_dict[None]:
1389  INDEX.write("<th>%s</th>" % labels[0])
1390  INDEX.write("<th>%s</th>" % labels[1])
1391  INDEX.write("<th>%s</th>" % labels[2])
1392  INDEX.write("</tr>\n")
1393  INDEX.write("</tbody></table>\n")
1394 
1395  INDEX.write("<br />")
1396 
1397 def stageIgProfReports(remotedir,arch,version):
1398  '''Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.'''
1399  #FIXME: can eliminate this part if using tar pipes... was done with rsynch in mind
1400  #Compose command to create remote dir:
1401  if ":" in remotedir: #Remote host local directory case
1402  (host,dir)=remotedir.split(":")
1403  mkdir_cmd="ssh %s (mkdir %s;mkdir %s/%s)"%(host,dir,dir,arch)
1404  else: #AFS or local case
1405  mkdir_cmd="mkdir %s;mkdir %s/%s"%(remotedir,remotedir,arch)
1406 
1407  #Create remote dir:
1408  try:
1409  print(mkdir_cmd)
1410  os.system(mkdir_cmd)
1411  print("Successfully created publication directory")
1412  except:
1413  print("Issues with publication directory existence/creation!")
1414 
1415  #Copy files over to remote dir
1416  #replacing rsync with tar pipes since it can hang on AFS (Andreas' experience):
1417  #rsync_cmd="rsync -avz *_IgProf_*/*.sql3 %s/%s/%s"%(remotedir,arch,version)
1418  if ":" in remotedir:
1419  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | ssh %s "cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*"'%(host,dir,arch,version,version)
1420  else:
1421  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | (cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*)'%(remotedir,arch,version,version)
1422  try:
1423  # print rsync_cmd
1424  # os.system(rsync_cmd)
1425  print(tarpipe_cmd)
1426  os.system(tarpipe_cmd)
1427  print("Successfully copied IgProf reports to %s"%remotedir)
1428  except:
1429  print("Issues with rsyncing to the remote directory %s!"%remotedir)
1430 
1431  #Make sure permissions are set for group to be able to write:
1432  if ":" in remotedir: #Remote host local directory case
1433  chmod_cmd="ssh %s chmod -R 775 %s/%s"%(host,dir,arch)
1434  else:
1435  chmod_cmd="chmod -R 775 %s/%s"%(remotedir,arch)
1436  try:
1437  print(chmod_cmd)
1438  os.system(chmod_cmd)
1439  print("Successfully set permissions for IgProf reports directory %s"%remotedir)
1440  except:
1441  print("(Potential) issues with chmoding the remote directory %s!"%remotedir)
1442 
1443  return #Later, report here something like the web link to the reports in igprof-navigator...
1444 
1445 
1446 #####################
1447 #
1448 # Create web report index and create HTML file for each candle
1449 #
1450 def createWebReports(WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
1451 
1452  #Some nomenclature
1453 
1454  Candle = Candles #These need to match the directory names in the work area
1455 
1456  CmsDriverCandle = CandFname #{ #These need to match the cmsDriver.py output filenames
1457 
1458 
1459  #Produce a "small" index.html file to navigate the html reports/logs etc
1460  IndexFile="%s/index.html" % WebArea
1461  TemplateHtml="%s/doc/index.html" % BASE_PERFORMANCE
1462 
1463  cmsverreg = re.compile("CMSSW_VERSION")
1464  hostreg = re.compile("HOST")
1465  lpathreg = re.compile("LocalPath")
1466  fsizereg = re.compile("FSizeTable")
1467  cpureg = re.compile("CPUTable")
1468  proddreg = re.compile("ProductionDate")
1469  logfreg = re.compile("LogfileLinks")
1470  dirbreg = re.compile("DirectoryBrowsing")
1471  pubdreg = re.compile("PublicationDate")
1472  candhreg = re.compile("CandlesHere")
1473  #Loop line by line to build our index.html based on the template one
1474  #Copy the perf_style.css file from Validation/Performance/doc
1475 
1476  CandlTmpltHTML="%s/doc/candle.html" % BASE_PERFORMANCE
1477  if _verbose:
1478  print("Copying %s/doc/perf_style.css style file to %s/." % (BASE_PERFORMANCE,WebArea))
1479  print("Template used: %s" % TemplateHtml)
1480 
1481  syscp((BASE_PERFORMANCE + "/doc/perf_style.css"),WebArea + "/.")
1482  pureg = re.compile("(.*)_PILEUP")
1483  try:
1484  INDEX = open(IndexFile,"w")
1485  for NewFileLine in open(TemplateHtml) :
1486  if cmsverreg.search(NewFileLine):
1487  if prevrev == "":
1488  INDEX.write("Performance Reports for %s\n" % CMSSW_VERSION)
1489  else:
1490  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1491  globs = glob.glob(globpath)
1492  if len(globs) < 1:
1493  pass
1494  else:
1495  latestreg = re.compile("REGRESSION.%s.vs.(.*)" % prevrev)
1496  found = latestreg.search(os.path.basename(globs[0]))
1497  if found:
1498  latestrel = found.groups()[0]
1499  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,latestrel))
1500  else:
1501  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,CMSSW_VERSION))
1502  elif hostreg.search(NewFileLine):
1503  INDEX.write(HOST + "\n")
1504  #File Size Summary Table
1505  elif fsizereg.search(NewFileLine):
1506  #Case of NO-REGRESSION:
1507  if prevrev == "":
1508  fsize_tab = Table()
1509 
1510  for cand in Candles:
1511  fname = CandFname[cand]
1512  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1513  rootfiles = glob.glob(globpath)
1514  rootfiles = map(lambda x: (fname,x), rootfiles)
1515  rootfiles.sort(cmp=rootfile_cmp)
1516  rootfiles = map(lambda x: x[1], rootfiles)
1517  stepreg = re.compile("%s_(.*).root" % fname)
1518  createNewRow = True
1519  curRow = None
1520  createPURow = True
1521  puRow = None
1522  for rootf in rootfiles:
1523  base = os.path.basename(rootf)
1524  found = stepreg.search(base)
1525  step = "Unknown-step"
1526  if found:
1527  step = found.groups()[0]
1528  realstep = "Unknown-step"
1529  if "PILEUP" in step:
1530  found = pureg.search(step)
1531  if found:
1532  realstep = found.groups()[0]
1533  if createPURow:
1534  createPURow = False
1535  puRow = Row(fsize_tab)
1536  try:
1537  statinfo = os.stat(rootf)
1538  fsize = statinfo.st_size
1539  if createNewRow:
1540  createNewRow = False
1541  curRow = fsize_tab.newRow(cand)
1542 
1543  if "PILEUP" in step:
1544  puRow.addEntry(realstep,fsize)
1545  else:
1546  if createNewRow:
1547  createNewRow = False
1548  curRow = fsize_tab.newRow(cand)
1549 
1550  curRow.addEntry(step,fsize)
1551  except IOError as detail:
1552  print(detail)
1553  except OSError as detail:
1554  print(detail)
1555  if puRow == None:
1556  pass
1557  else:
1558  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1559  (ordered_keys,table_dict) = fsize_tab.getTable(1)
1560  cols = len(ordered_keys)
1561 
1562  if len(table_dict) > 1 and cols > 0:
1563  createHTMLtab(INDEX,table_dict,ordered_keys,
1564  "Release ROOT file sizes",
1565  "Table showing current release ROOT filesizes in (k/M/G) bytes.",
1566  "Filesizes",2)
1567  #Case of REGRESSION vs previous release:
1568  else:
1569  try:
1570  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1571  globs = glob.glob(globpath)
1572  if len(globs) < 1:
1573  raise IOError(2,globpath,"File does not exist","File does not exist")
1574  idfile = open(globs[0])
1575  oldpath = ""
1576  for line in idfile:
1577  oldpath = line
1578  oldpath = oldpath.strip()
1579  #print "##########TABLE DEBUG :oldpath is %s"%oldpath
1580  fsize_tab = Table()
1581 
1582  for cand in Candles:
1583  fname = CandFname[cand]
1584  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1585  rootfiles = glob.glob(globpath)
1586  rootfiles = map(lambda x: (fname,x), rootfiles)
1587  rootfiles.sort(cmp=rootfile_cmp)
1588  rootfiles = map(lambda x: x[1], rootfiles)
1589  stepreg = re.compile("%s_(.*).root" % fname)
1590  createNewRow = True
1591  curRow = None
1592  createPURow = True
1593  puRow = None
1594  for rootf in rootfiles:
1595  base = os.path.basename(rootf)
1596  found = stepreg.search(base)
1597  step = "Unknown-step"
1598  if found:
1599  step = found.groups()[0]
1600 
1601  realstep = "Unknown-step"
1602  if "PILEUP" in step:
1603  found = pureg.search(step)
1604  if found:
1605  realstep = found.groups()[0]
1606  if createPURow:
1607  createPURow = False
1608  puRow = Row(fsize_tab)
1609  try:
1610  statinfo = os.stat(rootf)
1611  fsize2 = statinfo.st_size
1612  oldfile = os.path.join(oldpath,"%s_TimeSize" % cand,base)
1613  if "PILEUP" in step:
1614  oldfile = os.path.join(oldpath,"%s_PU_TimeSize" % cand,base)
1615  fsize1 = 0
1616 
1617  if os.path.exists(oldfile):
1618  statinfo = os.stat(oldfile)
1619  fsize1 = statinfo.st_size
1620  else:
1621  print("######DID NOT FIND Previous file (needed for the filesize table): %s"%oldfile)
1622 
1623  if createNewRow:
1624  createNewRow = False
1625  curRow = fsize_tab.newRow(cand)
1626 
1627  data_tuple = (fsize1,fsize2)
1628  if "PILEUP" in step:
1629  puRow.addEntry(realstep,data_tuple)
1630  else:
1631  if createNewRow:
1632  createNewRow = False
1633  curRow = fsize_tab.newRow(cand)
1634 
1635  curRow.addEntry(step,data_tuple)
1636  except IOError as detail:
1637  print(detail)
1638  except OSError as detail:
1639  print(detail)
1640  if puRow == None:
1641  pass
1642  else:
1643  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1644 
1645  (ordered_keys,table_dict) = fsize_tab.getTable()
1646  cols = len(ordered_keys)
1647 
1648  if len(table_dict) > 1 and cols > 0:
1649  createHTMLtab(INDEX,table_dict,ordered_keys,
1650  "Release ROOT file sizes",
1651  "Table showing previous release ROOT filesizes, fs1, latest sizes, fs2, and the difference between them &#x0394; in (k/M/G) bytes.",
1652  "Filesizes",1)
1653  except IOError as detail:
1654  print(detail)
1655  except OSError as detail:
1656  print(detail)
1657  #CPU Time Summary Table
1658  elif cpureg.search(NewFileLine):
1659  #Case of NO REGRESSION
1660  if prevrev == "":
1661  time_tab = Table()
1662 
1663  for cand in Candles:
1664  fname = CandFname[cand]
1665  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*_TimingReport.log" % fname)
1666  logfiles = glob.glob(globpath)
1667  logfiles = map(lambda x: (fname,x), logfiles)
1668  logfiles.sort(cmp=logrep_cmp)
1669  logfiles = map(lambda x: x[1], logfiles)
1670 
1671  stepreg = re.compile("%s_(.*)_TimingReport.log" % fname)
1672  createNewRow = True
1673  curRow = None
1674  createPURow = True
1675  puRow = None
1676  for log in logfiles:
1677  base = os.path.basename(log)
1678  found = stepreg.search(base)
1679  step = "Unknown-step"
1680  if found:
1681  step = found.groups()[0]
1682 
1683  realstep = "Unknown-step"
1684  if "PILEUP" in step:
1685  found = pureg.search(step)
1686  if found:
1687  realstep = found.groups()[0]
1688  if createPURow:
1689  createPURow = False
1690  puRow = Row(time_tab)
1691 
1692  data = cpr.getTimingLogData(log)
1693  mean = 0
1694  i = 0
1695  for evtnum, time in data:
1696  mean += time
1697  i += 1
1698  try:
1699  mean = mean / float(i)
1700  except ZeroDivisionError as detail:
1701  print("WARNING: Could not calculate mean CPU time from log because no events could be parsed", log)
1702 
1703  if "PILEUP" in step:
1704  puRow.addEntry(realstep,mean)
1705  else:
1706  if createNewRow:
1707  createNewRow = False
1708  curRow = time_tab.newRow(cand)
1709 
1710  curRow.addEntry(step,mean)
1711  if puRow == None:
1712  pass
1713  else:
1714  time_tab.addRow(puRow,"%s PILEUP" %cand)
1715 
1716  (ordered_keys,table_dict) = time_tab.getTable(1)
1717  cols = len(ordered_keys)
1718 
1719  if len(table_dict) > 1 and cols > 0:
1720  createHTMLtab(INDEX,table_dict,ordered_keys,
1721  "Release CPU times",
1722  "Table showing current release CPU times in secs.",
1723  "CPU Times (s)",3)
1724  #Case of REGRESSION (CPU Time Summary Table)
1725  else:
1726 
1727 
1728  ####
1729  #
1730  # Create the table data structure
1731  #
1732  cpu_time_tab = populateFromTupleRoot("cpu_time_tuple",repdir,"timing-regress.root",pureg)
1733 
1734 
1735  ###########
1736  #
1737  # Create HTML table from table data structure
1738  #
1739 
1740  (ordered_keys,table_dict) = cpu_time_tab.getTable()
1741 
1742  cols = len(ordered_keys)
1743  if len(table_dict) > 1 and cols > 0:
1744  createHTMLtab(INDEX,table_dict,ordered_keys,
1745  "Release CPU times",
1746  "Table showing previous release CPU times, t1, latest times, t2, and the difference between them &#x0394; in secs.",
1747  "CPU Times (s)")
1748 
1749 
1750 
1751  elif lpathreg.search(NewFileLine):
1752  INDEX.write(repdir + "\n")
1753  elif proddreg.search(NewFileLine):
1754  INDEX.write(ExecutionDate + "\n")
1755  elif logfreg.search(NewFileLine):
1756  INDEX.write("<br />\n")
1757  for log in LogFiles:
1758  log = os.path.basename(log)
1759  if _verbose:
1760  print("linking log file %s" % log)
1761  INDEX.write("<a href=\"./%s\"> %s </a>" % (log,log))
1762  INDEX.write("<br />\n")
1763  #Add the cmsScimark results here:
1764  INDEX.write("Results for cmsScimark2 benchmark (running on the other cores) available at:\n")
1765  for cmssci in cmsScimarkResults:
1766  scimarkdirs=cmssci.split("/")
1767  localdir=scimarkdirs[-2]
1768  #print localdir
1769  cmssci = os.path.basename(cmssci)
1770  relativelink=os.path.join(localdir,cmssci)
1771  #print relativelink
1772  INDEX.write("<a href=\"%s\"> %s </a>" % (relativelink,cmssci))
1773  INDEX.write("<br />\n")
1774 
1775 
1776  elif dirbreg.search(NewFileLine):
1777  #Create a subdirectory DirectoryBrowsing to circumvent the fact the dir is not browsable if there is an index.html in it.
1778  #Bug! This does not work since it points to index.html automatically!
1779  #os.symlink("./","%s/DirectoryBrowsing" % WebArea)
1780  #Actually all the following is done later...
1781  #Create a physical directory first
1782  #os.mkdir("%s/DirectoryBrowsing" % WebArea)
1783  #Then will populate it with symbolic links(once they have been copied there!) from all WebArea files, except index.html, see down towards the end!
1784  INDEX.write("Click <a href=\"./DirectoryBrowsing/\">here</a> to browse the directory containing all results (except the root files)\n")
1785 
1786  elif pubdreg.search(NewFileLine):
1787  INDEX.write(date + "\n")
1788  elif candhreg.search(NewFileLine):
1789  for acandle in Candle:
1790  globpath = os.path.join(repdir,"%s_*" % acandle)
1791  globs = glob.glob(globpath)
1792  if len(globs) > 0:
1793  candlHTML = "%s.html" % acandle
1794  INDEX.write("<a href=\"./%s\"> %s </a>" % (candlHTML,acandle))
1795  INDEX.write("<br />\n")
1796 
1797  candlHTML=os.path.join(WebArea,candlHTML)
1798  createCandlHTML(CandlTmpltHTML,candlHTML,acandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
1799  else:
1800  INDEX.write(NewFileLine)
1801 
1802  #End of while loop on template html file
1803  INDEX.close()
1804  except IOError as detail:
1805  print("Error: Could not create index Html file for some reason, check position. Details : %s" % detail)
1806 
1807 ########################
1808 #
1809 # Grab dirs that end in strings defined in DirName
1810 #
1811 def getDirnameDirs(repdir,WebArea):
1812  Dir = os.listdir(repdir)
1813  def _containsDirName(elem):
1814  return reduce(lambda x,y: x or y,map(lambda x: x in elem, DirName))
1815  def _print4Lambda(elem,WebArea):
1816  if _verbose:
1817  print("Copying %s to %s\n" % (elem,WebArea))
1818 
1819  dirstocp = [x for x in map(lambda x: repdir + x,Dir) if _containsDirName(x)]
1820  map(lambda x: _print4Lambda(x,WebArea),dirstocp)
1821  syscp(dirstocp,WebArea + "/")
1822  os.mkdir("%s/DirectoryBrowsing" % WebArea)
1823  for file in os.listdir(WebArea):
1824  if file != "index.html": #Potential maintenance issue if the index.html changes name to something the server automatically displays when pointing to the directory...
1825  #Use relative path ".." instead of WebArea to avoid problems when copying stuff to a remote server!
1826  os.symlink("%s/%s"%("..",file),"%s/DirectoryBrowsing/%s" % (WebArea,file))
1827 
1828 #######################
1829 #
1830 # Upload stage to remote location
1831 def syncToRemoteLoc(stage,drive,path,port):
1832  stage = addtrailingslash(stage)
1833  cmd = "rsync -avz"
1834  # We must, MUST, do os.path.normpath otherwise rsync will dump the files in the directory
1835  # we specify on the remote server, rather than creating the CMS_VERSION directory
1836  #--rsh=\"ssh -l relval\"
1837  args = "--port=%s %s %s:%s" % (port,os.path.normpath(stage),drive,path)
1838  retval = -1
1839  if _dryrun:
1840  print(cmd + " --dry-run " + args)
1841  retval = os.system(cmd + " --dry-run " + args )
1842  else:
1843  print(cmd+" "+args)
1844  retval = os.system(cmd + " " + args)
1845  return retval
1846 
1847 ################
1848 #
1849 # Delete tmp dir if we used it
1850 def delTmpDir():
1851  if os.path.exists(TMP_DIR) and IS_TMP:
1852  os.system("rm -Rf " + TMP_DIR)
1853 
1854 #####################
1855 #
1856 # Some functions used for copying
1857 
1858 def getRelativeDir(parent,child,keepTop=True):
1859  def _walkpath(path):
1860  dirs = []
1861  while True:
1862  head , tail = os.path.split(path)
1863  if tail == "":
1864  break
1865  dirs.append(tail)
1866  path = head
1867  for i in range(len(dirs)-1,-1,-1):
1868  adir = dirs[i]
1869  yield adir
1870  return
1871  pwalk = _walkpath(parent)
1872  n = 0
1873  try:
1874  while True:
1875  next(pwalk)
1876  n += 1
1877  except StopIteration:
1878  pass
1879 
1880  if keepTop:
1881  n = n - 1
1882 
1883  cwalk = _walkpath(child)
1884  try:
1885  #prewalk
1886  for x in range(n):
1887  next(cwalk)
1888  except StopIteration:
1889  print("ERROR: Unable to determine relative dir")
1890  raise ReldirExcept
1891 
1892  relpath = ""
1893  try:
1894  while True:
1895  relpath=os.path.join(relpath,next(cwalk))
1896  except StopIteration:
1897  pass
1898  return relpath
1899 
1900 def docopy(src,dest):
1901  try:
1902  copy2(src,dest)
1903  except OSError as detail:
1904  print("WARNING: Could not copy %s to %s because %s" % (src,dest,detail))
1905  except IOError as detail:
1906  print("WARNING: Could not copy %s to %s because %s" % (src,dest,detail))
1907  else:
1908  if _verbose:
1909  print("Copied %s to %s" % (src,dest))
1910 
1911 def copytree4(src,dest,keepTop=True):
1912  def _getNewLocation(source,child,dst,keepTop=keepTop):
1913  place = getRelativeDir(source,child,keepTop=keepTop)
1914  return os.path.join(dst,place)
1915  def _copyFilter(source,dst,curdir,fsnodes,filter,dirs=False):
1916  for node in fsnodes:
1917  dontFilter = True
1918  filterExist = not len(filter) == 0
1919  if filterExist:
1920  dontFilter = not reduce(lambda x,y: x or y,map(lambda x: fnmatch.fnmatch(node,x),filter))
1921  if dontFilter:
1922  node = os.path.join(curdir,node) # convert to absolute path
1923  try:
1924  newnode = _getNewLocation(source,node,dst)
1925  if dirs:
1926  os.mkdir(newnode)
1927  else:
1928  copy2(node,newnode)
1929  except IOError as detail:
1930  print("WARNING: Could not copy %s to %s because %s" % (node,newnode,detail))
1931  except OSError as detail:
1932  print("WARNING: Could not copy %s to %s because %s" % (src,dest,detail))
1933  except ReldirExcept:
1934  print("WARNING: Could not determine new location for source %s into destination %s" % (source,dst))
1935  else:
1936  if len(filter) > 0:
1937  try:
1938  match = fnmatch.fnmatch(node,filter[0])
1939  assert not match
1940  except AssertionError as detail:
1941  print(node, filter[0], match)
1942  raise RuntimeError
1943  if _verbose:
1944  if "root" in node:
1945  print("Filter %s Copied %s to %s" % (dontFilter,node,newnode))
1946  print("fnmatch %s" % fnmatch.fnmatch(node,cpFileFilter[0]))
1947 
1948  gen = os.walk(src)
1949  try:
1950  newloc = _getNewLocation(src,src,dest)
1951 
1952  os.mkdir(newloc)
1953  try:
1954  while True:
1955  step = next(gen)
1956  curdir = step[0]
1957  dirs = step[1]
1958  files = step[2]
1959 
1960  _copyFilter(src,dest,curdir,dirs,cpDirFilter,dirs=True)
1961  _copyFilter(src,dest,curdir,files,cpFileFilter)
1962 
1963  except StopIteration:
1964  pass
1965  except IOError as detail:
1966  print("WARNING: Could not copy %s to %s because %s" % (src,dest,detail))
1967  except OSError as detail:
1968  print("WARNING: Could not copy %s to %s because %s" % (src,dest,detail))
1969  except ReldirExcept:
1970  print("WARNING: Could not determine the new location for source %s into destination %s" % (src,dest))
1971 
1972 def syscp(srcs,dest):
1973  if isinstance(srcs, type("")):
1974  if os.path.exists(srcs):
1975  if os.path.isdir(srcs):
1976  copytree4(srcs,dest)
1977  else:
1978  docopy(srcs,dest)
1979  else:
1980  print("ERROR: file to be copied %s does not exist" % foo)
1981  else:
1982  for src in srcs:
1983  if os.path.exists(src):
1984  if os.path.isdir(src):
1985  #copy tree
1986  copytree4(src,dest)
1987  else:
1988  docopy(src,dest)
1989  else:
1990  print("ERROR: file to be copied %s does not exist" % foo)
1991 
1992 def print_header():
1993  print("%s\n" % PROG_NAME)
1994 
1995 if __name__ == "__main__":
1996  main()
cmsPerfPublish.rootfile_cmp
def rootfile_cmp(x, y)
Definition: cmsPerfPublish.py:718
cmsPerfPublish.optionparse
def optionparse()
Option parser.
Definition: cmsPerfPublish.py:334
cmsPerfPublish.getcmd
def getcmd(command)
Definition: cmsPerfPublish.py:86
cmsPerfPublish.docopy
def docopy(src, dest)
Definition: cmsPerfPublish.py:1900
cmsPerfPublish.timerep_cmp
def timerep_cmp(x, y)
Definition: cmsPerfPublish.py:766
cmsPerfPublish.getcmdBasic
def getcmdBasic(cmd)
Definition: cmsPerfPublish.py:81
cmsPerfPublish.Table.addRow
def addRow(self, row, name)
Definition: cmsPerfPublish.py:181
cmsPerfPublish.syscp
def syscp(srcs, dest)
Definition: cmsPerfPublish.py:1972
cmsPerfPublish.prettySize
def prettySize(size)
Definition: cmsPerfPublish.py:93
cmsPerfPublish.dirname_cmp
def dirname_cmp(x, y)
Definition: cmsPerfPublish.py:730
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
GetRecoTauVFromDQM_MC_cff.next
next
Definition: GetRecoTauVFromDQM_MC_cff.py:31
cmsPerfPublish.getArchVersionFromLog
def getArchVersionFromLog(logfile)
Definition: cmsPerfPublish.py:478
cmsPerfPublish.stageIgProfReports
def stageIgProfReports(remotedir, arch, version)
Definition: cmsPerfPublish.py:1397
mathSSE::return
return((rh^lh)&mask)
cmsPerfPublish.reg_dirname_cmp
def reg_dirname_cmp(x, y)
Definition: cmsPerfPublish.py:742
cmsPerfPublish.copytree4
def copytree4(src, dest, keepTop=True)
Definition: cmsPerfPublish.py:1911
cmsPerfPublish.Table._newCol
def _newCol(self, name)
Definition: cmsPerfPublish.py:135
cmsPerfPublish.get_environ
def get_environ()
Get require environment variables.
Definition: cmsPerfPublish.py:304
cmsPerfPublish.syncToRemoteLoc
def syncToRemoteLoc(stage, drive, path, port)
Upload stage to remote location.
Definition: cmsPerfPublish.py:1831
cmsPerfPublish.scanReportArea
def scanReportArea(repdir)
Scan report area for required things.
Definition: cmsPerfPublish.py:644
cmsPerfPublish.getOutputNames
def getOutputNames(base, reportName)
Definition: cmsPerfPublish.py:710
cmsPerfPublish.createRegressHTML
def createRegressHTML(reghtml, repdir, outd, CurrentCandle, htmNames)
Definition: cmsPerfPublish.py:685
cmsPerfPublish.addtrailingslash
def addtrailingslash(adir)
Definition: cmsPerfPublish.py:72
cmsPerfPublish.getStageRepDirs
def getStageRepDirs(options, args)
Determine locations of staging and report dirs.
Definition: cmsPerfPublish.py:497
cmsPerfPublish.logrep_cmp
def logrep_cmp(x, y)
Definition: cmsPerfPublish.py:754
genParticles_cff.map
map
Definition: genParticles_cff.py:11
cmsPerfPublish.createCandlHTML
def createCandlHTML(tmplfile, candlHTML, CurrentCandle, WebArea, repdir, ExecutionDate, LogFiles, cmsScimarkResults, date, prevrev)
Create HTML pages for candles.
Definition: cmsPerfPublish.py:848
cmsPerfPublish.Table.getTable
def getTable(self, mode=0)
Definition: cmsPerfPublish.py:153
cmsPerfPublish.Table.newRow
def newRow(self, name)
Definition: cmsPerfPublish.py:144
cmsPerfPublish.getNumOfEventsFromLog
def getNumOfEventsFromLog(logfile)
Definition: cmsPerfPublish.py:453
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
ALCARECOTkAlBeamHalo_cff.filter
filter
Definition: ALCARECOTkAlBeamHalo_cff.py:27
cmsPerfPublish.Row.addEntry
def addEntry(self, colname, value)
Definition: cmsPerfPublish.py:116
cmsPerfPublish.createHTMLtab
def createHTMLtab(INDEX, table_dict, ordered_keys, header, caption, name, mode=0)
Definition: cmsPerfPublish.py:1306
main
Definition: main.py:1
cmsPerfPublish.delTmpDir
def delTmpDir()
Delete tmp dir if we used it.
Definition: cmsPerfPublish.py:1850
cmsPerfPublish.createWebReports
def createWebReports(WebArea, repdir, ExecutionDate, LogFiles, cmsScimarkResults, date, prevrev)
Create web report index and create HTML file for each candle.
Definition: cmsPerfPublish.py:1450
cmsPerfPublish.step_cmp
def step_cmp(x, y)
Definition: cmsPerfPublish.py:778
cmsPerfPublish.fail
def fail(errstr="")
Definition: cmsPerfPublish.py:67
str
#define str(s)
Definition: TestProcessor.cc:48
cmsPerfPublish.Row.__init__
def __init__(self, table)
Definition: cmsPerfPublish.py:109
resolutioncreator_cfi.object
object
Definition: resolutioncreator_cfi.py:4
cmsPerfPublish.populateFromTupleRoot
def populateFromTupleRoot(tupname, repdir, rootfile, pureg)
Definition: cmsPerfPublish.py:1242
cmsPerfPublish.getRelativeDir
def getRelativeDir(parent, child, keepTop=True)
Some functions used for copying.
Definition: cmsPerfPublish.py:1858
cmsPerfPublish.getDirnameDirs
def getDirnameDirs(repdir, WebArea)
Grab dirs that end in strings defined in DirName.
Definition: cmsPerfPublish.py:1811
Generated for CMSSW Reference Manual by doxygen 1.8.11