14 import tempfile
as tmp
15 import optparse
as opt
16 import cmsPerfRegress
as cpr
17 import re, os, sys, time, glob, socket, fnmatch
18 from shutil
import copy2, copystat
20 from cmsPerfCommons
import CandFname, Step, ProductionSteps, Candles
25 PROG_NAME = os.path.basename(sys.argv[0])
26 DEF_RELVAL =
"/afs/cern.ch/cms/sdt/web/performance/RelVal"
27 DEF_SIMUL =
"/afs/cern.ch/cms/sdt/web/performance/simulation"
29 cpFileFilter = (
"*.root", )
32 TimeSizeNumOfEvents = -9999
33 IgProfNumOfEvents = -9999
34 CallgrindNumOfEvents = -9999
35 MemcheckNumOfEvents = -9999
53 Steps=set(Step+ProductionSteps+[
"GEN,FASTSIM",
"GEN,FASTSIM_PILEUP"])
62 "Relative directory could not be determined"
70 trail = re.compile(
"/$")
71 if os.path.isdir(adir)
and not trail.search(adir):
81 return subprocess.Popen(cmd,shell=
True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
88 return subprocess.Popen(command,shell=
True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
94 suffixes = [(
"B",2**10), (
"k",2**20), (
"M",2**30), (
"G",2**40), (
"T",2**50)]
95 for suf, lim
in suffixes:
100 return "-" + round(size/float(lim/2**10),2).__str__() + suf
102 return round(size/float(lim/2**10),2).__str__()+suf
114 self.table._newCol(colname)
129 out +=
"\n" + self.
rows
136 self.colNames.append(name)
142 if name
in self.rows.keys():
145 self.keys.append(name)
148 return self.
rows[name]
153 for key
in self.
keys:
159 rowobj = self.
rows[key]
160 rowdict = rowobj.getRowDict()
164 elif rowdict.has_key(col)
and not col == name:
166 total1 += rowdict[col]
168 (step_tot1, step_tot2) = rowdict[col]
172 rowobj.addEntry(name,total1)
174 rowobj.addEntry(name,(total1,total2))
179 if name
in self.rows.keys():
182 self.keys.append(name)
183 self.
rows[name] = row
187 for col
in self.colnames:
188 rowobj = transp.newRow(col)
189 for key
in self.
keys:
193 row_dict = self.
rows[key].getRowDict()
194 if row_dict.has_key(key):
195 rowobj.addEntry(key,row_dict[col])
203 global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
206 def _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage):
207 """Use function syscp to copy LogFiles and cmsScimarkDir over to staging area"""
209 print "Copying the logfiles to %s/." % stage
210 print "Copying the cmsScimark2 results to the %s/." % stage
212 syscp(LogFiles , stage +
"/")
213 syscp(cmsScimarkDir, stage +
"/")
216 def _createLogFile(LogFile,date,LocalPath,ShowTagsResult):
217 """Creating a small logfile with basic publication script running information (never used really by other scripts in the suite)."""
219 LOG = open(LogFile,
"w")
221 print "Writing Production Host, Location, Release and Tags information in %s" % LogFile
222 LOG.write(
"These performance tests were executed on host %s and published on %s" % (HOST,date))
223 LOG.write(
"They were run in %s" % LocalPath)
224 LOG.write(
"Results of showtags -r in the local release:\n%s" % ShowTagsResult)
226 except IOError, detail:
227 print "WARNING: Can't create log file"
235 print "\n Getting Environment variables..."
242 print "\n Determining locations for input and staging..."
243 (drive,path,remote,stage,port,repdir,prevrev,igprof_remotedir) =
getStageRepDirs(options,args)
246 print "\n Getting the number of events for each test..."
248 cmsPerfSuiteLogfile=
"%s/cmsPerfSuite.log"%repdir
250 if os.path.exists(cmsPerfSuiteLogfile):
252 (TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents)=
getNumOfEventsFromLog(cmsPerfSuiteLogfile)
257 print "There was an issue in reading out the number of events for the various tests or the architecture/CMSSW version using the standard logfile %s"%cmsPerfSuiteLogFile
258 print "Check that the format was not changed: this scripts relies on the initial perfsuite arguments to be dumped in the logfile one per line!"
259 print "For now taking the default values for all tests (0)!"
261 print "\n Scan report directory..."
263 (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir) =
scanReportArea(repdir)
264 print "cmsScimarkResults are %s"%cmsScimarkResults
265 print "\n Copy report files to staging directory..."
267 _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage)
269 print "\n Creating log file..."
271 _createLogFile(
"%s/ProductionLog.txt" % stage,date,repdir,ShowTagsResult)
274 for dirname
in os.listdir(repdir):
275 if "IgProf" in dirname:
276 print "\n Handling IgProf reports..."
280 print "\n Creating HTML files..."
282 createWebReports(stage,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
284 print "\n Copy profiling logs to staging directory..."
290 print "\n Uploading web report to remote location..."
292 print "\n Finished uploading! Now removing staging directory..."
295 print "\n Finished!!!"
302 global CMSSW_VERSION, CMSSW_RELEASE_BASE, CMSSW_BASE, HOST, USER, BASE_PERFORMANCE, CMSSW_WORK
303 global DEF_RELVAL, DEF_SIMUL
306 CMSSW_VERSION=os.environ[
'CMSSW_VERSION']
307 CMSSW_RELEASE_BASE=os.environ[
'CMSSW_RELEASE_BASE']
308 CMSSW_BASE=os.environ[
'CMSSW_BASE']
309 HOST=os.environ[
'HOST']
310 USER=os.environ[
'USER']
311 CMSSW_WORK = os.path.join(CMSSW_BASE,
"work/Results")
312 except KeyError, detail:
313 fail(
"ERROR: Could not retrieve some necessary environment variables. Have you ran scramv1 runtime -csh yet?. Details: %s" % detail)
319 PerformancePkg=
"%s/src/Validation/Performance" % CMSSW_BASE
320 if (os.path.exists(PerformancePkg)):
321 BASE_PERFORMANCE=PerformancePkg
322 print "**Using LOCAL version of Validation/Performance instead of the RELEASE version**"
324 BASE_PERFORMANCE=
"%s/src/Validation/Performance" % CMSSW_RELEASE_BASE
326 return (LocalPath,ShowTagsResult)
332 global PROG_NAME, _debug, _dryrun, _verbose
334 parser = opt.OptionParser(usage=(
"""%s [HOST:]DEST_PATH [Options]
337 [HOST:]DEST_PATH - This is where the report should be published, you can specify a local path or a directory on a remote machine (HOST is optional)
340 Publish report to default local directory
342 Publish report to \"/some/local/dir\"
344 Publish report to \"/some/other/dir\" remote host \"hal.cern.ch\"
345 ./%s hal.cern.ch:/some/other/dir
346 Publish report to default relval location (this could be remote or local depending on the hardcoded default)
348 % ( PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME)))
350 devel = opt.OptionGroup(parser,
"Developer Options",
351 "Caution: use these options at your own risk."
352 "It is believed that some of them bite.\n")
357 help=
'Use the default RelVal location',
366 help=
'output more information',
374 help=
'Use the default simulation location',
382 help=
'The override the name of the previous release. Default is the string obtain from the identification file REGRESSION.<prevrel>.vs.<newrel>',
391 help=
'The location of the report files to be published',
400 help=
'Use a particular port number to rsync material to a remote server',
407 default=
'IgProfData',
409 help=
'Specify an AFS or host:mydir remote directory instead of default one',
410 metavar=
'<IGPROF REMOTE DIRECTORY>'
418 help=
'Show debug output',
426 help=
'Do not send files to remote server, but run everything else',
430 repdirdef = os.getcwd()
431 parser.set_defaults(debug=0,simulation=
False,relval=
False,port=873,pretend=
False,repdir=repdirdef,verbose=
False)
432 parser.add_option_group(devel)
434 (options, args) = parser.parse_args()
436 _debug = options.debug
437 _dryrun = options.dryrun
438 _verbose = options.verbose
440 numofargs = len(args)
442 if (options.simulation
and options.relval)
or ((options.simulation
or options.relval)
and numofargs >= 1):
443 parser.error(
"You can not specify simulation and relval together. Neither can you specify simulation or relval AND a path")
446 return (options, args)
451 '''A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!'''
452 log=open(logfile,
"r")
461 if 'TimeSizeEvents' in line
and not TimeSizeEvents:
462 lineitems=line.split()
463 TimeSizeEvents=lineitems[lineitems.index(
'TimeSizeEvents')+1]
464 if 'IgProfEvents' in line
and not IgProfEvents:
465 lineitems=line.split()
466 IgProfEvents=lineitems[lineitems.index(
'IgProfEvents')+1]
467 if 'CallgrindEvents' in line
and not CallgrindEvents:
468 lineitems=line.split()
469 CallgrindEvents=lineitems[lineitems.index(
'CallgrindEvents')+1]
470 if 'MemcheckEvents' in line
and not MemcheckEvents:
471 lineitems=line.split()
472 MemcheckEvents=lineitems[lineitems.index(
'MemcheckEvents')+1]
473 return (TimeSizeEvents,IgProfEvents,CallgrindEvents,MemcheckEvents)
476 '''Another very fragile function to get the architecture and the CMSSW version parsing the logfile...'''
477 log=open(logfile,
"r")
478 arch=re.compile("^Current Architecture is")
479 version=re.compile(
"^Current CMSSW version is")
480 CMSSW_arch=
"UNKNOWN_ARCH"
481 CMSSW_version=
"UNKNOWN_VERSION"
483 if arch.search(line):
484 CMSSW_arch=line.split()[3]
485 if version.search(line):
486 CMSSW_version=line.split()[4]
487 return(CMSSW_arch,CMSSW_version)
495 global TMP_DIR, IS_TMP, DEF_LOCAL, CMSSW_VERSION
496 DEF_LOCAL = CMSSW_WORK
497 numofargs = len(args)
499 repdir = os.path.abspath(options.repdir)
502 if not os.path.exists(repdir):
503 fail(
"ERROR: The specified report directory %s to retrieve report information from does not exist, exiting" % repdir)
505 previousrev = options.previousrev
506 if previousrev ==
"":
507 regressfiles = glob.glob(
"%s/REGRESSION.*.vs.*" % repdir)
508 if not len(regressfiles) == 0:
509 regressID = regressfiles[0]
510 base = os.path.basename(regressID)
511 split = base.split(
".")
512 previousrev = split[1]
513 currentrel = split[3]
514 print "Regression Identification file exists, renaming report title for regression report. Old ver: %s" % previousrev
516 print "No regression ID file exists and previous release name was not specified. Producing normal report."
518 print "Previous release name was specified, renaming report title for regression report. Old ver %s" % previousrev
522 if options.simulation:
543 drive, path = uri.split(
":",1)
548 path = os.path.abspath(path)
549 remote =
not drive ==
""
554 socket.getaddrinfo(drive,53)
556 except socket.gaierror:
562 socket.gethostbyaddr(drive)
564 except socket.gaierror:
567 print "ERROR: Can not determine your hostname or ipv{4,6} address %s" % drive
568 if not (_dryrun
or _test):
571 if (
not remote)
and (
not options.port == 873) :
572 print "WARNING: Can not use a port if not performing a remote copy, ignoring"
581 localExists = os.path.exists(
"%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
586 TMP_DIR=tmp.mkdtemp(prefix=
"/build/%s" % PROG_NAME)
587 StagingArea = TMP_DIR
589 elif defaultlocal
and localExists:
590 TMP_DIR=tmp.mkdtemp(prefix=
"%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
591 StagingArea = TMP_DIR
592 print "WARNING: %s already exists, creating a temporary staging area %s" % (CMSSW_WORK,TMP_DIR)
595 StagingArea = CMSSW_WORK
597 os.mkdir(os.path.join(CMSSW_BASE,
"work"))
598 os.mkdir(os.path.join(CMSSW_BASE,
"work",
"Results"))
601 print "**User did not specify location of results, staging in default %s**" % StagingArea
603 print "**User chose to publish results in a local directory**"
605 if not os.path.exists(path):
607 os.mkdir(
"%s" % path)
608 except OSError, detail:
609 if detail.errno == 13:
610 fail(
"ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
611 elif detail.errno == 17:
615 fail(
"ERROR: There was some problem (%s) when creating the staging directory" % detail)
617 IS_TMP =
not TMP_DIR ==
""
623 StagingArea=
"%s/%s" % (StagingArea,CMSSW_VERSION)
625 os.mkdir(
"%s" % StagingArea)
626 except OSError, detail:
627 if detail.errno == 13:
628 fail(
"ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
629 elif detail.errno == 17:
633 fail(
"ERROR: There was some problem (%s) when creating the staging directory" % detail)
635 return (drive,path,remote,StagingArea,port,repdir,previousrev,options.ig_remotedir)
642 """Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results.
643 It returns Execution date (completion), current date, list of logfiles and cmsScimark results"""
645 LogFiles = glob.glob(repdir +
"cms*.log")
647 print "Found the following log files:"
650 cmsScimarkDir = glob.glob(repdir +
"cmsScimarkResults_*")
652 print "Found the following cmsScimark2 results directories:"
655 cmsScimarkResults = []
656 for adir
in cmsScimarkDir:
657 htmlfiles = glob.glob(adir +
"/*.html")
661 map(cmsScimarkResults.append,htmlfiles)
663 ExecutionDateLast =
""
666 cmsreg = re.compile(
"^cmsPerfSuite")
667 for logf
in LogFiles:
668 if cmsreg.search(logf):
669 ExecutionDateLastSec = os.stat(logf)[ST_CTIME]
670 ExecutionDateLast = os.stat(logf)[ST_MTIME]
672 print "Execution (completion) date for %s was: %s" % (logf,ExecutionDateLast)
673 if (ExecutionDateLastSec > ExecutionDateSec):
674 ExecutionDateSec = ExecutionDateLastSec
675 ExecutionDate = ExecutionDateLast
677 if ExecutionDate ==
"":
678 ExecutionDate = ExecutionDateLast
680 return (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir)
683 RegressTmplHTML=
"%s/doc/regress.html" % (BASE_PERFORMANCE)
684 candnreg = re.compile(
"CandleName")
685 candhreg = re.compile(
"CandlesHere")
687 REGR = open(reghtml,
"w")
688 for line
in open(RegressTmplHTML):
689 if candhreg.search(line):
693 abspath = os.path.join(repdir,outd)
694 if os.path.exists(abspath):
695 html +=
"<tr><td><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></td></tr>\n" % (outd,x,outd,x)
697 html +=
"<tr><td> %s does not exist probably because the log file for the previous release was missing</td></tr>" % (abspath)
700 elif candnreg.search(line):
701 REGR.write(CurrentCandle)
704 except IOError, detail:
705 print "ERROR: Could not write regression html %s because %s" % (os.path.basename(reghtml),detail)
708 logreg = re.compile(
"(.*)\.log$")
709 matches = logreg.search(reportName)
710 logdir = logreg.sub(matches.groups()[0],reportName)
711 outd = os.path.join(base,logdir)
712 nologext = matches.groups()[0]
713 return (nologext,outd)
718 x = os.path.basename(x)
719 y = os.path.basename(y)
720 stepreg = re.compile(
"%s_(..*)\.root" % fname)
721 if stepreg.search(x):
722 x = stepreg.search(x).groups()[0]
723 if stepreg.search(y):
724 y = stepreg.search(y).groups()[0]
730 x = os.path.basename(x)
731 y = os.path.basename(y)
732 stepreg = re.compile(
"%s_(..*)_%s" % (candle,prof))
733 if stepreg.search(x):
734 x = stepreg.search(x).groups()[0]
735 if stepreg.search(y):
736 y = stepreg.search(y).groups()[0]
742 x = os.path.basename(x)
743 y = os.path.basename(y)
744 stepreg = re.compile(
"%s_(..*)_%s_regression" % (candle,prof))
745 if stepreg.search(x):
746 x = stepreg.search(x).groups()[0]
747 if stepreg.search(y):
748 y = stepreg.search(y).groups()[0]
754 x = os.path.basename(x)
755 y = os.path.basename(y)
756 stepreg = re.compile(
"%s_(..*)_TimingReport.log" % fname)
757 if stepreg.search(x):
758 x = stepreg.search(x).groups()[0]
759 if stepreg.search(y):
760 y = stepreg.search(y).groups()[0]
766 x = os.path.basename(x)
767 y = os.path.basename(y)
768 stepreg = re.compile(
"%s_(..*)_TimingReport" % fname)
769 if stepreg.search(x):
770 x = stepreg.search(x).groups()[0]
771 if stepreg.search(y):
772 y = stepreg.search(y).groups()[0]
786 for i
in range(len(Step)):
787 stepreg = re.compile(
"^%s.*" % Step[i])
789 if Step[i]
in xstr
and sndbst_x == -1:
791 if Step[i]
in ystr
and sndbst_y == -1:
794 if xstr
in Step[i]
and bestx_idx == -1:
796 if ystr
in Step[i]
and besty_idx == -1:
799 if stepreg.search(xstr)
and x_idx == -1:
801 if stepreg.search(ystr)
and y_idx == -1:
804 if Step[i] == xstr
and x_idx == -1:
806 if Step[i] == ystr
and y_idx == -1:
808 if not ( x_idx == -1
or y_idx == -1):
829 if x_idx == -1
or y_idx == -1:
830 print "WARNING: No valid step names could be found in the logfiles or root filenames being sorted: x: %s y: %s." % (xstr,ystr)
831 print "x", x_idx,
"y", y_idx
845 def createCandlHTML(tmplfile,candlHTML,CurrentCandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
846 global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
847 def _stripbase(base, astr):
848 basereg = re.compile(
"^%s/?(.*)" % base)
850 found = basereg.search(astr)
852 out = found.groups()[0]
855 def _getProfileReportLink(repdir,CurrentCandle,CurDir,step,CurrentProfile,Profiler):
858 ProfileTemplate=os.path.join(repdir,
"%s_%s" % (CurrentCandle,CurDir),
"*_%s_%s*" % (step,CurrentProfile),Profiler)
861 ProfileTemplateLowCaps=os.path.join(repdir,
"%s_%s" % (CurrentCandle,CurDir),
"*_%s_%s*" % (step.lower(),CurrentProfile),Profiler)
862 ProfileReportLink = glob.glob(ProfileTemplate)
869 if len(ProfileReportLink) > 0:
871 if not reduce(
lambda x,y: x
or y,
map(
lambda x: CurrentCandle
in x,ProfileReportLink)):
872 ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
874 ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
875 ProfileReportLink =
map(
lambda x: _stripbase(repdir,x),ProfileReportLink)
877 return ProfileReportLink
879 def _writeReportLink(INDEX,ProfileReportLink,CurrentProfile,step,NumOfEvents,Profiler=""):
881 INDEX.write(
"<li><a href=\"%s\">%s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,step,NumOfEvents))
884 if CurrentProfile ==
"memcheck_valgrind":
885 INDEX.write(
"<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,
"5"))
887 INDEX.write(
"<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,NumOfEvents))
888 def IgProfDumpsTable(INDEX,ProfileLinks,step):
890 EndOfJobProfileLink=
filter(
lambda x:
"IgProfMemTotal.res" in x
or "IgProfMemLive.res" in x, ProfileLinks)[0]
892 ProfileLinks.remove(EndOfJobProfileLink)
894 ProfileLinks.sort(key=
lambda x: int(x.split(
".")[-2]))
896 IgProfMemLive_regexp=re.compile(
"IgProfMemLive")
897 if IgProfMemLive_regexp.search(EndOfJobProfileLink):
898 MemProfile=
"IgProf MEM LIVE"
900 MemProfile=
"IgProf MEM TOTAL"
902 INDEX.write(
"<li>%s"%MemProfile)
903 INDEX.write(
"<table><tr><td>Profile after event</td><td>Total Memory Size (bytes)</td><td>Total Calls (number)</td><td>Link to gzipped IgProf profile</td></tr>")
904 for link
in ProfileLinks:
906 gzProfile=os.path.join(link.split(
"/")[-3],link.split(
"/")[-1])[:-3]+
"gz"
907 if IgProfMemLive_regexp.search(gzProfile):
908 gzProfile=IgProfMemLive_regexp.sub(
r"IgProfMemTotal",gzProfile)
909 INDEX.write(
"<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(link.split(
".")[-2],open(link,
"r").readlines()[6].split()[1],open(link,"r").readlines()[6].split()[2],gzProfile,os.path.basename(gzProfile)))
911 gzEndOfJobProfileLink=os.path.join(EndOfJobProfileLink.split(
"/")[-3],EndOfJobProfileLink.split(
"/")[-1])[:-3]+
"gz"
912 if IgProfMemLive_regexp.search(gzEndOfJobProfileLink):
913 gzEndOfJobProfileLink=IgProfMemLive_regexp.sub(
r"IgProfMemTotal",gzEndOfJobProfileLink)
914 INDEX.write(
"<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(
"End of job",open(EndOfJobProfileLink,
"r").readlines()[6].split()[1],open(EndOfJobProfileLink,"r").readlines()[6].split()[2],gzEndOfJobProfileLink,os.path.basename(gzEndOfJobProfileLink)))
916 INDEX.write(
"</table>")
923 DirName[0] : TimeSizeNumOfEvents,
924 DirName[1] : IgProfNumOfEvents,
925 DirName[2] : IgProfNumOfEvents,
926 DirName[3] : IgProfNumOfEvents,
927 DirName[4] : CallgrindNumOfEvents,
928 DirName[5] : MemcheckNumOfEvents,
929 DirName[6] : TimeSizeNumOfEvents,
930 DirName[7] : IgProfNumOfEvents,
931 DirName[8] : IgProfNumOfEvents,
932 DirName[9] : IgProfNumOfEvents,
933 DirName[10] : CallgrindNumOfEvents,
934 DirName[11] : MemcheckNumOfEvents
950 IgProfMemAnalyseOut=(
951 "doBeginJob_output.html",
952 "doEvent_output.html",
956 memcheck_valgrindOut=(
962 Profile[0] :
"*TimingReport.html",
963 Profile[1] :
"TimeReport.html",
964 Profile[2] :
"*.html",
965 Profile[3] :
"objects_pp.html",
966 Profile[4] :
"overall.html",
967 Profile[5] :
"overall.html",
968 Profile[6] :
"overall.html",
969 Profile[7] :
"doBeginJob_output.html",
970 Profile[8] :
"overall.html",
971 Profile[9] :
"beginjob.html"
975 candnreg = re.compile(
"CandleName")
976 candhreg = re.compile(
"CandlesHere")
978 CAND = open(candlHTML,
"w")
979 for line
in open(tmplfile):
980 if candhreg.search(line):
981 CAND.write(
'<div id="content">')
983 CAND.write(CurrentCandle)
984 CAND.write(
"</h2>\n")
987 print "Producing candles html: ", CurrentCandle
989 for CurDir
in DirName:
991 LocalPath = os.path.join(repdir,
"%s_%s" % (CurrentCandle,CurDir))
992 LocalDirname = os.path.basename(LocalPath)
994 if not prevrev ==
"":
999 if CurDir == DirName[0]
or CurDir == DirName[6]:
1000 profs = Profile[0:4]
1001 elif CurDir == DirName[1]
or CurDir == DirName[7]:
1002 profs = Profile[4:8]
1003 elif CurDir == DirName[2]
or CurDir == DirName[8]:
1005 elif CurDir == DirName[3]
or CurDir == DirName[9]:
1007 elif CurDir == DirName[4]
or CurDir == DirName[10]:
1008 profs = [Profile[8]]
1009 elif CurDir == DirName[5]
or CurDir == DirName[11]:
1010 profs = [Profile[9]]
1015 print "Scanning for profile information for: ", prof
1018 fullprof = (CurrentCandle,prof)
1021 if prof ==
"TimingReport":
1022 timeReports = glob.glob(os.path.join(LocalPath,
"%s_*_%s.log" % (CandFname[CurrentCandle],prof)))
1023 if len(timeReports) > 0:
1025 CAND.write(
"<p><strong>%s %s</strong></p>\n" % (prof,
"Regression Analysis"))
1027 for log
in timeReports:
1028 reportName = os.path.basename(log)
1030 CAND.write(
"<h4>%s</h4>\n" % reportName)
1031 htmNames = [
"changes.png"]
1032 otherNames = [
"graphs.png" ,
"histos.png"]
1033 regressHTML=
"%s-regress.html" % nologext
1034 pathsExist = reduce (
lambda x,y: x
or y,
map(os.path.exists,
map(
lambda x: os.path.join(repdir,outd,x),otherNames)))
1037 html =
"<p>Performance graphs and histograms superimposed for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1039 html =
"<p>No change in performance graph available</p>\n"
1040 regressHTML=
"%s/%s" % (WebArea,regressHTML)
1043 abspath = os.path.join(repdir,outd,x)
1044 if os.path.exists(abspath):
1045 html +=
"<p><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></p>\n" % (outd,x,outd,x)
1047 html +=
"<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1051 CAND.write(
"\n</tr></table>")
1054 elif prof ==
"SimpleMemReport":
1055 simMemReports = glob.glob(os.path.join(LocalPath,
"%s_*_%s" % (CandFname[CurrentCandle],prof)))
1056 simMemReports =
map(
lambda x: (CandFname[CurrentCandle],prof,x), simMemReports )
1057 simMemReports.sort(cmp=dirname_cmp)
1058 simMemReports =
map(
lambda x: x[2], simMemReports )
1059 if len(simMemReports) > 0:
1061 CAND.write(
"<p><strong>%s %s</strong></p>\n" % (prof,
"Regression Analysis"))
1063 for adir
in simMemReports:
1064 reportName = os.path.basename(adir)
1065 CAND.write(
"<h4>%s</h4>\n" % reportName)
1066 htmNames = [
"vsize_change.png",
"rss_change.png"]
1067 otherNames = [
"vsize_graphs.png",
"rss_graphs.png"]
1068 nologext = reportName
1070 regressHTML=
"%s-regress.html" % nologext
1071 pathsExist = reduce (
lambda x,y: x
or y,
map(os.path.exists,
map(
lambda x: os.path.join(repdir,LocalDirname,outd,x),otherNames)))
1074 html =
"<p>Superimposed memory performance graphs for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1076 html =
"<p>No change in performance graph available</p>\n"
1077 regressHTML=
"%s/%s" % (WebArea,regressHTML)
1080 abspath = os.path.join(repdir,LocalDirname,outd,x)
1081 if os.path.exists(abspath):
1082 html +=
"<p><a href=\"./%s/%s/%s\"><img src=\"./%s/%s/%s\" /></a></p>\n" % (LocalDirname,outd,x,LocalDirname,outd,x)
1084 html +=
"<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1086 createRegressHTML(regressHTML,repdir,
"%s/%s" % (LocalDirname,outd),CurrentCandle,otherNames)
1088 CAND.write(
"\n</tr></table>")
1090 elif prof ==
"EdmSize" or prof ==
"IgProfMemTotal" or prof ==
"IgProfMemLive" or prof==
"IgProfperf" or prof ==
"Callgrind":
1091 regresPath = os.path.join(LocalPath,
"%s_*_%s_regression" % (CandFname[CurrentCandle],prof))
1092 regresses = glob.glob(regresPath)
1093 stepreg = re.compile(
"%s_([^_]*(_PILEUP)?)_%s_regression" % (CandFname[CurrentCandle],prof))
1094 if len(regresses) > 0:
1096 CAND.write(
"<p><strong>%s %s</strong></p>\n" % (prof,
"Regression Analysis"))
1098 regresses =
map(
lambda x: (CandFname[CurrentCandle],prof,x),regresses)
1099 regresses.sort(cmp=reg_dirname_cmp)
1100 regresses =
map(
lambda x: x[2],regresses)
1101 for rep
in regresses:
1102 base = os.path.basename(rep)
1103 found = stepreg.search(base)
1104 step =
"Unknown-step"
1106 step = found.groups()[0]
1108 if prof ==
"IgProfMemLive" or prof ==
"IgProfMemTotal" or prof==
"IgProfperf" or prof ==
"Callgrind":
1109 htmlpage =
"overall.html"
1111 htmlpage =
"objects_pp.html"
1112 CAND.write(
"<a href=\"%s/%s/%s\">%s %s regression report</a><br/>\n" % (LocalDirname,base,htmlpage,prof,step))
1115 if os.path.exists(LocalPath):
1116 thedir = os.listdir(LocalPath)
1117 CandleLogFiles =
filter(
lambda x: (x.endswith(
".log")
or x.endswith(
"EdmSize"))
and not os.path.isdir(x)
and os.path.exists(x),
map(
lambda x: os.path.abspath(os.path.join(LocalPath,x)),thedir))
1119 if (len(CandleLogFiles)>0):
1121 syscp(CandleLogFiles,WebArea +
"/")
1122 base = os.path.basename(LocalPath)
1125 for cand
in CandleLogFiles:
1126 cand = os.path.basename(cand)
1128 print "Found %s in %s\n" % (cand,LocalPath)
1130 if not "EdmSize" in cand:
1131 lfileshtml +=
"<a href=\"./%s/%s\">%s </a><br/>" % (base,cand,cand)
1133 CAND.write(
"<p><strong>Logfiles for %s</strong></p>\n" % CurDir)
1134 CAND.write(lfileshtml)
1137 for CurrentProfile
in Profile:
1142 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1146 OutputHtml[CurrentProfile])
1149 isProfinLink =
False
1150 if len (ProfileReportLink) > 0:
1151 isProfinLink = reduce(
lambda x,y: x
or y,
map(
lambda x: CurrentProfile
in x,ProfileReportLink))
1156 if (PrintedOnce==
False):
1160 CAND.write(
"<p><strong>%s</strong></p>\n" % CurDir)
1161 CAND.write(
"<ul>\n")
1166 if (CurrentProfile == Profile[5]
or CurrentProfile == Profile[6]):
1167 for prolink
in ProfileReportLink:
1168 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1169 for igprofmem
in [
"*IgProfMemTotal*.res",
"*IgProfMemLive*.res"]:
1170 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1175 isProfinLink =
False
1176 if len (ProfileReportLink) > 0:
1177 isProfinLink = reduce(
lambda x,y: x
or y,
map(
lambda x: CurrentProfile
in x,ProfileReportLink))
1180 IgProfDumpsTable(CAND,ProfileReportLink,step)
1183 elif (CurrentProfile == Profile[7]):
1184 for igprof
in IgProfMemAnalyseOut:
1185 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1190 isProfinLink =
False
1191 if len (ProfileReportLink) > 0:
1192 isProfinLink = reduce(
lambda x,y: x
or y,
map(
lambda x: CurrentProfile
in x,ProfileReportLink))
1194 for prolink
in ProfileReportLink:
1195 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=igprof)
1198 elif (CurrentProfile == Profile[9]):
1200 for memprof
in memcheck_valgrindOut:
1202 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1208 isProfinLink =
False
1209 if len (ProfileReportLink) > 0:
1210 isProfinLink = reduce(
lambda x,y: x
or y,
map(
lambda x: CurrentProfile
in x,ProfileReportLink))
1212 for prolink
in ProfileReportLink:
1213 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=memprof)
1216 for prolink
in ProfileReportLink:
1217 if "regression" not in prolink:
1218 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1223 CAND.write(
"</ul>\n")
1226 CAND.write(
"<hr />")
1227 CAND.write(
"<br />\n")
1228 CAND.write(
"</div>\n")
1229 elif candnreg.search(line):
1230 CAND.write(CurrentCandle)
1235 except IOError, detail:
1236 print "ERROR: Could not write candle html %s because %s" % (os.path.basename(candlHTML),detail)
1241 for cand
in Candles:
1242 fname = CandFname[cand]
1243 globpath = os.path.join(repdir,
"%s_TimeSize" % cand,
"%s_*_TimingReport" % fname)
1244 stepDirs = glob.glob(globpath)
1245 stepDirs =
map(
lambda x: (fname,x), stepDirs)
1246 stepDirs.sort(cmp=timerep_cmp)
1247 stepDirs =
map(
lambda x: x[1], stepDirs)
1248 stepreg = re.compile(
"%s_(.*)_TimingReport" % fname)
1253 for stepdir
in stepDirs:
1254 base = os.path.basename(stepdir)
1255 found = stepreg.search(base)
1256 step =
"Unknown-step"
1258 step = found.groups()[0]
1259 realstep =
"Unknown-step"
1260 if "PILEUP" in step:
1261 found = pureg.search(step)
1263 realstep = found.groups()[0]
1267 rootf = os.path.join(stepdir,rootfile)
1269 if os.path.exists(rootf):
1270 f = ROOT.TFile(rootf)
1272 cpu_time_tree = ROOT.TTree()
1273 f.GetObject(
"cpu_time_tuple;1",cpu_time_tree)
1275 if cpu_time_tree.InheritsFrom(
"TTree"):
1278 for t
in cpu_time_tree:
1283 createNewRow =
False
1284 curRow = table.newRow(cand)
1285 data_tuple = (data1,data2)
1287 if "PILEUP" in step:
1288 puRow.addEntry(realstep,data_tuple)
1291 createNewRow =
False
1292 curRow = table.newRow(cand)
1294 curRow.addEntry(step,data_tuple)
1299 table.addRow(puRow,
"%s PILEUP" %cand)
1304 cols = len(ordered_keys)
1305 totcols = (cols * 3) + 1
1307 colspan = totcols - 1
1310 labels = [
"fs1",
"fs2",
"Δ"]
1311 elif mode == 2
or mode == 3:
1315 labels = [
"t1" ,
"t2" ,
"Δ"]
1318 INDEX.write(
"<h3>%s</h3>\n" % header)
1319 INDEX.write(
"<table>\n")
1320 INDEX.write(
"<caption>%s</caption>\n" % caption)
1321 INDEX.write(
"<thead><tr><th></th><th colspan=\"%s\" scope=\"colgroup\">%s</th></tr></thead>" % (colspan,name))
1322 INDEX.write(
"<tbody>\n")
1323 for key
in ordered_keys:
1326 INDEX.write(
"<th></th>")
1328 INDEX.write(
"<td scope=\"row\">")
1330 INDEX.write(
"</td>")
1331 for col
in table_dict[
None]:
1333 INDEX.write(
"<th colspan=\"%s\" scope=\"col\">" % innercol)
1335 INDEX.write(
"</th>")
1337 rowdict = table_dict[key].getRowDict()
1338 if rowdict.has_key(col):
1342 INDEX.write(
"%s" % dat)
1343 INDEX.write(
"</td>")
1348 INDEX.write(
"%6.2f" % dat)
1349 INDEX.write(
"</td>")
1351 (data1, data2) = rowdict[col]
1352 diff = data2 - data1
1359 seq = [ data1, data2, diff ]
1361 INDEX.write(
"<td id=\"data\">")
1363 INDEX.write(
"%s" % dat)
1365 INDEX.write(
"%6.2f" % dat)
1367 INDEX.write(
"</td>")
1369 if mode == 2
or mode == 3:
1372 INDEX.write(
"</td>")
1377 INDEX.write(
"</td>")
1378 INDEX.write(
"</tr>\n")
1381 if not (mode == 2
or mode == 3 ):
1384 INDEX.write(
"<th>Candles</th>")
1385 for col
in table_dict[
None]:
1386 INDEX.write(
"<th>%s</th>" % labels[0])
1387 INDEX.write(
"<th>%s</th>" % labels[1])
1388 INDEX.write(
"<th>%s</th>" % labels[2])
1389 INDEX.write(
"</tr>\n")
1390 INDEX.write(
"</tbody></table>\n")
1392 INDEX.write(
"<br />")
1395 '''Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.'''
1398 if ":" in remotedir:
1399 (host,dir)=remotedir.split(
":")
1400 mkdir_cmd=
"ssh %s (mkdir %s;mkdir %s/%s)"%(host,dir,dir,arch)
1402 mkdir_cmd=
"mkdir %s;mkdir %s/%s"%(remotedir,remotedir,arch)
1407 os.system(mkdir_cmd)
1408 print "Successfully created publication directory"
1410 print "Issues with publication directory existence/creation!"
1415 if ":" in remotedir:
1416 tarpipe_cmd=
'tar cf - *_IgProf_*/*.sql3 | ssh %s "cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*"'%(host,dir,arch,version,version)
1418 tarpipe_cmd=
'tar cf - *_IgProf_*/*.sql3 | (cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*)'%(remotedir,arch,version,version)
1423 os.system(tarpipe_cmd)
1424 print "Successfully copied IgProf reports to %s"%remotedir
1426 print "Issues with rsyncing to the remote directory %s!"%remotedir
1429 if ":" in remotedir:
1430 chmod_cmd=
"ssh %s chmod -R 775 %s/%s"%(host,dir,arch)
1432 chmod_cmd=
"chmod -R 775 %s/%s"%(remotedir,arch)
1435 os.system(chmod_cmd)
1436 print "Successfully set permissions for IgProf reports directory %s"%remotedir
1438 print "(Potential) issues with chmoding the remote directory %s!"%remotedir
1453 CmsDriverCandle = CandFname
1457 IndexFile=
"%s/index.html" % WebArea
1458 TemplateHtml=
"%s/doc/index.html" % BASE_PERFORMANCE
1460 cmsverreg = re.compile(
"CMSSW_VERSION")
1461 hostreg = re.compile(
"HOST")
1462 lpathreg = re.compile(
"LocalPath")
1463 fsizereg = re.compile(
"FSizeTable")
1464 cpureg = re.compile(
"CPUTable")
1465 proddreg = re.compile(
"ProductionDate")
1466 logfreg = re.compile(
"LogfileLinks")
1467 dirbreg = re.compile(
"DirectoryBrowsing")
1468 pubdreg = re.compile(
"PublicationDate")
1469 candhreg = re.compile(
"CandlesHere")
1473 CandlTmpltHTML=
"%s/doc/candle.html" % BASE_PERFORMANCE
1475 print "Copying %s/doc/perf_style.css style file to %s/." % (BASE_PERFORMANCE,WebArea)
1476 print "Template used: %s" % TemplateHtml
1478 syscp((BASE_PERFORMANCE +
"/doc/perf_style.css"),WebArea +
"/.")
1479 pureg = re.compile(
"(.*)_PILEUP")
1481 INDEX = open(IndexFile,
"w")
1482 for NewFileLine
in open(TemplateHtml) :
1483 if cmsverreg.search(NewFileLine):
1485 INDEX.write(
"Performance Reports for %s\n" % CMSSW_VERSION)
1487 globpath = os.path.join(repdir,
"REGRESSION.%s.vs.*" % (prevrev))
1488 globs = glob.glob(globpath)
1492 latestreg = re.compile(
"REGRESSION.%s.vs.(.*)" % prevrev)
1493 found = latestreg.search(os.path.basename(globs[0]))
1495 latestrel = found.groups()[0]
1496 INDEX.write(
"Performance Reports with regression: %s VS %s\n" % (prevrev,latestrel))
1498 INDEX.write(
"Performance Reports with regression: %s VS %s\n" % (prevrev,CMSSW_VERSION))
1499 elif hostreg.search(NewFileLine):
1500 INDEX.write(HOST +
"\n")
1502 elif fsizereg.search(NewFileLine):
1507 for cand
in Candles:
1508 fname = CandFname[cand]
1509 globpath = os.path.join(repdir,
"%s*_TimeSize" % cand,
"%s_*.root" % fname)
1510 rootfiles = glob.glob(globpath)
1511 rootfiles =
map(
lambda x: (fname,x), rootfiles)
1512 rootfiles.sort(cmp=rootfile_cmp)
1513 rootfiles =
map(
lambda x: x[1], rootfiles)
1514 stepreg = re.compile(
"%s_(.*).root" % fname)
1519 for rootf
in rootfiles:
1520 base = os.path.basename(rootf)
1521 found = stepreg.search(base)
1522 step =
"Unknown-step"
1524 step = found.groups()[0]
1525 realstep =
"Unknown-step"
1526 if "PILEUP" in step:
1527 found = pureg.search(step)
1529 realstep = found.groups()[0]
1532 puRow =
Row(fsize_tab)
1534 statinfo = os.stat(rootf)
1535 fsize = statinfo.st_size
1537 createNewRow =
False
1538 curRow = fsize_tab.newRow(cand)
1540 if "PILEUP" in step:
1541 puRow.addEntry(realstep,fsize)
1544 createNewRow =
False
1545 curRow = fsize_tab.newRow(cand)
1547 curRow.addEntry(step,fsize)
1548 except IOError, detail:
1550 except OSError, detail:
1555 fsize_tab.addRow(puRow,
"%s PILEUP" %cand)
1556 (ordered_keys,table_dict) = fsize_tab.getTable(1)
1557 cols = len(ordered_keys)
1559 if len(table_dict) > 1
and cols > 0:
1561 "Release ROOT file sizes",
1562 "Table showing current release ROOT filesizes in (k/M/G) bytes.",
1567 globpath = os.path.join(repdir,
"REGRESSION.%s.vs.*" % (prevrev))
1568 globs = glob.glob(globpath)
1570 raise IOError(2,globpath,
"File does not exist",
"File does not exist")
1571 idfile = open(globs[0])
1575 oldpath = oldpath.strip()
1579 for cand
in Candles:
1580 fname = CandFname[cand]
1581 globpath = os.path.join(repdir,
"%s*_TimeSize" % cand,
"%s_*.root" % fname)
1582 rootfiles = glob.glob(globpath)
1583 rootfiles =
map(
lambda x: (fname,x), rootfiles)
1584 rootfiles.sort(cmp=rootfile_cmp)
1585 rootfiles =
map(
lambda x: x[1], rootfiles)
1586 stepreg = re.compile(
"%s_(.*).root" % fname)
1591 for rootf
in rootfiles:
1592 base = os.path.basename(rootf)
1593 found = stepreg.search(base)
1594 step =
"Unknown-step"
1596 step = found.groups()[0]
1598 realstep =
"Unknown-step"
1599 if "PILEUP" in step:
1600 found = pureg.search(step)
1602 realstep = found.groups()[0]
1605 puRow =
Row(fsize_tab)
1607 statinfo = os.stat(rootf)
1608 fsize2 = statinfo.st_size
1609 oldfile = os.path.join(oldpath,
"%s_TimeSize" % cand,base)
1610 if "PILEUP" in step:
1611 oldfile = os.path.join(oldpath,
"%s_PU_TimeSize" % cand,base)
1614 if os.path.exists(oldfile):
1615 statinfo = os.stat(oldfile)
1616 fsize1 = statinfo.st_size
1618 print "######DID NOT FIND Previous file (needed for the filesize table): %s"%oldfile
1621 createNewRow =
False
1622 curRow = fsize_tab.newRow(cand)
1624 data_tuple = (fsize1,fsize2)
1625 if "PILEUP" in step:
1626 puRow.addEntry(realstep,data_tuple)
1629 createNewRow =
False
1630 curRow = fsize_tab.newRow(cand)
1632 curRow.addEntry(step,data_tuple)
1633 except IOError, detail:
1635 except OSError, detail:
1640 fsize_tab.addRow(puRow,
"%s PILEUP" %cand)
1642 (ordered_keys,table_dict) = fsize_tab.getTable()
1643 cols = len(ordered_keys)
1645 if len(table_dict) > 1
and cols > 0:
1647 "Release ROOT file sizes",
1648 "Table showing previous release ROOT filesizes, fs1, latest sizes, fs2, and the difference between them Δ in (k/M/G) bytes.",
1650 except IOError, detail:
1652 except OSError, detail:
1655 elif cpureg.search(NewFileLine):
1660 for cand
in Candles:
1661 fname = CandFname[cand]
1662 globpath = os.path.join(repdir,
"%s*_TimeSize" % cand,
"%s_*_TimingReport.log" % fname)
1663 logfiles = glob.glob(globpath)
1664 logfiles =
map(
lambda x: (fname,x), logfiles)
1665 logfiles.sort(cmp=logrep_cmp)
1666 logfiles =
map(
lambda x: x[1], logfiles)
1668 stepreg = re.compile(
"%s_(.*)_TimingReport.log" % fname)
1673 for log
in logfiles:
1674 base = os.path.basename(log)
1675 found = stepreg.search(base)
1676 step =
"Unknown-step"
1678 step = found.groups()[0]
1680 realstep =
"Unknown-step"
1681 if "PILEUP" in step:
1682 found = pureg.search(step)
1684 realstep = found.groups()[0]
1687 puRow =
Row(time_tab)
1689 data = cpr.getTimingLogData(log)
1692 for evtnum, time
in data:
1696 mean = mean / float(i)
1697 except ZeroDivisionError, detail:
1698 print "WARNING: Could not calculate mean CPU time from log because no events could be parsed", log
1700 if "PILEUP" in step:
1701 puRow.addEntry(realstep,mean)
1704 createNewRow =
False
1705 curRow = time_tab.newRow(cand)
1707 curRow.addEntry(step,mean)
1711 time_tab.addRow(puRow,
"%s PILEUP" %cand)
1713 (ordered_keys,table_dict) = time_tab.getTable(1)
1714 cols = len(ordered_keys)
1716 if len(table_dict) > 1
and cols > 0:
1718 "Release CPU times",
1719 "Table showing current release CPU times in secs.",
1737 (ordered_keys,table_dict) = cpu_time_tab.getTable()
1739 cols = len(ordered_keys)
1740 if len(table_dict) > 1
and cols > 0:
1742 "Release CPU times",
1743 "Table showing previous release CPU times, t1, latest times, t2, and the difference between them Δ in secs.",
1748 elif lpathreg.search(NewFileLine):
1749 INDEX.write(repdir +
"\n")
1750 elif proddreg.search(NewFileLine):
1751 INDEX.write(ExecutionDate +
"\n")
1752 elif logfreg.search(NewFileLine):
1753 INDEX.write(
"<br />\n")
1754 for log
in LogFiles:
1755 log = os.path.basename(log)
1757 print "linking log file %s" % log
1758 INDEX.write(
"<a href=\"./%s\"> %s </a>" % (log,log))
1759 INDEX.write(
"<br />\n")
1761 INDEX.write(
"Results for cmsScimark2 benchmark (running on the other cores) available at:\n")
1762 for cmssci
in cmsScimarkResults:
1763 scimarkdirs=cmssci.split(
"/")
1764 localdir=scimarkdirs[-2]
1766 cmssci = os.path.basename(cmssci)
1767 relativelink=os.path.join(localdir,cmssci)
1769 INDEX.write(
"<a href=\"%s\"> %s </a>" % (relativelink,cmssci))
1770 INDEX.write(
"<br />\n")
1773 elif dirbreg.search(NewFileLine):
1781 INDEX.write(
"Click <a href=\"./DirectoryBrowsing/\">here</a> to browse the directory containing all results (except the root files)\n")
1783 elif pubdreg.search(NewFileLine):
1784 INDEX.write(date +
"\n")
1785 elif candhreg.search(NewFileLine):
1786 for acandle
in Candle:
1787 globpath = os.path.join(repdir,
"%s_*" % acandle)
1788 globs = glob.glob(globpath)
1790 candlHTML =
"%s.html" % acandle
1791 INDEX.write(
"<a href=\"./%s\"> %s </a>" % (candlHTML,acandle))
1792 INDEX.write(
"<br />\n")
1794 candlHTML=os.path.join(WebArea,candlHTML)
1795 createCandlHTML(CandlTmpltHTML,candlHTML,acandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
1797 INDEX.write(NewFileLine)
1801 except IOError, detail:
1802 print "Error: Could not create index Html file for some reason, check position. Details : %s" % detail
1809 Dir = os.listdir(repdir)
1810 def _containsDirName(elem):
1811 return reduce(
lambda x,y: x
or y,
map(
lambda x: x
in elem, DirName))
1812 def _print4Lambda(elem,WebArea):
1814 print "Copying %s to %s\n" % (elem,WebArea)
1816 dirstocp =
filter(
lambda x: _containsDirName(x),
map(
lambda x: repdir + x,Dir))
1817 map(
lambda x: _print4Lambda(x,WebArea),dirstocp)
1818 syscp(dirstocp,WebArea +
"/")
1819 os.mkdir(
"%s/DirectoryBrowsing" % WebArea)
1820 for file
in os.listdir(WebArea):
1821 if file !=
"index.html":
1823 os.symlink(
"%s/%s"%(
"..",file),
"%s/DirectoryBrowsing/%s" % (WebArea,file))
1834 args =
"--port=%s %s %s:%s" % (port,os.path.normpath(stage),drive,path)
1837 print cmd +
" --dry-run " + args
1838 retval = os.system(cmd +
" --dry-run " + args )
1841 retval = os.system(cmd +
" " + args)
1848 if os.path.exists(TMP_DIR)
and IS_TMP:
1849 os.system(
"rm -Rf " + TMP_DIR)
1856 def _walkpath(path):
1859 head , tail = os.path.split(path)
1864 for i
in range(len(dirs)-1,-1,-1):
1868 pwalk = _walkpath(parent)
1874 except StopIteration:
1880 cwalk = _walkpath(child)
1885 except StopIteration:
1886 print "ERROR: Unable to determine relative dir"
1892 relpath=os.path.join(relpath,cwalk.next())
1893 except StopIteration:
1900 except OSError, detail:
1901 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1902 except IOError, detail:
1903 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1906 print "Copied %s to %s" % (src,dest)
1909 def _getNewLocation(source,child,dst,keepTop=keepTop):
1911 return os.path.join(dst,place)
1912 def _copyFilter(source,dst,curdir,fsnodes,filter,dirs=False):
1913 for node
in fsnodes:
1915 filterExist =
not len(filter) == 0
1917 dontFilter =
not reduce(
lambda x,y: x
or y,
map(
lambda x: fnmatch.fnmatch(node,x),filter))
1919 node = os.path.join(curdir,node)
1921 newnode = _getNewLocation(source,node,dst)
1926 except IOError, detail:
1927 print "WARNING: Could not copy %s to %s because %s" % (node,newnode,detail)
1928 except OSError, detail:
1929 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1930 except ReldirExcept:
1931 print "WARNING: Could not determine new location for source %s into destination %s" % (source,dst)
1935 match = fnmatch.fnmatch(node,filter[0])
1937 except AssertionError, detail:
1938 print node, filter[0], match
1942 print "Filter %s Copied %s to %s" % (dontFilter,node,newnode)
1943 print "fnmatch %s" % fnmatch.fnmatch(node,cpFileFilter[0])
1947 newloc = _getNewLocation(src,src,dest)
1957 _copyFilter(src,dest,curdir,dirs,cpDirFilter,dirs=
True)
1958 _copyFilter(src,dest,curdir,files,cpFileFilter)
1960 except StopIteration:
1962 except IOError, detail:
1963 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1964 except OSError, detail:
1965 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1966 except ReldirExcept:
1967 print "WARNING: Could not determine the new location for source %s into destination %s" % (src,dest)
1970 if type(srcs) == type(
""):
1971 if os.path.exists(srcs):
1972 if os.path.isdir(srcs):
1977 print "ERROR: file to be copied %s does not exist" % foo
1980 if os.path.exists(src):
1981 if os.path.isdir(src):
1987 print "ERROR: file to be copied %s does not exist" % foo
1990 print "%s\n" % PROG_NAME
1992 if __name__ ==
"__main__":
def getNumOfEventsFromLog
def populateFromTupleRoot
def getRelativeDir
Some functions used for copying.
def getDirnameDirs
Grab dirs that end in strings defined in DirName.
def createCandlHTML
Create HTML pages for candles.
def scanReportArea
Scan report area for required things.
def delTmpDir
Delete tmp dir if we used it.
def syncToRemoteLoc
Upload stage to remote location.
def getStageRepDirs
Determine locations of staging and report dirs.
def createWebReports
Create web report index and create HTML file for each candle.
def get_environ
Get require environment variables.
def getArchVersionFromLog
return(e1-e2)*(e1-e2)+dp *dp
def optionparse
Option parser.