CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
cmsPerfPublish Namespace Reference

Classes

class  ReldirExcept
 Small functions. More...
 
class  Row
 
class  Table
 

Functions

def addtrailingslash
 
def copytree4
 
def createCandlHTML
 Create HTML pages for candles. More...
 
def createHTMLtab
 
def createRegressHTML
 
def createWebReports
 Create web report index and create HTML file for each candle. More...
 
def delTmpDir
 Delete tmp dir if we used it. More...
 
def dirname_cmp
 
def docopy
 
def fail
 
def get_environ
 Get require environment variables. More...
 
def getArchVersionFromLog
 
def getcmd
 
def getcmdBasic
 
def getDate
 
def getDirnameDirs
 Grab dirs that end in strings defined in DirName. More...
 
def getNumOfEventsFromLog
 
def getOutputNames
 
def getRelativeDir
 Some functions used for copying. More...
 
def getStageRepDirs
 Determine locations of staging and report dirs. More...
 
def logrep_cmp
 
def main
 
def optionparse
 Option parser. More...
 
def populateFromTupleRoot
 
def prettySize
 
def print_header
 
def reg_dirname_cmp
 
def rootfile_cmp
 
def scanReportArea
 Scan report area for required things. More...
 
def stageIgProfReports
 
def step_cmp
 
def syncToRemoteLoc
 Upload stage to remote location. More...
 
def syscp
 
def timerep_cmp
 

Variables

int CallgrindNumOfEvents = -9999
 
tuple cpDirFilter = ( )
 
tuple cpFileFilter = ( "*.root", )
 
string DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal"
 
string DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation"
 
tuple DirName
 
int IgProfNumOfEvents = -9999
 
int MemcheckNumOfEvents = -9999
 
tuple PROG_NAME = os.path.basename(sys.argv[0])
 
tuple Steps = set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"])
 
int TimeSizeNumOfEvents = -9999
 
string TMP_DIR = ""
 

Function Documentation

def cmsPerfPublish.addtrailingslash (   adir)

Definition at line 70 of file cmsPerfPublish.py.

Referenced by getStageRepDirs(), and syncToRemoteLoc().

70 
71 def addtrailingslash(adir):
72  trail = re.compile("/$")
73  if os.path.isdir(adir) and not trail.search(adir):
74  adir = adir + "/"
75  return adir
def cmsPerfPublish.copytree4 (   src,
  dest,
  keepTop = True 
)

Definition at line 1909 of file cmsPerfPublish.py.

References getRelativeDir(), and GetRecoTauVFromDQM_MC_cff.next.

Referenced by syscp().

1909 
1910 def copytree4(src,dest,keepTop=True):
1911  def _getNewLocation(source,child,dst,keepTop=keepTop):
1912  place = getRelativeDir(source,child,keepTop=keepTop)
1913  return os.path.join(dst,place)
1914  def _copyFilter(source,dst,curdir,fsnodes,filter,dirs=False):
1915  for node in fsnodes:
1916  dontFilter = True
1917  filterExist = not len(filter) == 0
1918  if filterExist:
1919  dontFilter = not reduce(lambda x,y: x or y,map(lambda x: fnmatch.fnmatch(node,x),filter))
1920  if dontFilter:
1921  node = os.path.join(curdir,node) # convert to absolute path
1922  try:
1923  newnode = _getNewLocation(source,node,dst)
1924  if dirs:
1925  os.mkdir(newnode)
1926  else:
1927  copy2(node,newnode)
1928  except IOError as detail:
1929  print "WARNING: Could not copy %s to %s because %s" % (node,newnode,detail)
1930  except OSError as detail:
1931  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1932  except ReldirExcept:
1933  print "WARNING: Could not determine new location for source %s into destination %s" % (source,dst)
1934  else:
1935  if len(filter) > 0:
1936  try:
1937  match = fnmatch.fnmatch(node,filter[0])
1938  assert not match
1939  except AssertionError as detail:
1940  print node, filter[0], match
1941  raise RuntimeError
1942  if _verbose:
1943  if "root" in node:
1944  print "Filter %s Copied %s to %s" % (dontFilter,node,newnode)
1945  print "fnmatch %s" % fnmatch.fnmatch(node,cpFileFilter[0])
1946 
1947  gen = os.walk(src)
1948  try:
1949  newloc = _getNewLocation(src,src,dest)
1950 
1951  os.mkdir(newloc)
1952  try:
1953  while True:
1954  step = next(gen)
1955  curdir = step[0]
1956  dirs = step[1]
1957  files = step[2]
1958 
1959  _copyFilter(src,dest,curdir,dirs,cpDirFilter,dirs=True)
1960  _copyFilter(src,dest,curdir,files,cpFileFilter)
1961 
1962  except StopIteration:
1963  pass
1964  except IOError as detail:
1965  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1966  except OSError as detail:
1967  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1968  except ReldirExcept:
1969  print "WARNING: Could not determine the new location for source %s into destination %s" % (src,dest)
cmsPerfPublish.getRelativeDir
def getRelativeDir
Some functions used for copying.
Definition: cmsPerfPublish.py:1856
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
def cmsPerfPublish.createCandlHTML (   tmplfile,
  candlHTML,
  CurrentCandle,
  WebArea,
  repdir,
  ExecutionDate,
  LogFiles,
  cmsScimarkResults,
  date,
  prevrev 
)

Create HTML pages for candles.

Definition at line 846 of file cmsPerfPublish.py.

References createRegressHTML(), alcazmumu_cfi.filter, getOutputNames(), and syscp().

Referenced by createWebReports().

846 
847 def createCandlHTML(tmplfile,candlHTML,CurrentCandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
848  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
849  def _stripbase(base, astr):
850  basereg = re.compile("^%s/?(.*)" % base)
851  out = astr
852  found = basereg.search(astr)
853  if found:
854  out = found.groups()[0]
855  return out
856 
857  def _getProfileReportLink(repdir,CurrentCandle,CurDir,step,CurrentProfile,Profiler):
858  #FIXME:
859  #Pileup now has it own directory... should add it in the DirName dictionary at the beginning?
860  ProfileTemplate=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step,CurrentProfile),Profiler)
861  #print ProfileTemplate
862  #There was the issue of SIM vs sim (same for DIGI) between the previous RelVal based performance suite and the current.
863  ProfileTemplateLowCaps=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step.lower(),CurrentProfile),Profiler)
864  ProfileReportLink = glob.glob(ProfileTemplate)
865  #Filter out the Pile up directories when step does not contain Pile up
866  #if not ('PILEUP' in step):
867  # print "NPNPNPNP BEFORE %s"%ProfileReportLink
868  # ProfileReportLink=filter(lambda x: "PU" not in x,ProfileReportLink)
869  # print "NPNPNPNP %s"%ProfileReportLink
870  #print ProfileReportLink
871  if len(ProfileReportLink) > 0:
872  #print ProfileReportLink
873  if not reduce(lambda x,y: x or y,map(lambda x: CurrentCandle in x,ProfileReportLink)):# match with caps try low caps
874  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
875  else:
876  ProfileReportLink = glob.glob(ProfileTemplateLowCaps)
877  ProfileReportLink = map(lambda x: _stripbase(repdir,x),ProfileReportLink)
878  #print ProfileReportLink
879  return ProfileReportLink
880 
881  def _writeReportLink(INDEX,ProfileReportLink,CurrentProfile,step,NumOfEvents,Profiler=""):
882  if Profiler == "":
883  INDEX.write("<li><a href=\"%s\">%s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,step,NumOfEvents))
884  else:
885  #FIXME: need to fix this: why do we have the number of Memcheck events hardcoded?
886  if CurrentProfile == "memcheck_valgrind":#FIXME:quick and dirty hack to report we have changed memcheck to 5 events now...
887  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,"5"))
888  else:
889  INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,NumOfEvents))
890  def IgProfDumpsTable(INDEX,ProfileLinks,step):
891  #See if the end of job profiles IgProfMemTotal.res or IgProfMemLive.res are in the list as they should:
892  EndOfJobProfileLink=filter(lambda x: "IgProfMemTotal.res" in x or "IgProfMemLive.res" in x, ProfileLinks)[0] #Assume only one of the two is there, as it should.
893  #Remove it from the list so we can order it:
894  ProfileLinks.remove(EndOfJobProfileLink)
895  #Sort the list in numerical order:
896  ProfileLinks.sort(key=lambda x: int(x.split(".")[-2]))
897  #Prepare regexp to find and replace MemTotal with MemLive for .gz links
898  IgProfMemLive_regexp=re.compile("IgProfMemLive")
899  if IgProfMemLive_regexp.search(EndOfJobProfileLink):
900  MemProfile="IgProf MEM LIVE"
901  else:
902  MemProfile="IgProf MEM TOTAL"
903  #Prepare the table header:
904  INDEX.write("<li>%s"%MemProfile)
905  INDEX.write("<table><tr><td>Profile after event</td><td>Total Memory Size (bytes)</td><td>Total Calls (number)</td><td>Link to gzipped IgProf profile</td></tr>")
906  for link in ProfileLinks:
907  #Build and check the link to the .gz profile that is always called IgProfMemTotal also for MemLive:
908  gzProfile=os.path.join(link.split("/")[-3],link.split("/")[-1])[:-3]+"gz"
909  if IgProfMemLive_regexp.search(gzProfile):
910  gzProfile=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzProfile)
911  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(link.split(".")[-2],open(link,"r").readlines()[6].split()[1],open(link,"r").readlines()[6].split()[2],gzProfile,os.path.basename(gzProfile)))
912  #Put in the end of job one by hand:
913  gzEndOfJobProfileLink=os.path.join(EndOfJobProfileLink.split("/")[-3],EndOfJobProfileLink.split("/")[-1])[:-3]+"gz"
914  if IgProfMemLive_regexp.search(gzEndOfJobProfileLink):
915  gzEndOfJobProfileLink=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzEndOfJobProfileLink)
916  INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%("End of job",open(EndOfJobProfileLink,"r").readlines()[6].split()[1],open(EndOfJobProfileLink,"r").readlines()[6].split()[2],gzEndOfJobProfileLink,os.path.basename(gzEndOfJobProfileLink)))
917  #Closing the table and the list item tags
918  INDEX.write("</table>")
919  INDEX.write("</li>")
920  #FIXME:
921  #These numbers are used in the index.html they are not automatically matched to the actual
922  #ones (one should automate this, by looking into the cmsCreateSimPerfTestPyRelVal.log logfile)
923 
924  NumOfEvents={
925  DirName[0] : TimeSizeNumOfEvents,
926  DirName[1] : IgProfNumOfEvents,
927  DirName[2] : IgProfNumOfEvents,
928  DirName[3] : IgProfNumOfEvents,
929  DirName[4] : CallgrindNumOfEvents,
930  DirName[5] : MemcheckNumOfEvents,
931  DirName[6] : TimeSizeNumOfEvents,
932  DirName[7] : IgProfNumOfEvents,
933  DirName[8] : IgProfNumOfEvents,
934  DirName[9] : IgProfNumOfEvents,
935  DirName[10] : CallgrindNumOfEvents,
936  DirName[11] : MemcheckNumOfEvents
937  }
938 
939  Profile=(
940  #These need to match the profile directory names ending within the candle directories
941  "TimingReport",
942  "TimeReport",
943  "SimpleMemReport",
944  "EdmSize",
945  "IgProfperf",
946  "IgProfMemTotal",
947  "IgProfMemLive",
948  "IgProfMemAnalyse",
949  "valgrind",
950  "memcheck_valgrind"
951  )
952  IgProfMemAnalyseOut=( #This is the special case of IgProfMemAnalyse
953  "doBeginJob_output.html",
954  "doEvent_output.html",
955  "mem_live.html",
956  "mem_total.html"
957  )
958  memcheck_valgrindOut=( #This is the special case of Valgrind MemCheck (published via Giovanni's script)
959  "beginjob.html",
960  "edproduce.html",
961  "esproduce.html"
962  )
963  OutputHtml={ #These are the filenames to be linked in the candle html page for each profile
964  Profile[0] : "*TimingReport.html", #The wildcard spares the need to know the candle name
965  Profile[1] : "TimeReport.html", #This is always the same (for all candles)
966  Profile[2] : "*.html", #This is supposed to be *SimpleMemoryCheck.html, but there is a bug in cmsRelvalreport.py and it is called TimingReport.html!
967  Profile[3] : "objects_pp.html", #This is one of 4 objects_*.html files, it's indifferent which one to pick, just need consistency
968  Profile[4] : "overall.html", #This is always the same (for all candles)
969  Profile[5] : "overall.html", #This is always the same (for all candles)
970  Profile[6] : "overall.html", #This is always the same (for all candles)
971  Profile[7] : "doBeginJob_output.html", #This is complicated... there are 3 html to link... (see IgProf MemAnalyse below)
972  Profile[8] : "overall.html", #This is always the same (for all candles)
973  Profile[9] : "beginjob.html" #This is complicated there are 3 html to link here too... (see Valgrind MemCheck below)
974  }
975 
976 
977  candnreg = re.compile("CandleName")
978  candhreg = re.compile("CandlesHere")
979  try:
980  CAND = open(candlHTML,"w")
981  for line in open(tmplfile):
982  if candhreg.search(line):
983  CAND.write('<div id="content">')
984  CAND.write("<h2>")
985  CAND.write(CurrentCandle)
986  CAND.write("</h2>\n")
987 
988  if _verbose:
989  print "Producing candles html: ", CurrentCandle
990 
991  for CurDir in DirName:
992 
993  LocalPath = os.path.join(repdir,"%s_%s" % (CurrentCandle,CurDir))
994  LocalDirname = os.path.basename(LocalPath)
995 
996  if not prevrev == "":
997 
998  profs = []
999  #FIXME!
1000  #Check what this was used for.. it now has different DirName list since we added IgProf_Perf and IgProf_Mem on top of IgProf alone (still there)
1001  if CurDir == DirName[0] or CurDir == DirName[6]: #TimeSize tests (with and without PU)
1002  profs = Profile[0:4]
1003  elif CurDir == DirName[1] or CurDir == DirName[7]: #IgProf tests (with and without PU)
1004  profs = Profile[4:8]
1005  elif CurDir == DirName[2] or CurDir == DirName[8]: #IgProf Perf tests (with and without PU)
1006  profs =[Profile[4]] #Keeping format to a list...
1007  elif CurDir == DirName[3] or CurDir == DirName[9]: #IgProf Mem tests (with and without PU)
1008  profs =Profile[5:8] #Keeping format to a list...
1009  elif CurDir == DirName[4] or CurDir == DirName[10]: #Callgrind tests (with and without PU)
1010  profs = [Profile[8]] #Keeping format to a list...
1011  elif CurDir == DirName[5] or CurDir == DirName[11]: #Memcheck tests (with and without PU)
1012  profs = [Profile[9]] #Keeping format to a list...
1013  #This could be optimized, but for now just comment the code:
1014  #This for cycle takes care of the case in which there are regression reports to link to the html:
1015  for prof in profs:
1016  if _verbose:
1017  print "Scanning for profile information for: ", prof
1018 
1019  printed = False
1020  fullprof = (CurrentCandle,prof)
1021  outd = ""
1022  nologext = ""
1023  if prof == "TimingReport":
1024  timeReports = glob.glob(os.path.join(LocalPath,"%s_*_%s.log" % (CandFname[CurrentCandle],prof)))
1025  if len(timeReports) > 0:
1026  if not printed:
1027  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1028  printed = True
1029  for log in timeReports:
1030  reportName = os.path.basename(log)
1031  (nologext, outd) = getOutputNames(LocalDirname,reportName)
1032  CAND.write("<h4>%s</h4>\n" % reportName)
1033  htmNames = ["changes.png"]
1034  otherNames = ["graphs.png" , "histos.png"]
1035  regressHTML= "%s-regress.html" % nologext
1036  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,outd,x),otherNames)))
1037  html = ""
1038  if pathsExist:
1039  html = "<p>Performance graphs and histograms superimposed for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1040  else:
1041  html = "<p>No change in performance graph available</p>\n"
1042  regressHTML="%s/%s" % (WebArea,regressHTML)
1043 
1044  for x in htmNames:
1045  abspath = os.path.join(repdir,outd,x)
1046  if os.path.exists(abspath):
1047  html += "<p><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></p>\n" % (outd,x,outd,x)
1048  else:
1049  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1050 
1051  createRegressHTML(regressHTML,repdir,outd,CurrentCandle,otherNames)
1052  CAND.write(html)
1053  CAND.write("\n</tr></table>")
1054 
1055 
1056  elif prof == "SimpleMemReport":
1057  simMemReports = glob.glob(os.path.join(LocalPath,"%s_*_%s" % (CandFname[CurrentCandle],prof)))
1058  simMemReports = map(lambda x: (CandFname[CurrentCandle],prof,x), simMemReports )
1059  simMemReports.sort(cmp=dirname_cmp)
1060  simMemReports = map(lambda x: x[2], simMemReports )
1061  if len(simMemReports) > 0:
1062  if not printed:
1063  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1064  printed = True
1065  for adir in simMemReports:
1066  reportName = os.path.basename(adir)
1067  CAND.write("<h4>%s</h4>\n" % reportName)
1068  htmNames = ["vsize_change.png", "rss_change.png"]
1069  otherNames = ["vsize_graphs.png","rss_graphs.png"]
1070  nologext = reportName
1071  outd = reportName
1072  regressHTML= "%s-regress.html" % nologext
1073  pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,LocalDirname,outd,x),otherNames)))
1074  html = ""
1075  if pathsExist:
1076  html = "<p>Superimposed memory performance graphs for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML)
1077  else:
1078  html = "<p>No change in performance graph available</p>\n"
1079  regressHTML="%s/%s" % (WebArea,regressHTML)
1080 
1081  for x in htmNames:
1082  abspath = os.path.join(repdir,LocalDirname,outd,x)
1083  if os.path.exists(abspath):
1084  html += "<p><a href=\"./%s/%s/%s\"><img src=\"./%s/%s/%s\" /></a></p>\n" % (LocalDirname,outd,x,LocalDirname,outd,x)
1085  else:
1086  html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath)
1087 
1088  createRegressHTML(regressHTML,repdir,"%s/%s" % (LocalDirname,outd),CurrentCandle,otherNames)
1089  CAND.write(html)
1090  CAND.write("\n</tr></table>")
1091 
1092  elif prof == "EdmSize" or prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof== "IgProfperf" or prof == "Callgrind":
1093  regresPath = os.path.join(LocalPath,"%s_*_%s_regression" % (CandFname[CurrentCandle],prof))
1094  regresses = glob.glob(regresPath)
1095  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s_regression" % (CandFname[CurrentCandle],prof))
1096  if len(regresses) > 0:
1097  if not printed:
1098  CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis"))
1099  printed = True
1100  regresses = map(lambda x: (CandFname[CurrentCandle],prof,x),regresses)
1101  regresses.sort(cmp=reg_dirname_cmp)
1102  regresses = map(lambda x: x[2],regresses)
1103  for rep in regresses:
1104  base = os.path.basename(rep)
1105  found = stepreg.search(base)
1106  step = "Unknown-step"
1107  if found:
1108  step = found.groups()[0]
1109  htmlpage = ""
1110  if prof == "IgProfMemLive" or prof == "IgProfMemTotal" or prof== "IgProfperf" or prof == "Callgrind":
1111  htmlpage = "overall.html"
1112  else:
1113  htmlpage = "objects_pp.html"
1114  CAND.write("<a href=\"%s/%s/%s\">%s %s regression report</a><br/>\n" % (LocalDirname,base,htmlpage,prof,step))
1115  #Here we go back to the "non-regression" reports listing
1116  CandleLogFiles = []
1117  if os.path.exists(LocalPath):
1118  thedir = os.listdir(LocalPath)
1119  CandleLogFiles = filter(lambda x: (x.endswith(".log") or x.endswith("EdmSize")) and not os.path.isdir(x) and os.path.exists(x), map(lambda x: os.path.abspath(os.path.join(LocalPath,x)),thedir))
1120 
1121  if (len(CandleLogFiles)>0):
1122 
1123  syscp(CandleLogFiles,WebArea + "/")
1124  base = os.path.basename(LocalPath)
1125  lfileshtml = ""
1126 
1127  for cand in CandleLogFiles:
1128  cand = os.path.basename(cand)
1129  if _verbose:
1130  print "Found %s in %s\n" % (cand,LocalPath)
1131 
1132  if not "EdmSize" in cand:
1133  lfileshtml += "<a href=\"./%s/%s\">%s </a><br/>" % (base,cand,cand)
1134 
1135  CAND.write("<p><strong>Logfiles for %s</strong></p>\n" % CurDir)
1136  CAND.write(lfileshtml)
1137 
1138  PrintedOnce = False
1139  for CurrentProfile in Profile:
1140  #print Steps
1141  for step in Steps: #Using Steps here that is Step+ProductionSteps!
1142  #print step
1143  #print "DEBUG:%s,%s,%s,%s,%s,%s"%(repdir,CurrentCandle,CurDir,step,CurrentProfile,OutputHtml[CurrentProfile])
1144  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1145  CurDir,
1146  step,
1147  CurrentProfile,
1148  OutputHtml[CurrentProfile])
1149  #if ProfileReportLink:
1150  # print ProfileReportLink
1151  isProfinLink = False
1152  if len (ProfileReportLink) > 0:
1153  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1154 
1155  if isProfinLink:
1156  #It could also not be there
1157 
1158  if (PrintedOnce==False):
1159  #Making sure it's printed only once per directory (TimeSize, IgProf, Valgrind) each can have multiple profiles
1160 
1161  #This is the "title" of a series of profiles, (TimeSize, IgProf, Valgrind)
1162  CAND.write("<p><strong>%s</strong></p>\n" % CurDir)
1163  CAND.write("<ul>\n")
1164  PrintedOnce=True
1165 
1166  #Special cases first (IgProf MemAnalyse and Valgrind MemCheck)
1167  #Add among the special cases any IgProfMem (5,6,7) since now we added the dumping:
1168  if (CurrentProfile == Profile[5] or CurrentProfile == Profile[6]):
1169  for prolink in ProfileReportLink:
1170  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1171  for igprofmem in ["*IgProfMemTotal*.res","*IgProfMemLive*.res"]:
1172  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1173  CurDir,
1174  step,
1175  CurrentProfile,
1176  igprofmem)
1177  isProfinLink = False
1178  if len (ProfileReportLink) > 0:
1179  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1180  if isProfinLink :#It could also not be there
1181  #for prolink in ProfileReportLink:
1182  IgProfDumpsTable(CAND,ProfileReportLink,step)
1183  # _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=os.path.basename(prolink))
1184 
1185  elif (CurrentProfile == Profile[7]):
1186  for igprof in IgProfMemAnalyseOut:
1187  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1188  CurDir,
1189  step,
1190  CurrentProfile,
1191  igprof)
1192  isProfinLink = False
1193  if len (ProfileReportLink) > 0:
1194  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1195  if isProfinLink :#It could also not be there
1196  for prolink in ProfileReportLink:
1197  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=igprof)
1198 
1199 
1200  elif (CurrentProfile == Profile[9]):
1201 
1202  for memprof in memcheck_valgrindOut:
1203  #print memprof
1204  ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle,
1205  CurDir,
1206  step,
1207  CurrentProfile,
1208  memprof
1209  )
1210  isProfinLink = False
1211  if len (ProfileReportLink) > 0:
1212  isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink))
1213  if isProfinLink :#It could also not be there
1214  for prolink in ProfileReportLink:
1215  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=memprof)
1216 
1217  else:
1218  for prolink in ProfileReportLink:
1219  if "regression" not in prolink: #To avoid duplication of links to regression reports!
1220  _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir])
1221  #print "Step is %s, CurrentProfile is %s and ProfileReportLink is %s and prolink is %s"%(step,CurrentProfile,ProfileReportLink,prolink)
1222 
1223 
1224  if PrintedOnce:
1225  CAND.write("</ul>\n")
1226  PrintedOnce=False
1227 
1228  CAND.write("<hr />")
1229  CAND.write("<br />\n")
1230  CAND.write("</div>\n")
1231  elif candnreg.search(line):
1232  CAND.write(CurrentCandle)
1233  else:
1234  CAND.write(line)
1235 
1236  CAND.close()
1237  except IOError as detail:
1238  print "ERROR: Could not write candle html %s because %s" % (os.path.basename(candlHTML),detail)
1239 
cmsPerfPublish.createCandlHTML
def createCandlHTML
Create HTML pages for candles.
Definition: cmsPerfPublish.py:846
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def cmsPerfPublish.createHTMLtab (   INDEX,
  table_dict,
  ordered_keys,
  header,
  caption,
  name,
  mode = 0 
)

Definition at line 1304 of file cmsPerfPublish.py.

References prettySize().

Referenced by createWebReports().

1304 
1305 def createHTMLtab(INDEX,table_dict,ordered_keys,header,caption,name,mode=0):
1306  cols = len(ordered_keys)
1307  totcols = (cols * 3) + 1
1308  innercol = 3
1309  colspan = totcols - 1
1310  labels = []
1311  if mode == 1:
1312  labels = ["fs1","fs2","&#x0394;"]
1313  elif mode == 2 or mode == 3:
1314  colspan = cols
1315  innercol = 1
1316  else:
1317  labels = ["t1" ,"t2" ,"&#x0394;"]
1318 
1319 
1320  INDEX.write("<h3>%s</h3>\n" % header)
1321  INDEX.write("<table>\n")
1322  INDEX.write("<caption>%s</caption>\n" % caption)
1323  INDEX.write("<thead><tr><th></th><th colspan=\"%s\" scope=\"colgroup\">%s</th></tr></thead>" % (colspan,name))
1324  INDEX.write("<tbody>\n")
1325  for key in ordered_keys:
1326  INDEX.write("<tr>")
1327  if key == None:
1328  INDEX.write("<th></th>")
1329  else:
1330  INDEX.write("<td scope=\"row\">")
1331  INDEX.write(key)
1332  INDEX.write("</td>")
1333  for col in table_dict[None]:
1334  if key == None:
1335  INDEX.write("<th colspan=\"%s\" scope=\"col\">" % innercol)
1336  INDEX.write(col)
1337  INDEX.write("</th>")
1338  else:
1339  rowdict = table_dict[key].getRowDict()
1340  if col in rowdict:
1341  if mode == 2:
1342  dat = prettySize(rowdict[col])
1343  INDEX.write("<td>")
1344  INDEX.write("%s" % dat)
1345  INDEX.write("</td>")
1346 
1347  elif mode == 3:
1348  dat = rowdict[col]
1349  INDEX.write("<td>")
1350  INDEX.write("%6.2f" % dat)
1351  INDEX.write("</td>")
1352  else:
1353  (data1, data2) = rowdict[col]
1354  diff = data2 - data1
1355 
1356  if mode == 1:
1357  diff = prettySize(diff)
1358  data1 = prettySize(data1)
1359  data2 = prettySize(data2)
1360 
1361  seq = [ data1, data2, diff ]
1362  for dat in seq:
1363  INDEX.write("<td id=\"data\">")
1364  if mode == 1:
1365  INDEX.write("%s" % dat) # %s if
1366  else:
1367  INDEX.write("%6.2f" % dat) # %s if
1368 
1369  INDEX.write("</td>")
1370  else:
1371  if mode == 2 or mode == 3:
1372  INDEX.write("<td>")
1373  INDEX.write("N/A")
1374  INDEX.write("</td>")
1375  else:
1376  for i in range(3):
1377  INDEX.write("<td>")
1378  INDEX.write("N/A")
1379  INDEX.write("</td>")
1380  INDEX.write("</tr>\n")
1381  # write an additional row if this row is the header row
1382  # we need to describe the sub columns
1383  if not (mode == 2 or mode == 3 ):
1384  if key == None:
1385  INDEX.write("<tr>")
1386  INDEX.write("<th>Candles</th>")
1387  for col in table_dict[None]:
1388  INDEX.write("<th>%s</th>" % labels[0])
1389  INDEX.write("<th>%s</th>" % labels[1])
1390  INDEX.write("<th>%s</th>" % labels[2])
1391  INDEX.write("</tr>\n")
1392  INDEX.write("</tbody></table>\n")
1393 
1394  INDEX.write("<br />")
def cmsPerfPublish.createRegressHTML (   reghtml,
  repdir,
  outd,
  CurrentCandle,
  htmNames 
)

Definition at line 683 of file cmsPerfPublish.py.

Referenced by createCandlHTML().

683 
684 def createRegressHTML(reghtml,repdir,outd,CurrentCandle,htmNames):
685  RegressTmplHTML="%s/doc/regress.html" % (BASE_PERFORMANCE)
686  candnreg = re.compile("CandleName")
687  candhreg = re.compile("CandlesHere")
688  try:
689  REGR = open(reghtml,"w")
690  for line in open(RegressTmplHTML):
691  if candhreg.search(line):
692  html = "<table>"
693 
694  for x in htmNames:
695  abspath = os.path.join(repdir,outd)
696  if os.path.exists(abspath):
697  html += "<tr><td><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></td></tr>\n" % (outd,x,outd,x)
698  else:
699  html += "<tr><td> %s does not exist probably because the log file for the previous release was missing</td></tr>" % (abspath)
700  html += "</table>"
701  REGR.write(html)
702  elif candnreg.search(line):
703  REGR.write(CurrentCandle)
704  else:
705  REGR.write(line)
706  except IOError as detail:
707  print "ERROR: Could not write regression html %s because %s" % (os.path.basename(reghtml),detail)
def cmsPerfPublish.createWebReports (   WebArea,
  repdir,
  ExecutionDate,
  LogFiles,
  cmsScimarkResults,
  date,
  prevrev 
)

Create web report index and create HTML file for each candle.

Definition at line 1448 of file cmsPerfPublish.py.

References createCandlHTML(), createHTMLtab(), populateFromTupleRoot(), and syscp().

Referenced by main().

1448 
1449 def createWebReports(WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev):
1450 
1451  #Some nomenclature
1452 
1453  Candle = Candles #These need to match the directory names in the work area
1454 
1455  CmsDriverCandle = CandFname #{ #These need to match the cmsDriver.py output filenames
1456 
1457 
1458  #Produce a "small" index.html file to navigate the html reports/logs etc
1459  IndexFile="%s/index.html" % WebArea
1460  TemplateHtml="%s/doc/index.html" % BASE_PERFORMANCE
1461 
1462  cmsverreg = re.compile("CMSSW_VERSION")
1463  hostreg = re.compile("HOST")
1464  lpathreg = re.compile("LocalPath")
1465  fsizereg = re.compile("FSizeTable")
1466  cpureg = re.compile("CPUTable")
1467  proddreg = re.compile("ProductionDate")
1468  logfreg = re.compile("LogfileLinks")
1469  dirbreg = re.compile("DirectoryBrowsing")
1470  pubdreg = re.compile("PublicationDate")
1471  candhreg = re.compile("CandlesHere")
1472  #Loop line by line to build our index.html based on the template one
1473  #Copy the perf_style.css file from Validation/Performance/doc
1474 
1475  CandlTmpltHTML="%s/doc/candle.html" % BASE_PERFORMANCE
1476  if _verbose:
1477  print "Copying %s/doc/perf_style.css style file to %s/." % (BASE_PERFORMANCE,WebArea)
1478  print "Template used: %s" % TemplateHtml
1479 
1480  syscp((BASE_PERFORMANCE + "/doc/perf_style.css"),WebArea + "/.")
1481  pureg = re.compile("(.*)_PILEUP")
1482  try:
1483  INDEX = open(IndexFile,"w")
1484  for NewFileLine in open(TemplateHtml) :
1485  if cmsverreg.search(NewFileLine):
1486  if prevrev == "":
1487  INDEX.write("Performance Reports for %s\n" % CMSSW_VERSION)
1488  else:
1489  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1490  globs = glob.glob(globpath)
1491  if len(globs) < 1:
1492  pass
1493  else:
1494  latestreg = re.compile("REGRESSION.%s.vs.(.*)" % prevrev)
1495  found = latestreg.search(os.path.basename(globs[0]))
1496  if found:
1497  latestrel = found.groups()[0]
1498  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,latestrel))
1499  else:
1500  INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,CMSSW_VERSION))
1501  elif hostreg.search(NewFileLine):
1502  INDEX.write(HOST + "\n")
1503  #File Size Summary Table
1504  elif fsizereg.search(NewFileLine):
1505  #Case of NO-REGRESSION:
1506  if prevrev == "":
1507  fsize_tab = Table()
1508 
1509  for cand in Candles:
1510  fname = CandFname[cand]
1511  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1512  rootfiles = glob.glob(globpath)
1513  rootfiles = map(lambda x: (fname,x), rootfiles)
1514  rootfiles.sort(cmp=rootfile_cmp)
1515  rootfiles = map(lambda x: x[1], rootfiles)
1516  stepreg = re.compile("%s_(.*).root" % fname)
1517  createNewRow = True
1518  curRow = None
1519  createPURow = True
1520  puRow = None
1521  for rootf in rootfiles:
1522  base = os.path.basename(rootf)
1523  found = stepreg.search(base)
1524  step = "Unknown-step"
1525  if found:
1526  step = found.groups()[0]
1527  realstep = "Unknown-step"
1528  if "PILEUP" in step:
1529  found = pureg.search(step)
1530  if found:
1531  realstep = found.groups()[0]
1532  if createPURow:
1533  createPURow = False
1534  puRow = Row(fsize_tab)
1535  try:
1536  statinfo = os.stat(rootf)
1537  fsize = statinfo.st_size
1538  if createNewRow:
1539  createNewRow = False
1540  curRow = fsize_tab.newRow(cand)
1541 
1542  if "PILEUP" in step:
1543  puRow.addEntry(realstep,fsize)
1544  else:
1545  if createNewRow:
1546  createNewRow = False
1547  curRow = fsize_tab.newRow(cand)
1548 
1549  curRow.addEntry(step,fsize)
1550  except IOError as detail:
1551  print detail
1552  except OSError as detail:
1553  print detail
1554  if puRow == None:
1555  pass
1556  else:
1557  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1558  (ordered_keys,table_dict) = fsize_tab.getTable(1)
1559  cols = len(ordered_keys)
1560 
1561  if len(table_dict) > 1 and cols > 0:
1562  createHTMLtab(INDEX,table_dict,ordered_keys,
1563  "Release ROOT file sizes",
1564  "Table showing current release ROOT filesizes in (k/M/G) bytes.",
1565  "Filesizes",2)
1566  #Case of REGRESSION vs previous release:
1567  else:
1568  try:
1569  globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev))
1570  globs = glob.glob(globpath)
1571  if len(globs) < 1:
1572  raise IOError(2,globpath,"File does not exist","File does not exist")
1573  idfile = open(globs[0])
1574  oldpath = ""
1575  for line in idfile:
1576  oldpath = line
1577  oldpath = oldpath.strip()
1578  #print "##########TABLE DEBUG :oldpath is %s"%oldpath
1579  fsize_tab = Table()
1580 
1581  for cand in Candles:
1582  fname = CandFname[cand]
1583  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname)
1584  rootfiles = glob.glob(globpath)
1585  rootfiles = map(lambda x: (fname,x), rootfiles)
1586  rootfiles.sort(cmp=rootfile_cmp)
1587  rootfiles = map(lambda x: x[1], rootfiles)
1588  stepreg = re.compile("%s_(.*).root" % fname)
1589  createNewRow = True
1590  curRow = None
1591  createPURow = True
1592  puRow = None
1593  for rootf in rootfiles:
1594  base = os.path.basename(rootf)
1595  found = stepreg.search(base)
1596  step = "Unknown-step"
1597  if found:
1598  step = found.groups()[0]
1599 
1600  realstep = "Unknown-step"
1601  if "PILEUP" in step:
1602  found = pureg.search(step)
1603  if found:
1604  realstep = found.groups()[0]
1605  if createPURow:
1606  createPURow = False
1607  puRow = Row(fsize_tab)
1608  try:
1609  statinfo = os.stat(rootf)
1610  fsize2 = statinfo.st_size
1611  oldfile = os.path.join(oldpath,"%s_TimeSize" % cand,base)
1612  if "PILEUP" in step:
1613  oldfile = os.path.join(oldpath,"%s_PU_TimeSize" % cand,base)
1614  fsize1 = 0
1615 
1616  if os.path.exists(oldfile):
1617  statinfo = os.stat(oldfile)
1618  fsize1 = statinfo.st_size
1619  else:
1620  print "######DID NOT FIND Previous file (needed for the filesize table): %s"%oldfile
1621 
1622  if createNewRow:
1623  createNewRow = False
1624  curRow = fsize_tab.newRow(cand)
1625 
1626  data_tuple = (fsize1,fsize2)
1627  if "PILEUP" in step:
1628  puRow.addEntry(realstep,data_tuple)
1629  else:
1630  if createNewRow:
1631  createNewRow = False
1632  curRow = fsize_tab.newRow(cand)
1633 
1634  curRow.addEntry(step,data_tuple)
1635  except IOError as detail:
1636  print detail
1637  except OSError as detail:
1638  print detail
1639  if puRow == None:
1640  pass
1641  else:
1642  fsize_tab.addRow(puRow,"%s PILEUP" %cand)
1643 
1644  (ordered_keys,table_dict) = fsize_tab.getTable()
1645  cols = len(ordered_keys)
1646 
1647  if len(table_dict) > 1 and cols > 0:
1648  createHTMLtab(INDEX,table_dict,ordered_keys,
1649  "Release ROOT file sizes",
1650  "Table showing previous release ROOT filesizes, fs1, latest sizes, fs2, and the difference between them &#x0394; in (k/M/G) bytes.",
1651  "Filesizes",1)
1652  except IOError as detail:
1653  print detail
1654  except OSError as detail:
1655  print detail
1656  #CPU Time Summary Table
1657  elif cpureg.search(NewFileLine):
1658  #Case of NO REGRESSION
1659  if prevrev == "":
1660  time_tab = Table()
1661 
1662  for cand in Candles:
1663  fname = CandFname[cand]
1664  globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*_TimingReport.log" % fname)
1665  logfiles = glob.glob(globpath)
1666  logfiles = map(lambda x: (fname,x), logfiles)
1667  logfiles.sort(cmp=logrep_cmp)
1668  logfiles = map(lambda x: x[1], logfiles)
1669 
1670  stepreg = re.compile("%s_(.*)_TimingReport.log" % fname)
1671  createNewRow = True
1672  curRow = None
1673  createPURow = True
1674  puRow = None
1675  for log in logfiles:
1676  base = os.path.basename(log)
1677  found = stepreg.search(base)
1678  step = "Unknown-step"
1679  if found:
1680  step = found.groups()[0]
1681 
1682  realstep = "Unknown-step"
1683  if "PILEUP" in step:
1684  found = pureg.search(step)
1685  if found:
1686  realstep = found.groups()[0]
1687  if createPURow:
1688  createPURow = False
1689  puRow = Row(time_tab)
1690 
1691  data = cpr.getTimingLogData(log)
1692  mean = 0
1693  i = 0
1694  for evtnum, time in data:
1695  mean += time
1696  i += 1
1697  try:
1698  mean = mean / float(i)
1699  except ZeroDivisionError as detail:
1700  print "WARNING: Could not calculate mean CPU time from log because no events could be parsed", log
1701 
1702  if "PILEUP" in step:
1703  puRow.addEntry(realstep,mean)
1704  else:
1705  if createNewRow:
1706  createNewRow = False
1707  curRow = time_tab.newRow(cand)
1708 
1709  curRow.addEntry(step,mean)
1710  if puRow == None:
1711  pass
1712  else:
1713  time_tab.addRow(puRow,"%s PILEUP" %cand)
1714 
1715  (ordered_keys,table_dict) = time_tab.getTable(1)
1716  cols = len(ordered_keys)
1717 
1718  if len(table_dict) > 1 and cols > 0:
1719  createHTMLtab(INDEX,table_dict,ordered_keys,
1720  "Release CPU times",
1721  "Table showing current release CPU times in secs.",
1722  "CPU Times (s)",3)
1723  #Case of REGRESSION (CPU Time Summary Table)
1724  else:
1725 
1726 
1727  ####
1728  #
1729  # Create the table data structure
1730  #
1731  cpu_time_tab = populateFromTupleRoot("cpu_time_tuple",repdir,"timing-regress.root",pureg)
1732 
1733 
1734  ###########
1735  #
1736  # Create HTML table from table data structure
1737  #
1738 
1739  (ordered_keys,table_dict) = cpu_time_tab.getTable()
1740 
1741  cols = len(ordered_keys)
1742  if len(table_dict) > 1 and cols > 0:
1743  createHTMLtab(INDEX,table_dict,ordered_keys,
1744  "Release CPU times",
1745  "Table showing previous release CPU times, t1, latest times, t2, and the difference between them &#x0394; in secs.",
1746  "CPU Times (s)")
1747 
1748 
1749 
1750  elif lpathreg.search(NewFileLine):
1751  INDEX.write(repdir + "\n")
1752  elif proddreg.search(NewFileLine):
1753  INDEX.write(ExecutionDate + "\n")
1754  elif logfreg.search(NewFileLine):
1755  INDEX.write("<br />\n")
1756  for log in LogFiles:
1757  log = os.path.basename(log)
1758  if _verbose:
1759  print "linking log file %s" % log
1760  INDEX.write("<a href=\"./%s\"> %s </a>" % (log,log))
1761  INDEX.write("<br />\n")
1762  #Add the cmsScimark results here:
1763  INDEX.write("Results for cmsScimark2 benchmark (running on the other cores) available at:\n")
1764  for cmssci in cmsScimarkResults:
1765  scimarkdirs=cmssci.split("/")
1766  localdir=scimarkdirs[-2]
1767  #print localdir
1768  cmssci = os.path.basename(cmssci)
1769  relativelink=os.path.join(localdir,cmssci)
1770  #print relativelink
1771  INDEX.write("<a href=\"%s\"> %s </a>" % (relativelink,cmssci))
1772  INDEX.write("<br />\n")
1773 
1774 
1775  elif dirbreg.search(NewFileLine):
1776  #Create a subdirectory DirectoryBrowsing to circumvent the fact the dir is not browsable if there is an index.html in it.
1777  #Bug! This does not work since it points to index.html automatically!
1778  #os.symlink("./","%s/DirectoryBrowsing" % WebArea)
1779  #Actually all the following is done later...
1780  #Create a physical directory first
1781  #os.mkdir("%s/DirectoryBrowsing" % WebArea)
1782  #Then will populate it with symbolic links(once they have been copied there!) from all WebArea files, except index.html, see down towards the end!
1783  INDEX.write("Click <a href=\"./DirectoryBrowsing/\">here</a> to browse the directory containing all results (except the root files)\n")
1784 
1785  elif pubdreg.search(NewFileLine):
1786  INDEX.write(date + "\n")
1787  elif candhreg.search(NewFileLine):
1788  for acandle in Candle:
1789  globpath = os.path.join(repdir,"%s_*" % acandle)
1790  globs = glob.glob(globpath)
1791  if len(globs) > 0:
1792  candlHTML = "%s.html" % acandle
1793  INDEX.write("<a href=\"./%s\"> %s </a>" % (candlHTML,acandle))
1794  INDEX.write("<br />\n")
1795 
1796  candlHTML=os.path.join(WebArea,candlHTML)
1797  createCandlHTML(CandlTmpltHTML,candlHTML,acandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
1798  else:
1799  INDEX.write(NewFileLine)
1800 
1801  #End of while loop on template html file
1802  INDEX.close()
1803  except IOError as detail:
1804  print "Error: Could not create index Html file for some reason, check position. Details : %s" % detail
cmsPerfPublish.createCandlHTML
def createCandlHTML
Create HTML pages for candles.
Definition: cmsPerfPublish.py:846
cmsPerfPublish.createWebReports
def createWebReports
Create web report index and create HTML file for each candle.
Definition: cmsPerfPublish.py:1448
def cmsPerfPublish.delTmpDir ( )

Delete tmp dir if we used it.

Definition at line 1848 of file cmsPerfPublish.py.

Referenced by main().

1848 
1849 def delTmpDir():
1850  if os.path.exists(TMP_DIR) and IS_TMP:
1851  os.system("rm -Rf " + TMP_DIR)
cmsPerfPublish.delTmpDir
def delTmpDir
Delete tmp dir if we used it.
Definition: cmsPerfPublish.py:1848
def cmsPerfPublish.dirname_cmp (   x,
  y 
)

Definition at line 728 of file cmsPerfPublish.py.

References step_cmp().

728 
729 def dirname_cmp(x,y):
730  (candle,prof,x) = x
731  (candle,prof,y) = y
732  x = os.path.basename(x)
733  y = os.path.basename(y)
734  stepreg = re.compile("%s_(..*)_%s" % (candle,prof))
735  if stepreg.search(x):
736  x = stepreg.search(x).groups()[0]
737  if stepreg.search(y):
738  y = stepreg.search(y).groups()[0]
739  return step_cmp(x,y)
def cmsPerfPublish.docopy (   src,
  dest 
)

Definition at line 1898 of file cmsPerfPublish.py.

Referenced by syscp().

1898 
1899 def docopy(src,dest):
1900  try:
1901  copy2(src,dest)
1902  except OSError as detail:
1903  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1904  except IOError as detail:
1905  print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail)
1906  else:
1907  if _verbose:
1908  print "Copied %s to %s" % (src,dest)
def cmsPerfPublish.fail (   errstr = "")

Definition at line 65 of file cmsPerfPublish.py.

Referenced by CalibrationScanTask.checkAndSave(), CalibrationTask.checkAndSave(), HLTLevel1GTSeed.convertStringToBitNumber(), EgHLTOfflineClient.createLooseTightTrigEff(), EgHLTOfflineClient.createTrigTagProbeEffHistsNewAlgo(), TagProbeFitter.doCntEfficiency(), editESInputTag(), editInputTag(), editNumericParameter(), editVInputTag(), SurveyPxbImage.fill(), LogErrorEventFilter.filter(), ZeeCandidateFilter.filter(), dqmTnP::GaussianPlusLinearFitter.fit(), dqmTnP::VoigtianPlusExponentialFitter.fit(), lumi.from_string(), HcalDbOmds.from_string(), get_environ(), getStageRepDirs(), L1GtUtils::LogicalExpressionL1Results.initialize(), SurveyResidual.inverseCovariance(), operator!=(), EcalUncalibRecHitFixedAlphaBetaAlgo< C >.PerformAnalyticFit(), ContentsXRange.runTest(), ContentsYRange.runTest(), DeadChannel.runTest(), NoisyChannel.runTest(), ContentsWithinExpected.runTest(), SimpleCosmicBONSeeder.seeds(), stringToNumber(), cscdqm.stringToNumber(), TagProbeFitTreeAnalyzer.TagProbeFitTreeAnalyzer(), L1GtTriggerMenuXmlParser.workAlgorithm(), l1t::TriggerMenuXmlParser.workAlgorithm(), and L1GtTriggerMenuXmlParser.workTechTrigger().

65 
66 def fail(errstr=""):
67  print errstr
68  delTmpDir()
69  sys.exit()
cmsPerfPublish.delTmpDir
def delTmpDir
Delete tmp dir if we used it.
Definition: cmsPerfPublish.py:1848
def cmsPerfPublish.get_environ ( )

Get require environment variables.

Definition at line 302 of file cmsPerfPublish.py.

References fail(), and getcmdBasic().

Referenced by main().

302 
303 def get_environ():
304  global CMSSW_VERSION, CMSSW_RELEASE_BASE, CMSSW_BASE, HOST, USER, BASE_PERFORMANCE, CMSSW_WORK
305  global DEF_RELVAL, DEF_SIMUL
306 
307  try:
308  CMSSW_VERSION=os.environ['CMSSW_VERSION']
309  CMSSW_RELEASE_BASE=os.environ['CMSSW_RELEASE_BASE']
310  CMSSW_BASE=os.environ['CMSSW_BASE']
311  HOST=os.environ['HOST']
312  USER=os.environ['USER']
313  CMSSW_WORK = os.path.join(CMSSW_BASE,"work/Results")
314  except KeyError as detail:
315  fail("ERROR: Could not retrieve some necessary environment variables. Have you ran scramv1 runtime -csh yet?. Details: %s" % detail)
316 
317  LocalPath=getcmdBasic("pwd")
318  ShowTagsResult=getcmdBasic("showtags -r")
319 
320  #Adding a check for a local version of the packages
321  PerformancePkg="%s/src/Validation/Performance" % CMSSW_BASE
322  if (os.path.exists(PerformancePkg)):
323  BASE_PERFORMANCE=PerformancePkg
324  print "**Using LOCAL version of Validation/Performance instead of the RELEASE version**"
325  else:
326  BASE_PERFORMANCE="%s/src/Validation/Performance" % CMSSW_RELEASE_BASE
327 
328  return (LocalPath,ShowTagsResult)
cmsPerfPublish.get_environ
def get_environ
Get require environment variables.
Definition: cmsPerfPublish.py:302
def cmsPerfPublish.getArchVersionFromLog (   logfile) 
Another very fragile function to get the architecture and the CMSSW version parsing the logfile...

Definition at line 476 of file cmsPerfPublish.py.

References reco.return().

Referenced by main().

476 
477 def getArchVersionFromLog(logfile):
478  '''Another very fragile function to get the architecture and the CMSSW version parsing the logfile...'''
479  log=open(logfile,"r")
480  arch=re.compile("^Current Architecture is")
481  version=re.compile("^Current CMSSW version is")
482  CMSSW_arch="UNKNOWN_ARCH"
483  CMSSW_version="UNKNOWN_VERSION"
484  for line in log:
485  if arch.search(line):
486  CMSSW_arch=line.split()[3]
487  if version.search(line):
488  CMSSW_version=line.split()[4]
489  return(CMSSW_arch,CMSSW_version)
490 
reco::return
return(e1-e2)*(e1-e2)+dp *dp
def cmsPerfPublish.getcmd (   command)

Definition at line 84 of file cmsPerfPublish.py.

84 
85 def getcmd(command):
86  if _debug > 2:
87  print command
88  #Obsolete popen4-> subprocess.Popen
89  #return os.popen4(command)[1].read().strip()
90  return subprocess.Popen(command,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
def cmsPerfPublish.getcmdBasic (   cmd)

Definition at line 79 of file cmsPerfPublish.py.

Referenced by get_environ().

79 
80 def getcmdBasic(cmd):
81  #Obsolete popen4-> subprocess.Popen
82  #return os.popen4(cmd)[1].read().strip()
83  return subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().strip()
def cmsPerfPublish.getDate ( )

Definition at line 76 of file cmsPerfPublish.py.

Referenced by scanReportArea().

76 
77 def getDate():
78  return time.ctime()
def cmsPerfPublish.getDirnameDirs (   repdir,
  WebArea 
)

Grab dirs that end in strings defined in DirName.

Definition at line 1809 of file cmsPerfPublish.py.

References alcazmumu_cfi.filter, and syscp().

Referenced by main().

1809 
1810 def getDirnameDirs(repdir,WebArea):
1811  Dir = os.listdir(repdir)
1812  def _containsDirName(elem):
1813  return reduce(lambda x,y: x or y,map(lambda x: x in elem, DirName))
1814  def _print4Lambda(elem,WebArea):
1815  if _verbose:
1816  print "Copying %s to %s\n" % (elem,WebArea)
1817 
1818  dirstocp = filter(lambda x: _containsDirName(x),map(lambda x: repdir + x,Dir))
1819  map(lambda x: _print4Lambda(x,WebArea),dirstocp)
1820  syscp(dirstocp,WebArea + "/")
1821  os.mkdir("%s/DirectoryBrowsing" % WebArea)
1822  for file in os.listdir(WebArea):
1823  if file != "index.html": #Potential maintenance issue if the index.html changes name to something the server automatically displays when pointing to the directory...
1824  #Use relative path ".." instead of WebArea to avoid problems when copying stuff to a remote server!
1825  os.symlink("%s/%s"%("..",file),"%s/DirectoryBrowsing/%s" % (WebArea,file))
cmsPerfPublish.getDirnameDirs
def getDirnameDirs
Grab dirs that end in strings defined in DirName.
Definition: cmsPerfPublish.py:1809
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
def cmsPerfPublish.getNumOfEventsFromLog (   logfile) 
A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!

Definition at line 451 of file cmsPerfPublish.py.

Referenced by main().

451 
452 def getNumOfEventsFromLog(logfile):
453  '''A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!'''
454  log=open(logfile,"r")
455  TimeSizeEvents=0
456  IgProfEvents=0
457  CallgrindEvents=0
458  MemcheckEvents=0
459  for line in log:
460  #FIXME:
461  #For robustness could read this from the Launching the X tests (.....) with N events each and keep that format decent for parsing.
462  #One more place where XML would seem a better choice to extract information (Timing of the performance suite in general is also, but publishing and harvesting some other info as well).
463  if 'TimeSizeEvents' in line and not TimeSizeEvents:
464  lineitems=line.split()
465  TimeSizeEvents=lineitems[lineitems.index('TimeSizeEvents')+1]
466  if 'IgProfEvents' in line and not IgProfEvents:
467  lineitems=line.split()
468  IgProfEvents=lineitems[lineitems.index('IgProfEvents')+1]
469  if 'CallgrindEvents' in line and not CallgrindEvents:
470  lineitems=line.split()
471  CallgrindEvents=lineitems[lineitems.index('CallgrindEvents')+1]
472  if 'MemcheckEvents' in line and not MemcheckEvents:
473  lineitems=line.split()
474  MemcheckEvents=lineitems[lineitems.index('MemcheckEvents')+1]
475  return (TimeSizeEvents,IgProfEvents,CallgrindEvents,MemcheckEvents)
def cmsPerfPublish.getOutputNames (   base,
  reportName 
)

Definition at line 708 of file cmsPerfPublish.py.

Referenced by createCandlHTML().

708 
709 def getOutputNames(base,reportName):
710  logreg = re.compile("(.*)\.log$")
711  matches = logreg.search(reportName)
712  logdir = logreg.sub(matches.groups()[0],reportName)
713  outd = os.path.join(base,logdir)
714  nologext = matches.groups()[0]
715  return (nologext,outd)
def cmsPerfPublish.getRelativeDir (   parent,
  child,
  keepTop = True 
)

Some functions used for copying.

Definition at line 1856 of file cmsPerfPublish.py.

References GetRecoTauVFromDQM_MC_cff.next.

Referenced by copytree4().

1856 
1857 def getRelativeDir(parent,child,keepTop=True):
1858  def _walkpath(path):
1859  dirs = []
1860  while True:
1861  head , tail = os.path.split(path)
1862  if tail == "":
1863  break
1864  dirs.append(tail)
1865  path = head
1866  for i in range(len(dirs)-1,-1,-1):
1867  adir = dirs[i]
1868  yield adir
1869  return
1870  pwalk = _walkpath(parent)
1871  n = 0
1872  try:
1873  while True:
1874  next(pwalk)
1875  n += 1
1876  except StopIteration:
1877  pass
1878 
1879  if keepTop:
1880  n = n - 1
1881 
1882  cwalk = _walkpath(child)
1883  try:
1884  #prewalk
1885  for x in range(n):
1886  next(cwalk)
1887  except StopIteration:
1888  print "ERROR: Unable to determine relative dir"
1889  raise ReldirExcept
1890 
1891  relpath = ""
1892  try:
1893  while True:
1894  relpath=os.path.join(relpath,next(cwalk))
1895  except StopIteration:
1896  pass
1897  return relpath
cmsPerfPublish.getRelativeDir
def getRelativeDir
Some functions used for copying.
Definition: cmsPerfPublish.py:1856
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
def cmsPerfPublish.getStageRepDirs (   options,
  args 
)

Determine locations of staging and report dirs.

Definition at line 495 of file cmsPerfPublish.py.

References addtrailingslash(), and fail().

Referenced by main().

495 
496 def getStageRepDirs(options,args):
497  global TMP_DIR, IS_TMP, DEF_LOCAL, CMSSW_VERSION
498  DEF_LOCAL = CMSSW_WORK
499  numofargs = len(args)
500 
501  repdir = os.path.abspath(options.repdir)
502  repdir = addtrailingslash(repdir)
503 
504  if not os.path.exists(repdir):
505  fail("ERROR: The specified report directory %s to retrieve report information from does not exist, exiting" % repdir)
506 
507  previousrev = options.previousrev
508  if previousrev == "":
509  regressfiles = glob.glob("%s/REGRESSION.*.vs.*" % repdir)
510  if not len(regressfiles) == 0:
511  regressID = regressfiles[0]
512  base = os.path.basename(regressID)
513  split = base.split(".")
514  previousrev = split[1]
515  currentrel = split[3]
516  print "Regression Identification file exists, renaming report title for regression report. Old ver: %s" % previousrev
517  else:
518  print "No regression ID file exists and previous release name was not specified. Producing normal report."
519  else:
520  print "Previous release name was specified, renaming report title for regression report. Old ver %s" % previousrev
521 
522  uri = ""
523  defaultlocal = False
524  if options.simulation:
525  uri = DEF_SIMUL
526  elif options.relval:
527  uri = DEF_RELVAL
528  elif numofargs >= 1:
529  uri = args[0] # Add directory CMSSW_VERSION later in temp! Not now, otherwise we get into a mess if this is a remote dir
530  else:
531  defaultlocal = True
532  uri = DEF_LOCAL
533 
534  ####
535  #
536  # Determine if location is remote
537  #
538  # Try not to re-arrange we don't want to assume that default locations are not remote
539  #
540  ####
541 
542  drive = ""
543  path = ""
544  if ":" in uri:
545  drive, path = uri.split(":",1)
546  else:
547  path = uri
548 
549  if drive == "":
550  path = os.path.abspath(path)
551  remote = not drive == ""
552 
553  if remote:
554  unResolved = True
555  try:
556  socket.getaddrinfo(drive,53)
557  unResolved = False
558  except socket.gaierror:
559  unResolved = True
560 
561  # try see if it's an ipaddress
562  if unResolved:
563  try:
564  socket.gethostbyaddr(drive)
565  unResolved = False
566  except socket.gaierror:
567  unResolved = True
568  if unResolved:
569  print "ERROR: Can not determine your hostname or ipv{4,6} address %s" % drive
570  if not (_dryrun or _test):
571  fail("exiting...")
572 
573  if (not remote) and (not options.port == 873) :
574  print "WARNING: Can not use a port if not performing a remote copy, ignoring"
575  port = options.port
576 
577  ###
578  #
579  # Determine Staging Area
580  #
581 
582  StagingArea=""
583  localExists = os.path.exists("%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
584 
585  if remote:
586  #Cannot use this since the /tmp is limited to 2GB on lxbuild machines!
587  #TMP_DIR=tmp.mkdtemp(prefix="/tmp/%s" % PROG_NAME)
588  TMP_DIR=tmp.mkdtemp(prefix="/build/%s" % PROG_NAME)
589  StagingArea = TMP_DIR
590  #Local but dir already exists
591  elif defaultlocal and localExists:
592  TMP_DIR=tmp.mkdtemp(prefix="%s/%s" % (CMSSW_WORK,CMSSW_VERSION))
593  StagingArea = TMP_DIR
594  print "WARNING: %s already exists, creating a temporary staging area %s" % (CMSSW_WORK,TMP_DIR)
595  #Local cases
596  elif defaultlocal:
597  StagingArea = CMSSW_WORK
598  try:
599  os.mkdir(os.path.join(CMSSW_BASE,"work"))
600  os.mkdir(os.path.join(CMSSW_BASE,"work","Results"))
601  except OSError:
602  pass
603  print "**User did not specify location of results, staging in default %s**" % StagingArea
604  else:
605  print "**User chose to publish results in a local directory**"
606  StagingArea = path
607  if not os.path.exists(path):
608  try:
609  os.mkdir("%s" % path)
610  except OSError as detail:
611  if detail.errno == 13:
612  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
613  elif detail.errno == 17:
614  #If the directory already exists just carry on
615  pass
616  else:
617  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
618 
619  IS_TMP = not TMP_DIR == ""
620  ######
621  #
622  # create Version dir
623  # This is why when we create a Tmp dir we get the version(tmpname)/version
624  # structure. We should remove this if we don't want it but i dont think it matters
625  StagingArea="%s/%s" % (StagingArea,CMSSW_VERSION)
626  try:
627  os.mkdir("%s" % StagingArea)
628  except OSError as detail:
629  if detail.errno == 13:
630  fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea)
631  elif detail.errno == 17:
632  #If the directory already exists just carry on
633  pass
634  else:
635  fail("ERROR: There was some problem (%s) when creating the staging directory" % detail)
636 
637  return (drive,path,remote,StagingArea,port,repdir,previousrev,options.ig_remotedir)
cmsPerfPublish.getStageRepDirs
def getStageRepDirs
Determine locations of staging and report dirs.
Definition: cmsPerfPublish.py:495
def cmsPerfPublish.logrep_cmp (   x,
  y 
)

Definition at line 752 of file cmsPerfPublish.py.

References step_cmp().

752 
753 def logrep_cmp(x,y):
754  (fname,x) = x
755  (fname,y) = y
756  x = os.path.basename(x)
757  y = os.path.basename(y)
758  stepreg = re.compile("%s_(..*)_TimingReport.log" % fname)
759  if stepreg.search(x):
760  x = stepreg.search(x).groups()[0]
761  if stepreg.search(y):
762  y = stepreg.search(y).groups()[0]
763  return step_cmp(x,y)
def cmsPerfPublish.main ( )

Definition at line 203 of file cmsPerfPublish.py.

References createWebReports(), delTmpDir(), get_environ(), getArchVersionFromLog(), getDirnameDirs(), getNumOfEventsFromLog(), getStageRepDirs(), optionparse(), print_header(), scanReportArea(), stageIgProfReports(), syncToRemoteLoc(), and syscp().

203 
204 def main():
205  global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents
206  #Bad design... why is this function defined here?
207  #Either no function, or define somewhere else.
208  def _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage):
209  """Use function syscp to copy LogFiles and cmsScimarkDir over to staging area"""
210  if _verbose:
211  print "Copying the logfiles to %s/." % stage
212  print "Copying the cmsScimark2 results to the %s/." % stage
213 
214  syscp(LogFiles , stage + "/")
215  syscp(cmsScimarkDir, stage + "/")
216 
217  #Same comment as above about the opportunity of this function definition here.
218  def _createLogFile(LogFile,date,LocalPath,ShowTagsResult):
219  """Creating a small logfile with basic publication script running information (never used really by other scripts in the suite)."""
220  try:
221  LOG = open(LogFile,"w")
222  if _verbose:
223  print "Writing Production Host, Location, Release and Tags information in %s" % LogFile
224  LOG.write("These performance tests were executed on host %s and published on %s" % (HOST,date))
225  LOG.write("They were run in %s" % LocalPath)
226  LOG.write("Results of showtags -r in the local release:\n%s" % ShowTagsResult)
227  LOG.close()
228  except IOError as detail:
229  print "WARNING: Can't create log file"
230  print detail
231 
232  # Print Program header
233  print_header()
234 
235  # Get environment variables
236  #FIXME: should check into this and make sure the proper variables for the tests being published are read from logfile (case of deprecated releases...)
237  print "\n Getting Environment variables..."
238  (LocalPath, ShowTagsResult) = get_environ()
239 
240  # Parse options
241  (options,args) = optionparse()
242 
243  # Determine program parameters and input/staging locations
244  print "\n Determining locations for input and staging..."
245  (drive,path,remote,stage,port,repdir,prevrev,igprof_remotedir) = getStageRepDirs(options,args)
246 
247  #Get the number of events for each test from logfile:
248  print "\n Getting the number of events for each test..."
249  #Let's do a quick implementation of something that looks at the logfile:
250  cmsPerfSuiteLogfile="%s/cmsPerfSuite.log"%repdir
251 
252  if os.path.exists(cmsPerfSuiteLogfile):
253  try:
254  (TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents)=getNumOfEventsFromLog(cmsPerfSuiteLogfile)
255  #Get the CMSSW version and SCRAM architecture from log (need these for the IgProf new publishing with igprof-navigator)
256  (CMSSW_arch,CMSSW_version)=getArchVersionFromLog(cmsPerfSuiteLogfile)
257  #For now keep the dangerous default? Better set it to a negative number...
258  except:
259  print "There was an issue in reading out the number of events for the various tests or the architecture/CMSSW version using the standard logfile %s"%cmsPerfSuiteLogFile
260  print "Check that the format was not changed: this scripts relies on the initial perfsuite arguments to be dumped in the logfile one per line!"
261  print "For now taking the default values for all tests (0)!"
262 
263  print "\n Scan report directory..."
264  # Retrieve some directories and information about them
265  (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir) = scanReportArea(repdir)
266  print "cmsScimarkResults are %s"%cmsScimarkResults
267  print "\n Copy report files to staging directory..."
268  # Copy reports to staging area
269  _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage)
270 
271  print "\n Creating log file..."
272  # Produce a small logfile with basic info on the Production area
273  _createLogFile("%s/ProductionLog.txt" % stage,date,repdir,ShowTagsResult)
274 
275  #Check if there are IgProf tests:
276  for dirname in os.listdir(repdir):
277  if "IgProf" in dirname:
278  print "\n Handling IgProf reports..."
279  # add a function to handle the IgProf reports
280  stageIgProfReports(igprof_remotedir,CMSSW_arch,CMSSW_version)
281 
282  print "\n Creating HTML files..."
283  # create HTML files
284  createWebReports(stage,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev)
285 
286  print "\n Copy profiling logs to staging directory..."
287  # Copy over profiling logs...
288  getDirnameDirs(repdir,stage)
289 
290  # Send files to remote location
291  if remote:
292  print "\n Uploading web report to remote location..."
293  syncToRemoteLoc(stage,drive,path,port)
294  print "\n Finished uploading! Now removing staging directory..."
295  delTmpDir()
296 
297  print "\n Finished!!!"
cmsPerfPublish.getDirnameDirs
def getDirnameDirs
Grab dirs that end in strings defined in DirName.
Definition: cmsPerfPublish.py:1809
cmsPerfPublish.scanReportArea
def scanReportArea
Scan report area for required things.
Definition: cmsPerfPublish.py:642
cmsPerfPublish.delTmpDir
def delTmpDir
Delete tmp dir if we used it.
Definition: cmsPerfPublish.py:1848
cmsPerfPublish.syncToRemoteLoc
def syncToRemoteLoc
Upload stage to remote location.
Definition: cmsPerfPublish.py:1829
cmsPerfPublish.getStageRepDirs
def getStageRepDirs
Determine locations of staging and report dirs.
Definition: cmsPerfPublish.py:495
cmsPerfPublish.createWebReports
def createWebReports
Create web report index and create HTML file for each candle.
Definition: cmsPerfPublish.py:1448
cmsPerfPublish.get_environ
def get_environ
Get require environment variables.
Definition: cmsPerfPublish.py:302
cmsPerfPublish.optionparse
def optionparse
Option parser.
Definition: cmsPerfPublish.py:332
def cmsPerfPublish.optionparse ( )

Option parser.

Definition at line 332 of file cmsPerfPublish.py.

Referenced by main().

332 
333 def optionparse():
334  global PROG_NAME, _debug, _dryrun, _verbose
335 
336  parser = opt.OptionParser(usage=("""%s [HOST:]DEST_PATH [Options]
337 
338  Arguments:
339  [HOST:]DEST_PATH - This is where the report should be published, you can specify a local path or a directory on a remote machine (HOST is optional)
340 
341  Examples:
342  Publish report to default local directory
343  ./%s
344  Publish report to \"/some/local/dir\"
345  ./%s /some/local/dir
346  Publish report to \"/some/other/dir\" remote host \"hal.cern.ch\"
347  ./%s hal.cern.ch:/some/other/dir
348  Publish report to default relval location (this could be remote or local depending on the hardcoded default)
349  ./%s --relval"""
350  % ( PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME)))
351 
352  devel = opt.OptionGroup(parser, "Developer Options",
353  "Caution: use these options at your own risk."
354  "It is believed that some of them bite.\n")
355  parser.add_option(
356  '--relval',
357  action="store_true",
358  dest='relval',
359  help='Use the default RelVal location',
360  #metavar='<STEPS>',
361  )
362 
363  parser.add_option(
364  '-v',
365  '--verbose',
366  action="store_true",
367  dest='verbose',
368  help='output more information',
369  #metavar='<STEPS>',
370  )
371 
372  parser.add_option(
373  '--simul',
374  action="store_true",
375  dest='simulation',
376  help='Use the default simulation location',
377  #metavar='<STEPS>',
378  )
379 
380  parser.add_option(
381  '--prev',
382  type="string",
383  dest='previousrev',
384  help='The override the name of the previous release. Default is the string obtain from the identification file REGRESSION.<prevrel>.vs.<newrel>',
385  metavar='<NAME>',
386  default="",
387  )
388 
389  parser.add_option(
390  '--input',
391  type="string",
392  dest='repdir',
393  help='The location of the report files to be published',
394  metavar='<DIR>'
395  )
396 
397  parser.add_option(
398  '-p',
399  '--port',
400  type='int',
401  dest='port',
402  help='Use a particular port number to rsync material to a remote server',
403  metavar='<PORT>'
404  )
405  parser.add_option(
406  '--igprof',
407  type='string',
408  dest='ig_remotedir',
409  default='IgProfData', #Reverting to local default publication for now#'/afs/cern.ch/cms/sdt/web/qa/igprof-testbed/data', #For now going straight into AFS... later implement security via local disk on cmsperfvm and cron job there:
410  #default='cmsperfvm:/data/projects/conf/PerfSuiteDB/IgProfData', #This should not change often! In this virtual machine a cron job will run to move stuff to AFS.
411  help='Specify an AFS or host:mydir remote directory instead of default one',
412  metavar='<IGPROF REMOTE DIRECTORY>'
413  )
414 
415  devel.add_option(
416  '-d',
417  '--debug',
418  type='int',
419  dest='debug',
420  help='Show debug output',
421  #metavar='DEBUG',
422  )
423 
424  devel.add_option(
425  '--dry-run',
426  action="store_true",
427  dest='dryrun',
428  help='Do not send files to remote server, but run everything else',
429  #metavar='DEBUG',
430  )
431 
432  repdirdef = os.getcwd()
433  parser.set_defaults(debug=0,simulation=False,relval=False,port=873,pretend=False,repdir=repdirdef,verbose=False)
434  parser.add_option_group(devel)
435 
436  (options, args) = parser.parse_args()
437 
438  _debug = options.debug
439  _dryrun = options.dryrun
440  _verbose = options.verbose
441 
442  numofargs = len(args)
443 
444  if (options.simulation and options.relval) or ((options.simulation or options.relval) and numofargs >= 1):
445  parser.error("You can not specify simulation and relval together. Neither can you specify simulation or relval AND a path")
446  sys.exit()
447 
448  return (options, args)
449 
450 
#A function to get the number of events for each test from the logfile:
cmsPerfPublish.optionparse
def optionparse
Option parser.
Definition: cmsPerfPublish.py:332
def cmsPerfPublish.populateFromTupleRoot (   tupname,
  repdir,
  rootfile,
  pureg 
)

Definition at line 1240 of file cmsPerfPublish.py.

Referenced by createWebReports().

1240 
1241 def populateFromTupleRoot(tupname,repdir,rootfile,pureg):
1242  table = Table()
1243  for cand in Candles:
1244  fname = CandFname[cand]
1245  globpath = os.path.join(repdir,"%s_TimeSize" % cand,"%s_*_TimingReport" % fname)
1246  stepDirs = glob.glob(globpath)
1247  stepDirs = map(lambda x: (fname,x), stepDirs)
1248  stepDirs.sort(cmp=timerep_cmp)
1249  stepDirs = map(lambda x: x[1], stepDirs)
1250  stepreg = re.compile("%s_(.*)_TimingReport" % fname)
1251  createNewRow = True
1252  curRow = None
1253  createPURow = True
1254  puRow = None
1255  for stepdir in stepDirs:
1256  base = os.path.basename(stepdir)
1257  found = stepreg.search(base)
1258  step = "Unknown-step"
1259  if found:
1260  step = found.groups()[0]
1261  realstep = "Unknown-step"
1262  if "PILEUP" in step:
1263  found = pureg.search(step)
1264  if found:
1265  realstep = found.groups()[0]
1266  if createPURow:
1267  createPURow = False
1268  puRow = Row(table)
1269  rootf = os.path.join(stepdir,rootfile)
1270 
1271  if os.path.exists(rootf):
1272  f = ROOT.TFile(rootf)
1273 
1274  cpu_time_tree = ROOT.TTree()
1275  f.GetObject("cpu_time_tuple;1",cpu_time_tree)
1276  if cpu_time_tree:
1277  if cpu_time_tree.InheritsFrom("TTree"):
1278  data1 = None
1279  data2 = None
1280  for t in cpu_time_tree:
1281  data1 = t.total1
1282  data2 = t.total2
1283  if data1 and data2:
1284  if createNewRow:
1285  createNewRow = False
1286  curRow = table.newRow(cand)
1287  data_tuple = (data1,data2)
1288 
1289  if "PILEUP" in step:
1290  puRow.addEntry(realstep,data_tuple)
1291  else:
1292  if createNewRow:
1293  createNewRow = False
1294  curRow = table.newRow(cand)
1295 
1296  curRow.addEntry(step,data_tuple)
1297  f.Close()
1298  if puRow == None:
1299  pass
1300  else:
1301  table.addRow(puRow,"%s PILEUP" %cand)
1302  return table
1303 
def cmsPerfPublish.prettySize (   size)

Definition at line 91 of file cmsPerfPublish.py.

References TriggerAnalyzer.__str__().

Referenced by createHTMLtab().

91 
92 def prettySize(size):
93  nega = size < 0
94  if nega:
95  size = -size
96  suffixes = [("B",2**10), ("k",2**20), ("M",2**30), ("G",2**40), ("T",2**50)]
97  for suf, lim in suffixes:
98  if size > lim:
99  continue
100  else:
101  if nega:
102  return "-" + round(size/float(lim/2**10),2).__str__() + suf
103  else:
104  return round(size/float(lim/2**10),2).__str__()+suf
def cmsPerfPublish.print_header ( )

Definition at line 1990 of file cmsPerfPublish.py.

Referenced by main().

1990 
1991 def print_header():
1992  print "%s\n" % PROG_NAME
def cmsPerfPublish.reg_dirname_cmp (   x,
  y 
)

Definition at line 740 of file cmsPerfPublish.py.

References step_cmp().

740 
741 def reg_dirname_cmp(x,y):
742  (candle,prof,x) = x
743  (candle,prof,y) = y
744  x = os.path.basename(x)
745  y = os.path.basename(y)
746  stepreg = re.compile("%s_(..*)_%s_regression" % (candle,prof))
747  if stepreg.search(x):
748  x = stepreg.search(x).groups()[0]
749  if stepreg.search(y):
750  y = stepreg.search(y).groups()[0]
751  return step_cmp(x,y)
def cmsPerfPublish.rootfile_cmp (   x,
  y 
)

Definition at line 716 of file cmsPerfPublish.py.

References step_cmp().

716 
717 def rootfile_cmp(x,y):
718  (fname,x) = x
719  (fname,y) = y
720  x = os.path.basename(x)
721  y = os.path.basename(y)
722  stepreg = re.compile("%s_(..*)\.root" % fname)
723  if stepreg.search(x):
724  x = stepreg.search(x).groups()[0]
725  if stepreg.search(y):
726  y = stepreg.search(y).groups()[0]
727  return step_cmp(x,y)
def cmsPerfPublish.scanReportArea (   repdir)

Scan report area for required things. 

Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results.
It returns Execution date (completion), current date, list of logfiles and cmsScimark results

Definition at line 642 of file cmsPerfPublish.py.

References getDate().

Referenced by main().

642 
643 def scanReportArea(repdir):
644  """Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results.
645  It returns Execution date (completion), current date, list of logfiles and cmsScimark results"""
646  date=getDate()
647  LogFiles = glob.glob(repdir + "cms*.log")
648  if _verbose:
649  print "Found the following log files:"
650  print LogFiles
651 
652  cmsScimarkDir = glob.glob(repdir + "cmsScimarkResults_*")
653  if _verbose:
654  print "Found the following cmsScimark2 results directories:"
655  print cmsScimarkDir
656 
657  cmsScimarkResults = []
658  for adir in cmsScimarkDir:
659  htmlfiles = glob.glob(adir + "/*.html")
660  #FIXME:
661  #Really unnecessary use of map in my opinion
662  #Could do with cmsScimarkResults.extend(htmlfiles)
663  map(cmsScimarkResults.append,htmlfiles)
664 
665  ExecutionDateLast = ""
666  ExecutionDate = ""
667  ExecutionDateSec=0
668  cmsreg = re.compile("^cmsPerfSuite")
669  for logf in LogFiles:
670  if cmsreg.search(logf):
671  ExecutionDateLastSec = os.stat(logf)[ST_CTIME]
672  ExecutionDateLast = os.stat(logf)[ST_MTIME]
673  if _verbose:
674  print "Execution (completion) date for %s was: %s" % (logf,ExecutionDateLast)
675  if (ExecutionDateLastSec > ExecutionDateSec):
676  ExecutionDateSec = ExecutionDateLastSec
677  ExecutionDate = ExecutionDateLast
678 
679  if ExecutionDate == "":
680  ExecutionDate = ExecutionDateLast
681 
682  return (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir)
cmsPerfPublish.scanReportArea
def scanReportArea
Scan report area for required things.
Definition: cmsPerfPublish.py:642
def cmsPerfPublish.stageIgProfReports (   remotedir,
  arch,
  version 
) 
Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.

Definition at line 1395 of file cmsPerfPublish.py.

Referenced by main().

1395 
1396 def stageIgProfReports(remotedir,arch,version):
1397  '''Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.'''
1398  #FIXME: can eliminate this part if using tar pipes... was done with rsynch in mind
1399  #Compose command to create remote dir:
1400  if ":" in remotedir: #Remote host local directory case
1401  (host,dir)=remotedir.split(":")
1402  mkdir_cmd="ssh %s (mkdir %s;mkdir %s/%s)"%(host,dir,dir,arch)
1403  else: #AFS or local case
1404  mkdir_cmd="mkdir %s;mkdir %s/%s"%(remotedir,remotedir,arch)
1405 
1406  #Create remote dir:
1407  try:
1408  print mkdir_cmd
1409  os.system(mkdir_cmd)
1410  print "Successfully created publication directory"
1411  except:
1412  print "Issues with publication directory existence/creation!"
1413 
1414  #Copy files over to remote dir
1415  #replacing rsync with tar pipes since it can hang on AFS (Andreas' experience):
1416  #rsync_cmd="rsync -avz *_IgProf_*/*.sql3 %s/%s/%s"%(remotedir,arch,version)
1417  if ":" in remotedir:
1418  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | ssh %s "cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*"'%(host,dir,arch,version,version)
1419  else:
1420  tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | (cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*)'%(remotedir,arch,version,version)
1421  try:
1422  # print rsync_cmd
1423  # os.system(rsync_cmd)
1424  print tarpipe_cmd
1425  os.system(tarpipe_cmd)
1426  print "Successfully copied IgProf reports to %s"%remotedir
1427  except:
1428  print "Issues with rsyncing to the remote directory %s!"%remotedir
1429 
1430  #Make sure permissions are set for group to be able to write:
1431  if ":" in remotedir: #Remote host local directory case
1432  chmod_cmd="ssh %s chmod -R 775 %s/%s"%(host,dir,arch)
1433  else:
1434  chmod_cmd="chmod -R 775 %s/%s"%(remotedir,arch)
1435  try:
1436  print chmod_cmd
1437  os.system(chmod_cmd)
1438  print "Successfully set permissions for IgProf reports directory %s"%remotedir
1439  except:
1440  print "(Potential) issues with chmoding the remote directory %s!"%remotedir
1441 
1442  return #Later, report here something like the web link to the reports in igprof-navigator...
1443 
def cmsPerfPublish.step_cmp (   x,
  y 
)

Definition at line 776 of file cmsPerfPublish.py.

Referenced by dirname_cmp(), logrep_cmp(), reg_dirname_cmp(), rootfile_cmp(), and timerep_cmp().

776 
777 def step_cmp(x,y):
778  xstr = x
779  ystr = y
780  x_idx = -1
781  y_idx = -1
782  bestx_idx = -1
783  besty_idx = -1
784  sndbst_x = -1
785  sndbst_y = -1
786  last_x = -1
787  last_y = -1
788  for i in range(len(Step)):
789  stepreg = re.compile("^%s.*" % Step[i])
790  # fallback
791  if Step[i] in xstr and sndbst_x == -1:
792  last_x = i
793  if Step[i] in ystr and sndbst_y == -1:
794  last_y = i
795  # second best
796  if xstr in Step[i] and bestx_idx == -1:
797  sndbst_x = i
798  if ystr in Step[i] and besty_idx == -1:
799  sndbst_y = i
800  # next best
801  if stepreg.search(xstr) and x_idx == -1:
802  bestx_idx = i # If an exact match has not been found but something similar has, set x best index
803  if stepreg.search(ystr) and y_idx == -1:
804  besty_idx = i # If an exact match has not been found but something similar has, set y best index
805  # actual
806  if Step[i] == xstr and x_idx == -1:
807  x_idx = i # if an exact match has been found then set x index
808  if Step[i] == ystr and y_idx == -1:
809  y_idx = i # if an exact match has been found then set y index
810  if not ( x_idx == -1 or y_idx == -1):
811  break # if an exact match has been found for both, stop
812 
813  # use best match if we still can't find indices
814  if x_idx == -1:
815  x_idx = bestx_idx
816  if y_idx == -1:
817  y_idx = besty_idx
818 
819  # use second best if we still can't find indices
820  if x_idx == -1:
821  x_idx = sndbst_x
822  if y_idx == -1:
823  y_idx = sndbst_y
824 
825  # use fallback if we still can't find indices
826  if x_idx == -1:
827  x_idx = last_x
828  if y_idx == -1:
829  y_idx = last_y
830 
831  if x_idx == -1 or y_idx == -1:
832  print "WARNING: No valid step names could be found in the logfiles or root filenames being sorted: x: %s y: %s." % (xstr,ystr)
833  print "x", x_idx, "y", y_idx
834 
835  if x_idx < y_idx:
836  return -1
837  elif x_idx == y_idx:
838  return 0
839  elif y_idx < x_idx:
840  return 1
841 
def cmsPerfPublish.syncToRemoteLoc (   stage,
  drive,
  path,
  port 
)

Upload stage to remote location.

Definition at line 1829 of file cmsPerfPublish.py.

References addtrailingslash().

Referenced by main().

1829 
1830 def syncToRemoteLoc(stage,drive,path,port):
1831  stage = addtrailingslash(stage)
1832  cmd = "rsync -avz"
1833  # We must, MUST, do os.path.normpath otherwise rsync will dump the files in the directory
1834  # we specify on the remote server, rather than creating the CMS_VERSION directory
1835  #--rsh=\"ssh -l relval\"
1836  args = "--port=%s %s %s:%s" % (port,os.path.normpath(stage),drive,path)
1837  retval = -1
1838  if _dryrun:
1839  print cmd + " --dry-run " + args
1840  retval = os.system(cmd + " --dry-run " + args )
1841  else:
1842  print cmd+" "+args
1843  retval = os.system(cmd + " " + args)
1844  return retval
cmsPerfPublish.syncToRemoteLoc
def syncToRemoteLoc
Upload stage to remote location.
Definition: cmsPerfPublish.py:1829
def cmsPerfPublish.syscp (   srcs,
  dest 
)

Definition at line 1970 of file cmsPerfPublish.py.

References copytree4(), and docopy().

Referenced by createCandlHTML(), createWebReports(), getDirnameDirs(), and main().

1970 
1971 def syscp(srcs,dest):
1972  if type(srcs) == type(""):
1973  if os.path.exists(srcs):
1974  if os.path.isdir(srcs):
1975  copytree4(srcs,dest)
1976  else:
1977  docopy(srcs,dest)
1978  else:
1979  print "ERROR: file to be copied %s does not exist" % foo
1980  else:
1981  for src in srcs:
1982  if os.path.exists(src):
1983  if os.path.isdir(src):
1984  #copy tree
1985  copytree4(src,dest)
1986  else:
1987  docopy(src,dest)
1988  else:
1989  print "ERROR: file to be copied %s does not exist" % foo
def cmsPerfPublish.timerep_cmp (   x,
  y 
)

Definition at line 764 of file cmsPerfPublish.py.

References step_cmp().

764 
765 def timerep_cmp(x,y):
766  (fname,x) = x
767  (fname,y) = y
768  x = os.path.basename(x)
769  y = os.path.basename(y)
770  stepreg = re.compile("%s_(..*)_TimingReport" % fname)
771  if stepreg.search(x):
772  x = stepreg.search(x).groups()[0]
773  if stepreg.search(y):
774  y = stepreg.search(y).groups()[0]
775  return step_cmp(x,y)

Variable Documentation

int cmsPerfPublish.CallgrindNumOfEvents = -9999

Definition at line 35 of file cmsPerfPublish.py.

tuple cmsPerfPublish.cpDirFilter = ( )

Definition at line 31 of file cmsPerfPublish.py.

tuple cmsPerfPublish.cpFileFilter = ( "*.root", )

Definition at line 30 of file cmsPerfPublish.py.

string cmsPerfPublish.DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal"

Definition at line 27 of file cmsPerfPublish.py.

string cmsPerfPublish.DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation"

Definition at line 28 of file cmsPerfPublish.py.

tuple cmsPerfPublish.DirName
Initial value:
1 = ( #These need to match the candle directory names ending (depending on the type of profiling)
2  "TimeSize",
3  "IgProf",
4  "IgProf_Perf",
5  "IgProf_Mem",
6  "Callgrind",
7  "Memcheck",
8  #Adding the extra PU directories:
9  "PU_TimeSize",
10  "PU_IgProf",
11  "PU_IgProf_Perf",
12  "PU_IgProf_Mem",
13  "PU_Callgrind",
14  "PU_Memcheck"
15  )

Definition at line 38 of file cmsPerfPublish.py.

Referenced by JetAnalyzer.analyze(), METAnalyzer.analyze(), JetAnalyzer.bookHistograms(), METAnalyzer.bookHistograms(), and METAnalyzer.endRun().

int cmsPerfPublish.IgProfNumOfEvents = -9999

Definition at line 34 of file cmsPerfPublish.py.

int cmsPerfPublish.MemcheckNumOfEvents = -9999

Definition at line 36 of file cmsPerfPublish.py.

tuple cmsPerfPublish.PROG_NAME = os.path.basename(sys.argv[0])

Definition at line 26 of file cmsPerfPublish.py.

tuple cmsPerfPublish.Steps = set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"])

Definition at line 54 of file cmsPerfPublish.py.

int cmsPerfPublish.TimeSizeNumOfEvents = -9999

Definition at line 33 of file cmsPerfPublish.py.

string cmsPerfPublish.TMP_DIR = ""

Definition at line 29 of file cmsPerfPublish.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5