CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
cmsRelRegress.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import os, re, sys, glob
4 import optparse as opt
5 import cmsPerfRegress as cpr
6 from cmsPerfCommons import Candles, CandFname, getVerFromLog
7 
8 def getParameters():
9  global _debug
10  global PROG_NAME
11  PROG_NAME = os.path.basename(sys.argv[0])
12  parser = opt.OptionParser(usage="""%s [OLD_REL_DIR] [NEW_REL_DIR]
13 
14 To compare 2 cmsPerfSuite.py directories pass the previous release as the first argument and the latest release as the second argument """ % PROG_NAME)
15  #
16  # Options
17  #
18  devel = opt.OptionGroup(parser, "Developer Options",
19  "Caution: use these options at your own risk."
20  "It is believed that some of them bite.\n")
21  devel.add_option(
22  '-d',
23  '--debug',
24  type='int',
25  dest='debug',
26  default = 0,
27  help='Show debug output',
28  #metavar='DEBUG',
29  )
30  parser.add_option_group(devel)
31  (options,args) = parser.parse_args()
32  _debug = options.debug
33 
34  if not len(args) == 2:
35  print "ERROR: Not enough arguments"
36  sys.exit()
37 
38  path1 = os.path.abspath(args[0])
39  path2 = os.path.abspath(args[1])
40  if os.path.exists(path1) and os.path.exists(path2):
41  return (path1, path2)
42  else:
43  print "Error: one of the paths does not exist"
44  sys.exit()
45 
46 def getOldRelName(oldRelName,adir):
47  #Not sure this function is used but as it was written before it was useless.
48  #Now it parses the adir directory looking for the CMSSW_X_Y_Z(_preN) in the path.
49  if oldRelName == "":
50  oldRelPath = os.path.dirname(adir)
51  oldRelPathDirs = oldRelPath.split("/")
52  for dir in oldRelPathDirs:
53  if 'CMSSW' in dir:
54  oldRelName=dir
55  return oldRelName
56 
57 def compareSimMemPair(newLog,candle,profdir,curdir,oldlog,oldRelName=""):
58  print "oldlog %s"%oldlog
59  print "curdir %s"%curdir
60  #oldRelName = getOldRelName(oldRelName,olddir)
61  oldRelName = getOldRelName(oldRelName,oldlog)
62  print "OLD REL NAME: %s"%oldRelName
63  #base = os.path.basename(newLog)
64  #oldlog = os.path.join(olddir,curdir,base)
65  rootf = "simpmem-regress.root"
66  try:
67  print "TRY candle %s"%candle
68  print "HERE Oldlog:%s"%oldlog
69  print "HERE newLog:%s"%newLog
70  cpr.cmpSimpMemReport(rootf,curdir,oldlog,newLog,1,True,candle,prevrev = oldRelName)
71  except cpr.SimpMemParseErr as detail:
72  print "WARNING: Could not parse data from log file %s; not performing regression" % detail.message
73  except OSError as detail:
74  print "WARNING: The OS returned the following error when comparing %s and %s" % (oldlog,log), detail
75  except IOError as detail:
76  print "IOError:", detail
77  else:
78  print "Successfully compared %s and %s" % (oldlog,newLog)
79 
80 def regressReports(olddir,newdir,oldRelName = "",newRelName=""):
81 
82  profSets = ["Callgrind",
83  #"Memcheck", #No regression on Memcheck profiles!
84  "IgProf",
85  "TimeSize",
86  #Adding the PU directories:
87  "PU_Callgrind",
88  "PU_IgProf",
89  "PU_TimeSize"
90  ]
91  for candle in Candles:
92  #Loop over the known profilers sets (tests) defined above:
93  for profset in profSets:
94  #Check there is a directory with the profile set (test) being considered:
95  adir = os.path.join(newdir,"%s_%s" % (candle,profset))
96  if os.path.exists(adir):
97  #Start working in directory adir (e.g. MinBias_TimeSize)
98  print "Found directory %s"%adir
99 
100  #Set up the profilers based on the directory name
101  Profs = []
102  if profset == "Callgrind" or profset == "PU_Callgrind":
103  Profs = ["valgrind"] # callgrind actually
104  elif profset == "TimeSize" or profset == "PU_TimeSize":
105  Profs = [ "TimingReport",
106  #"TimeReport", We do not run regression on the plain html TimeReport profile...
107  "SimpleMemoryCheck",
108  "EdmSize"]
109  elif profset == "IgProf" or profset == "PU_IgProf" :
110  Profs = [ "IgProfperf", #This was missing!
111  "IgProfMemTotal",
112  "IgProfMemLive"]
113  #Now for each individual profile in the profile set (e.g for TimeSize TimeReport, TimingReport, SimpleMemoryCheck, EdmSize
114  #collect the various logfiles
115  for prof in Profs:
116  print "Checking %s profile(s)"%prof
117  if prof == "EdmSize" or prof == "valgrind":
118  stepLogs = glob.glob("%s/%s_*_%s" % (adir,CandFname[candle],prof))
119  elif prof == "IgProfMemLive" or prof == "IgProfMemTotal":
120  stepLogs = glob.glob("%s/%s_*_%s.gz" % (adir,CandFname[candle],"IgProfMemTotal")) #This hack necessary since we reuse the IgProfMemTotal profile for MemLive too (it's a unique IgProfMem profile, read with different counters)
121  elif prof == "IgProfperf":
122  stepLogs = glob.glob("%s/%s_*_%s.gz" % (adir,CandFname[candle],prof))
123  elif prof == "SimpleMemoryCheck":
124  #With the change in the use of tee now the SimpleMemoryCheck info will be in the _TimingReport.log too...
125  #The following lines only will work for the unprofiled steps... hence... no need to report them!
126  #stepLogs = os.path.join(adir,"%s.log" % candle)
127  stepLogs = glob.glob("%s/%s_*_%s.log" % (adir,CandFname[candle],'TimingReport'))
128  elif prof == "TimingReport":
129  stepLogs = glob.glob("%s/%s_*_%s.log" % (adir,CandFname[candle],prof))
130 
131  #Debug:
132  print "Found the following step logs: %s"%stepLogs
133 
134  profdir = os.path.basename(adir)
135 
136  #Giant if to single out the SimpleMemoryCheck case that is in the elif at the bottom... maybe should flip things around...
137  #Basically here we do everything but SimpleMemoryCheck:
138  if prof == "TimingReport" or prof == "EdmSize" or prof == "valgrind" or prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof == "IgProfperf":
139  #This hack necessary since we reuse the IgProfMemTotal profile for MemLive too
140  #(it's a unique IgProfMem profile, read with different counters)
141  if prof == "IgProfMemLive":
142  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s((.log)|(.gz))?" % (CandFname[candle],"IgProfMemTotal"))
143  else:
144  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s((.log)|(.gz))?" % (CandFname[candle],prof))
145 
146  #Loop on the step logfiles collected above
147  for log in stepLogs:
148  base = os.path.basename(log)
149  #Handle the fact the profile ("log") for IgProf is always compressed (.gz):
150  if prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof == "IgProfperf":
151  base = base.split(".gz")[0]
152  #Use the regular expression defined above to read out the step from the log/profile
153  searchob = stepreg.search(base)
154 
155  #If in this log the regular expression was able match (and so to extract the step)
156  if searchob:
157  #print searchob.groups()
158  step = searchob.groups()[0]
159  #print "and the step taken is %s"%step
160  outpath = os.path.join(adir,"%s_%s_%s_regression" % (CandFname[candle],step,prof))
161  oldlog = os.path.join(olddir,"%s_%s" % (candle,profset),base)
162  #Again handle the fact the profile ("log") for IgProf is always compressed (.gz):
163  if prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof == "IgProfperf":
164  oldlog = os.path.join(olddir,"%s_%s" % (candle,profset),base + ".gz")
165  if not os.path.exists(outpath):
166  os.mkdir(outpath)
167  if os.path.exists(oldlog):
168  try:
169  print ""
170  print "** "
171  if not prof == "TimingReport":
172  print "** Comparing", candle, step, prof, "previous release: %s, latest release %s" % (oldlog,log)
173  print "**"
174 
175  if prof == "EdmSize":
176  cpr.cmpEdmSizeReport(outpath,oldlog,log)
177  elif prof == "TimingReport":
178  logdir = "%s_%s_%s" % (CandFname[candle],step,prof)
179  outd = os.path.join(adir,logdir)
180  rootf = "timing-regress.root"
181  oldlog = os.path.join(olddir,profdir,base)
182  if os.path.exists(log) and os.path.exists(oldlog) and os.path.exists(outd):
183  print "** Comparing", candle, step, prof, "previous release: %s and latest release: %s" % (oldlog,log)
184  print "**"
185  oldRelName = getOldRelName("",oldlog)
186  #print "TIMING OLD REL extracted from %s :\n %s"%(oldlog,oldRelName)
187  cpr.cmpTimingReport(rootf, outd, oldlog, log, 1, batch = True, prevrev = oldRelName)
188  else:
189  print "WARNING: While comparing", candle, step, prof, " at least one of the logfiles/directories: old (%s) or new (%s) was not found!!!" % (oldlog,log)
190  break
191  elif prof == "valgrind":
192  cpr.cmpCallgrindReport(outpath,oldlog,log)
193  elif prof == "IgProfperf":
194  IgProfMemOpt="" #No need to specify the counter, for IgProfPerf...
195  cpr.cmpIgProfReport(outpath,oldlog,log,IgProfMemOpt)
196  elif prof == "IgProfMemTotal":
197  IgProfMemOpt="-y MEM_TOTAL"
198  cpr.cmpIgProfReport(outpath,oldlog,log,IgProfMemOpt)
199  elif prof == "IgProfMemLive":
200  IgProfMemOpt="-y MEM_LIVE"
201  cpr.cmpIgProfReport(outpath,oldlog,log,IgProfMemOpt)
202  except cpr.PerfReportErr as detail:
203  print "WARNING: Perfreport return non-zero exit status when comparing %s and %s. Perfreport output follows" % (oldlog,log)
204  print detail.message
205  except cpr.TimingParseErr as detail:
206  print "WARNING: Could not parse data from log file %s; not performing regression" % detail.message
207  except OSError as detail:
208  print "WARNING: The OS returned the following error when comparing %s and %s" % (oldlog,log), detail
209  except IOError as detail:
210  print "IOError:", detail
211  else:
212  print "Successfully compared %s and %s" % (oldlog,log)
213  else:
214  print "WARNING: Could not find an equivalent logfile for %s in the previous release dir %s " % (log,oldlog)
215 
216 
217  else:
218  continue
219  elif prof == "SimpleMemoryCheck":
220  #print "The logfiles for SimpleMemoryCheck are %s"%stepLogs
221  for log in stepLogs:
222  #print "The logfile considered now is %s"%log
223  stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s((.log)|(.gz))?" % (CandFname[candle],"TimingReport"))
224  base = os.path.basename(log)
225  #Use the regular expression defined above to read out the step from the log/profile
226  searchob = stepreg.search(base)
227  #print "Value of searchob is %s"%searchob
228  #If in this log the regular expression was able match (and so to extract the step)
229  if searchob:
230  #print searchob.groups()
231  step = searchob.groups()[0]
232  print "and the step taken is %s"%step
233  #outpath = os.path.join(adir,"%s_%s_%s_regression" % (CandFname[candle],step,prof))
234  oldlog = os.path.join(olddir,"%s_%s" % (candle,profset),base)
235  if os.path.exists(oldlog):
236  print ""
237  print "** "
238  print "** Comparing for SimpleMemoryCheck", candle, step, prof, "previous release: %s, latest release %s" % (oldlog,log)
239  print "**"
240  #The try/except is folded in the following function for SimpleMemoryCheck:
241  compareSimMemPair(log,candle,profdir,adir,oldlog,oldRelName="")
242 
243  if newRelName == "":
244  newRelName = getVerFromLog(newdir)
245  regress = open("%s/REGRESSION.%s.vs.%s" % (newdir,getVerFromLog(olddir),newRelName),"w")
246  regress.write(olddir)
247  regress.close()
248 
249 def _main():
250  (oldpath,newpath) = getParameters()
251  regressReports(oldpath,newpath,oldRelName=getVerFromLog(oldpath))
252 
253 
254 if __name__ == "__main__":
255  _main()
256 
Generated for CMSSW Reference Manual by doxygen 1.8.5