cmsPerfRegress.py

Go to the documentation of this file.

#!/usr/bin/env python

from __future__ import print_function
import time, os, sys, math, re, gzip
import tempfile as tmp
import optparse as opt
from cmsPerfCommons import CandFname
#from ROOT import gROOT, TCanvas, TF1
import ROOT
from array import array
#Substitute popen with subprocess.Popen! popen obsolete...
import subprocess
from functools import reduce

_cmsver = os.environ['CMSSW_VERSION']
values_set=('vsize','delta_vsize','rss','delta_rss')

class Error(Exception):
    """Base class for exceptions in this module."""
    pass

class TimingParseErr(Error):
    """Exception raised when Could not parse TimingReport Log.

    Attributes:
        expression -- input expression in which the error occurred
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message

class SimpMemParseErr(Error):
    """Exception raised when Could not parse TimingReport Log.

    Attributes:
        expression -- input expression in which the error occurred
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message

class EdmSizeErr(Error):
    """Exception raised when Could not parse TimingReport Log.

    Attributes:
        expression -- input expression in which the error occurred
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message            

class PerfReportErr(Error):
    """Exception raised when Could not parse TimingReport Log.

    Attributes:
        expression -- input expression in which the error occurred
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message 

#############
# Option parser
# 
def getParameters():
    parser = opt.OptionParser()
    #
    # Options
    #
    parser.add_option('-n',
                      type="int",
                      help='Number of secs per bin. Default is 1.' ,
                      default=1,
                      dest='startevt')
    parser.add_option('-t',
                      '--report-type',
                      type="choice",
                      choices= ("timing", "simplememory","edmsize","igprof","callgrind",""),
                      help='Type of report to perform regrssion on. Default is TimingReport.' ,
                      default="timing",
                      dest='reporttype')
    parser.add_option('-i',
                      '--IgProfMemOption',
                      type="choice",
                      choices= ("-y MEM_TOTAL", "-y MEM_LIVE",""),
                      help='Eventual IgProfMem counter to use for the regression. Default is no argument (IgProfPerf).' ,
                      default="",
                      dest='igprofmem')
    (options,args) = parser.parse_args()
    if len(args) < 2:
        parser.error("ERROR: Not enough arguments")
        sys.exit()


    path1 = os.path.abspath(args[0])
    path2 = os.path.abspath(args[1])    
    if os.path.exists(path1) and os.path.exists(path2):
        return (path1, path2, options.startevt, options.reporttype, options.igprofmem)
    else:
        print("Error: one of the paths does not exist")
        sys.exit()

###########
# Get max value in data set
#
def get_max(data,index=1):
    max_time=-1
    for el in data:
        sec=el[index]
        if max_time<sec:
            max_time=sec
    return max_time

###########
# Get min value in data set
#
def get_min(data,index=1):
    min_time=1e20
    for el in data:
        sec=el[index]
        if min_time>sec:
            min_time=sec
    return min_time  

###########
# Setup PyRoot as a batch run
#
def setBatch():
    __argv=sys.argv # trick for a strange behaviour of the TApp..
    sys.argv=sys.argv[:1]
    ROOT.gROOT.SetStyle("Plain") # style paranoia
    sys.argv=__argv
    #Cannot use this option when the logfile includes
    #a large number of events... PyRoot seg-faults.
    #Set ROOT in batch mode to avoid canvases popping up!
    ROOT.gROOT.SetBatch(1)

##########
# Create the root file to save the graphs in
#
def createROOT(outdir,filename):

    # Save in file
    rootfilename = os.path.join(outdir,filename)
    myfile = None
    exists = os.path.exists(rootfilename)
    if exists:
        myfile=ROOT.TFile(rootfilename,'UPDATE')
    else:
        myfile=ROOT.TFile(rootfilename,'RECREATE')        
    return myfile

##########
# Parse timing data from log file
#
def getTimingLogData(logfile_name):
    data=[]
    
    # open file and read it and fill the structure!
    logfile=open(logfile_name,'r')
    logfile_lines=logfile.readlines()
    logfile.close()

    # we get the info we need!
    i=0
    while i < len(logfile_lines):
        line=logfile_lines[i]
        if 'TimeEvent>' in line:
            line=line.strip()
            line_content_list = line.split(' ')[0:]
            event_number = int(line_content_list[1])
            seconds = float(line_content_list[3])
            data.append((event_number,seconds))
        i+=1

    return data

###########
# Parse memory check data from log file
#
def getSimpleMemLogData(logfile_name,startevt, candle):
    data=[]
    values_set=('vsize','delta_vsize','rss','delta_rss')
    
    # open file and read it and fill the structure!
    logfile=open(logfile_name,'r')
    logfile_lines=logfile.readlines()
    logfile.close()
    
    step = ""
    steps = []
    #print candle
    #print CandFname[candle]
    #Get the step from log filename:

    #Catching the case of command line use in which the candle is unknown (and the log name does not match the perfsuite naming convention
    if candle:
        stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s((.log)|(.gz))?" % (CandFname[candle],"TimingReport"))
    else:
        stepreg = re.compile("([^_]*(_PILEUP)?)_%s((.log)|(.gz))?"%"TimingReport")
        
    #print logfile_name
    found=stepreg.search(logfile_name)
    if found:
        step=found.groups()[0]
        print("Determined step from log filename to be %s"%step)
    else:
        print("Could not determine step from log filename")
        
    #steps.append((step,data))

    #Rewriting the following code... what a mess!
    # we get the info we need!
    #i=0   
    #while i < len(logfile_lines):
        #line=logfile_lines[i]
        #Format has changed... we use the output of individual steps, so no need to read into the logfile to find which step we are
        #referring too (by the way it would not work now!)... the step info comes from the logfilename done above...
        #if "RelValreport" in line and "cmsDriver" in line and "step" in line:
        #    stepreg = re.compile("--step=([^ ]*)")
        #    found = stepreg.search(line)
        #    if found:
        #        if step == "":
        #            step = found.groups()[0]
        #        else:
        #            steps.append((step,data))
        #            step = found.groups()[0]
        #            data = []        
        #if '%MSG-w MemoryCheck:' in line:
        #    line=line[:-1] #no \n!
        #    line_content_list=line.split(' ')
        #    event_number=int(line_content_list[-1])
        #    if event_number<startevt:
        #        i+=1
        #        continue
        #    i+=1 # we inspect the following line
        #    try:
        #        line=logfile_lines[i]
        #    except IndexError:
        #        continue
        #    line=line[:-1] #no \n!
        #    line_content_list=line.split(' ')
        #    vsize=float(line_content_list[4])
        #    delta_vsize=float(line_content_list[5])
        #    rss=float(line_content_list[7])
        #    delta_rss=float(line_content_list[8])
        #    
        #    data.append((event_number,{'vsize':vsize,
        #                               'delta_vsize':delta_vsize,
        #                               'rss':rss,
        #                               'delta_rss':delta_rss}))
        #i += 1
    event_number= startevt
    for line in logfile_lines:
        #Match SimpleMemoryCheck output to get the event number
        if '%MSG-w MemoryCheck:' in line: #Harcoded based on format: %MSG-w MemoryCheck:  PostModule 12-Jan-2009 16:00:29 CET Run: 1 Event: 1
            tokens=line.split()
            if int(tokens[-1])>= startevt:
                event_number=int(tokens[-1])
        if 'VSIZE' in line: #Harcoded based on format: MemoryCheck: event : VSIZE 648.426 0 RSS 426.332 0
            tokens=line.split()
            vsize=float(tokens[4])
            delta_vsize=float(tokens[5])
            rss=float(tokens[7])
            delta_rss=float(tokens[8])
            data.append((event_number,{'vsize':vsize,
                                       'delta_vsize':delta_vsize,
                                       'rss':rss,
                                       'delta_rss':delta_rss
                                       }
                         )
                        )
            
    if not len(data) == 0:
        #print "!!!!!!!!!!!!!Adding step %s and data!"%step
        steps.append((step,data))

    #print "PRINTOUT@@@@@@"
    #print steps
    return steps

###############
#
# Create a new timing graph and histogram 
#
def newGraphAndHisto(histoleg,leg,npoints,nbins,min_val,max_val,data,graph_num,prevrev=""):
    
    colors = [2,4]
    prevRevName = "???"
    if not prevrev == "":
        (head, tail) = os.path.split(prevrev)
        prevRevName  = os.path.basename(tail)
    releaseNames = ["Previous (%s)" % prevRevName,_cmsver]

    histo=ROOT.TH1F("Seconds per event (histo: %s)" % graph_num,'Seconds per event',nbins,min_val,max_val)

    graph=ROOT.TGraph(npoints)
        
    evt_counter=0
    peak = data[0][1]
    for evt_num,secs in data:
        if secs > peak:
            peak = secs
        graph.SetPoint(evt_counter,evt_num,secs)
        histo.Fill(secs)        
        evt_counter+=1

    allsecs = []
    map(lambda x: allsecs.append(x[1]),data)
    total = reduce(lambda x,y: x + y,allsecs)
    mean  = total / evt_counter
    rms   = math.sqrt(reduce(lambda x,y: x + y,map(lambda x: x * x,allsecs)) / evt_counter)
                
        
    graph.SetMarkerStyle(8)
    graph.SetMarkerSize(.7)
    graph.SetMarkerColor(1)
    graph.SetLineWidth(3)
    graph.SetLineColor(colors[graph_num]) # for each iterate through colors
    histo.SetLineColor(colors[graph_num])
    histo.SetStats(1)
    histoleg.AddEntry(histo, "%s release" % releaseNames[graph_num], "l")
    leg.AddEntry(graph     , "%s release" % releaseNames[graph_num], "l")
    leg.AddEntry(graph     , "Mean: %s" % str(mean)                , "l")            
    leg.AddEntry(graph     , "RMS : %s" % str(rms)                 , "l")
    leg.AddEntry(graph     , "Peak: %s" % str(peak)                , "l")
    leg.AddEntry(graph     , "Total time: %s" % str(total)         , "l")                    
    if graph_num == 0 :
        histo.SetFillColor(colors[graph_num])

    return (graph,histo,mean)

###########
# Get limits to plot the graph
#
def getLimits(data,secsperbin):
    min_val=get_min(data,1)
    max_val=get_max(data,1)
    interval=int(max_val-min_val)
    
    min_val=min_val-interval*0.2
    max_val=max_val+interval*0.2
    interval=int(max_val-min_val)
    
    nbins=int(interval/secsperbin)

    npoints=len(data)

    last_event=data[-1][0]

    return (min_val,max_val,interval,npoints,last_event)

##############
# Setup graph information for one graph (no need to it on both if they are superimposed)
#
def setupSuperimpose(graph1,graph2,last_event,max_val,reporttype=0, title = ""):
    name   = ""
    xtitle = ""
    ytitle = ""
    if   reporttype == 0:
        title  = 'Seconds per event'
        name   = 'SecondsPerEvent'
        xtitle = "Event"
        ytitle = "Processing time for each event (s)"
        graph1.GetYaxis().SetRangeUser(0,max_val)
        graph2.GetYaxis().SetRangeUser(0,max_val)            
    elif reporttype == 1:
        name   = "%s_graph" % title
        xtitle = "Event"
        ytitle = "MB"
        
    graph1.SetTitle(title)
    graph1.SetName(name)
    graph1.GetXaxis().SetTitle(xtitle)
    graph1.GetYaxis().SetTitleOffset(1.3)
    graph1.GetYaxis().SetTitle(ytitle)
    graph1.GetXaxis().SetLimits(0,last_event)
    # Do I need to set limits on graph 2? I don't know
    # I'm doing it anyway, can't hurt.
    graph2.GetXaxis().SetLimits(0,last_event)

#############
# Plot the mean line on a graph
#
def getMeanLines(avg,last_event,graph_num):
    colors = [2,4]
    avg_line=ROOT.TLine(1,avg,last_event,avg)
    avg_line.SetLineColor(colors[graph_num])
    avg_line.SetLineWidth(2)

    return avg_line

############
# Get the difference in timing data (for comparison of two releases)
#
def getTimingDiff(data1,data2,npoints,last_event,orig_max_val):
    data3 = []
    for x in range(len(data2)):
        try:
            if data2[x][0] == data1[x][0]:
                avgEventNum = data2[x][0]
                diffSecs    = data2[x][1] - data1[x][1]                
                data3.append((avgEventNum,diffSecs))                
        except IndexError:
            pass
        except ValueError:
            pass

    graph=ROOT.TGraph(npoints)

    evt_counter=0
    peak = data3[0][1]
    for evt_num,secs in data3:
        if secs > peak:
            peak = secs
        graph.SetPoint(evt_counter,evt_num,secs)
        evt_counter+=1

    allsecs = []
    map(lambda x: allsecs.append(x[1]),data3)
    total = reduce(lambda x,y: x + y,allsecs)
    mean  = total / evt_counter
    rms   = math.sqrt(reduce(lambda x,y: x + y,map(lambda x: x * x,allsecs)) / evt_counter)
        
    min_val=get_min(data3,1)
    max_val=get_max(data3,1)
    interval=int(max_val-min_val)
    
    min_val=min_val-interval*0.2
    max_val=max_val+interval*0.2
    interval=int(max_val-min_val)

    # Determine the max value to be something that makes the scale similar to what
    # the original graph had. Unless we can't seem the maximum value.

    new_max = min_val + orig_max_val
    if new_max < max_val:
        pass
    else :
        max_val = new_max
    
    graph.SetTitle('Change in processing time for each event between revs')
    graph.SetName('SecondsPerEvent')    
    graph.GetXaxis().SetTitle("Event Number")
    graph.GetYaxis().SetTitle("Change in processing time between revs (s)")    
    graph.GetYaxis().SetTitleOffset(1.3)
    graph.SetLineColor(2)
    graph.SetMarkerStyle(8)
    graph.SetMarkerSize(.7)
    graph.SetMarkerColor(1)
    graph.SetLineWidth(3)        
    graph.GetXaxis().SetLimits(0,last_event)
    graph.GetYaxis().SetRangeUser(min_val,max_val)
    leg = ROOT.TLegend(0.5,0.7,0.89,0.89)
    leg.AddEntry(graph, "Mean: %s" % str(mean), "l")            
    leg.AddEntry(graph, "RMS : %s" % str(rms) , "l")
    leg.AddEntry(graph, "Peak: %s" % str(peak), "l")
    leg.AddEntry(graph, "Total time change: %s" % str(total)  , "l")                    

    return (graph,leg)

##########
# Draw superimposed graphs on a separate canvas
#
def drawGraphs(graph1,graph2,avg1,avg2,leg):
    graph_canvas = ROOT.TCanvas("graph_canvas")
    graph_canvas.cd()
    graph1.Draw("ALP")
    graph2.Draw("LP")
    avg1.Draw("Same")
    avg2.Draw("Same")
    leg.Draw()
    return graph_canvas

##########
# Draw superimposed histograms on a separate canvas
#
def drawHistos(histo_stack,hstleg):
    histo_canvas = ROOT.TCanvas("histo_canvas")
    histo_canvas.cd()
    histo_stack.Draw("nostack")
    hstleg.Draw()
    return histo_canvas

##########
# Draw data differences (comparison between two data sets or releases) on a separate canvas
# 
def drawChanges(graph,chgleg):
    graph_canvas = ROOT.TCanvas("change_canvas")
    graph_canvas.cd()
    graph.Draw("ALP")
    chgleg.Draw()
    return graph_canvas    

###########
# Get limits for two graphs that will be superimposed upon one another
# 
def getTwoGraphLimits(last_event1,max_val1,last_event2,max_val2,min_val1=-1,min_val2=-1):
    biggestLastEvt = last_event1
    biggestMaxval  = max_val1
    lowest_val     = min_val1
    
    if min_val2 < lowest_val:
        lowest_val = min_val2
    if last_event2 > biggestLastEvt:
        biggestLastEvt = last_event2
    if max_val2 > biggestMaxval:
        biggestMaxval  = max_val2
    return (biggestLastEvt,biggestMaxval,lowest_val)

def getNpoints(data):
    new_data=[]
    try:
        #This was necessary due to a bug in the getSimpleMemLogData parsing!!! no more necessary!
        if data[0][0]==data[1][0]:
            print('Two modules seem to have some output.\nCollapsing ...')
            i=0
            while True:
                dataline1=data[i]
                i+=1
                dataline2=data[i]
                new_eventnumber=dataline1[0]
                new_vsize=dataline2[1]['vsize']
                new_delta_vsize=dataline1[1]['delta_vsize']+dataline2[1]['delta_vsize']
                new_rss=dataline2[1]['rss']
                new_delta_rss=dataline1[1]['delta_rss']+dataline2[1]['delta_rss']

                new_data.append((new_eventnumber,{'vsize':new_vsize,
                                                  'delta_vsize':new_delta_vsize,
                                                  'rss':new_rss,
                                                  'delta_rss':new_delta_rss}))
                i+=1
                if i==len(data): break

            data=new_data
            print('Collapsing: Done!')        
    except IndexError:
        pass

    return (data,len(data))

###########
# Create simple memory check graphs 
#
def createSimplMemGraphs(data,npoints,graph_num,legs,prevrev=""):
    colors = [2,4]    
    values = ["vsize", "rss"]
    
    prevRevName = "???"
    if not prevrev == "":
        (head, tail) = os.path.split(prevrev)
        prevRevName  = os.path.basename(tail)
        
    releaseNames = ["Previous (%s)" % prevRevName,_cmsver]        

    #fill the graphs
    graphs = []
    minim  = -1
    peak   = -1
    peaks  = []
    minims = []
    idx = 0
    for value in values:
        leg = legs[idx]

        graph = ROOT.TGraph(npoints)
        graph.SetTitle(value)
        graph.SetLineColor(colors[graph_num])
        graph.SetMarkerStyle(8)
        graph.SetMarkerSize(.7)
        graph.SetMarkerColor(1)
        graph.SetLineWidth(3)
        graph.GetXaxis().SetTitle("Event")
        graph.GetYaxis().SetTitleOffset(1.3)
        graph.GetYaxis().SetTitle("MB")

        total = 0
        point_counter=0
        rms   = 0
        first = True
        for event_number,vals_dict in data:
            if first:
                minim = vals_dict[value]
                peak  = vals_dict[value]
                first = False
            if vals_dict[value] > peak:
                peak = vals_dict[value]
            if vals_dict[value] < minim:
                minim = vals_dict[value]
            graph.SetPoint(point_counter, event_number, vals_dict[value])
            total += vals_dict[value]
            rms   += vals_dict[value] * vals_dict[value]
            point_counter+=1

        rms  = math.sqrt(rms / float(point_counter))
        mean = total / float(point_counter)
        last_event=data[-1][0]
        peaks.append(peak)
        minims.append(minim)
        graph.GetXaxis().SetRangeUser(0,last_event+1)
        leg.AddEntry(graph     , "%s release" % releaseNames[graph_num], "l")
        leg.AddEntry(graph     , "Mean: %s" % str(mean)                , "l")            
        leg.AddEntry(graph     , "RMS : %s" % str(rms)                 , "l")
        leg.AddEntry(graph     , "Peak: %s" % str(peak)                , "l")
        graphs.append(graph)
        idx += 1

    return (graphs[0] , last_event, peaks[0], minims[0], graphs[1], last_event, peaks[1], minims[1])

#############
# Produce the difference of two memory data sets  
#
def getMemDiff(data1,data2,npoints,last_event,orig_max_val,stepname,rss=False):
    data3 = []
    memtype = "vsize"
    if rss:
        memtype = "rss"

    graph=ROOT.TGraph(npoints)
    total = 0
    rms = 0
    evt_counter=0
    peak  = -1
    minum = -1
    first = True
    for x in range(len(data2)):
        try:
            (evtnum2,valdict2) = data2[x]
            (evtnum1,valdict1) = data1[x]
            if evtnum2 == evtnum1:
                diffMBytes = valdict2[memtype] - valdict1[memtype]

                if first:
                    peak  = diffMBytes
                    minum = diffMBytes
                    first = False
                if diffMBytes > peak:
                    peak  = diffMBytes
                if diffMBytes < minum:
                    minum = diffMBytes   
                graph.SetPoint(evt_counter,evtnum2,diffMBytes)
                evt_counter+=1
                total += diffMBytes
                rms += (diffMBytes * diffMBytes)
        except IndexError:
            pass
        except ValueError:
            pass
   
    mean  = total / evt_counter
    rms   = math.sqrt(rms / float(evt_counter))
        
    min_val  = minum
    max_val  = peak
    interval = int(max_val-min_val)
    
    min_val=min_val-interval*0.2
    max_val=max_val+interval*0.2
    interval=int(max_val-min_val)
    # Determine the max value to be something that makes the scale similar to what
    # the original graph had. Unless we can't seem the maximum value.

    new_max = min_val + orig_max_val
    if new_max < max_val:
        pass
    else :
        max_val = new_max
    
    graph.SetTitle("Change in %s memory usage for each event between revs for step %s" % (memtype,stepname))
    graph.SetName('MemoryUsagePerEvent')    
    graph.GetXaxis().SetTitle("Event Number")
    graph.GetYaxis().SetTitle("Change in memory usage between revs (MBs)")    
    graph.GetYaxis().SetTitleOffset(1.3)
    graph.SetLineColor(2)
    graph.SetMarkerStyle(8)
    graph.SetMarkerSize(.7)
    graph.SetMarkerColor(1)
    graph.SetLineWidth(3)    
    graph.GetXaxis().SetLimits(0,last_event)
    graph.GetYaxis().SetRangeUser(min_val,max_val)
    leg = ROOT.TLegend(0.5,0.7,0.89,0.89)
    leg.AddEntry(graph, "Mean: %s" % str(mean), "l")            
    leg.AddEntry(graph, "RMS : %s" % str(rms) , "l")
    leg.AddEntry(graph, "Peak: %s" % str(peak), "l")
    leg.AddEntry(graph, "Trough: %s" % str(minum)  , "l")                    

    return (graph,leg)

############
# Draw two memory graphs superimposed on one another
#
def drawMemGraphs(graph1,graph2,min_val,max_val,last_event,leg,memtype,stepname):
    graph_canvas=ROOT.TCanvas("%s_%s_canvas" % (memtype,stepname))
    graph_canvas.cd()
    graph1.GetYaxis().SetRangeUser(min_val,max_val)
    graph1.GetXaxis().SetRangeUser(0,last_event)        
    graph1.Draw("ALP")
    graph2.Draw("LP" )
    leg.Draw()    
    graph_canvas.ForceUpdate()
    graph_canvas.Flush()
    return graph_canvas

###########
# Draw the comparison graphs of two memory graphs
#
def drawMemChangeGraphs(graph,leg,memtype,stepname):
    graph_canvas=ROOT.TCanvas("%s_%s_change_canvas" % (memtype,stepname))
    graph_canvas.cd()
    graph.Draw("ALP" )
    leg.Draw()    
    graph_canvas.ForceUpdate()
    graph_canvas.Flush()
    return graph_canvas

def getMemOrigScale(fst_min,snd_min,fst_max,snd_max):
    minim = fst_min
    if snd_min < minim:
        minim = snd_min
    maxim = fst_max
    if snd_max > maxim:
        maxim = snd_max
    return (minim,maxim)
        
def cmpSimpMemReport(rootfilename,outdir,oldLogfile,newLogfile,startevt,batch=True,candle="",prevrev=""):
    if batch:
        setBatch()
    # the fundamental structure: the key is the evt number the value is a list containing
    # VSIZE deltaVSIZE RSS deltaRSS
    print("#####LOGFILES in cmsPErfRegress:")
    print(oldLogfile)
    print(newLogfile)
    try:
        info1 = getSimpleMemLogData(oldLogfile,startevt, candle)
        if len(info1) == 0:
            raise IndexError
    except IndexError:
        raise SimpMemParseErr(oldLogfile)
    except IOError:
        raise SimpMemParseErr(oldLogfile)        
    
    try:
        info2 = getSimpleMemLogData(newLogfile,startevt, candle)
        if len(info2) == 0:
            raise IndexError
    except IndexError:
        raise SimpMemParseErr(newLogfile)
    except IOError:
        raise SimpMemParseErr(newLogfile)        
    #print "DDDDD info1 %s"%info1 #Format is of the type:[('GEN,SIM', [(1, {'delta_vsize': 0.0, 'delta_rss': 0.0, 'vsize': 648.42600000000004, 'rss': 426.33199999999999}), (2,{...
    #print "DDDDD info2 %s"%info2
    canvases = []
    # skim the second entry when the event number is the same BUG!!!!!!!
    # i take elements in couples!

    candreg = re.compile("(.*)(?:\.log)")
    vsize_canvas = None
    rss_canvas = None
    i = 0
    firstRoot = True
    newrootfile = None
    #The following whileis confusing and not necessary info1 and info2 are supposed to only contain one set of simplememorycheck numbers
    #in a tuple as seen above ('GEN,SIM',[(event_number,{'delta_vsize': 0.0, etc
    #The data structure is questionable... but it is a remnant of past format of the log files I guess.
    #while ( i < len(info1) and i < len(info2)):
    #curinfo1 = info1[i]
    #curinfo2 = info2[i]
    (stepname1, data1) = info1[0]
    (stepname2, data2) = info2[0]

    #Again this is not necessary anymore...
    #if not stepname1 == stepname2:
    #print "WARNING: Could not compare %s step and %s step because they are not the same step" % (stepname1, stepname2)
    #        print " Searching for next occurence"
    #        x = 1
    #        if not (i + 1) > len(info1):
    #            found = False
    #            for info in info1[i + 1:]:
    #               (stepname,trash) = info
    #               if stepname == stepname2:
    #                    i += x
    #print " Next occurence found, skipping in-between steps"
    #                    assert i < len(info1)
    #                    found = True
    #                    break
    #                x += 1
    #            if found:
    #                continue
    #        print " No more occurences of this step, continuing" 
    #        i += 1
    #        continue

    #Not sure what this is for!
    #OK it was due to the bug of duplicated info in getSimpleMemLogData parsing!
    #No need!
    #(data1,npoints1) = getNpoints(data1)
    #(data2,npoints2) = getNpoints(data2)
    npoints1=len(data1)
    npoints2=len(data2)
    
    legs = []
    leg      = ROOT.TLegend(0.6,0.99,0.89,0.8)
    legs.append(leg)
    leg      = ROOT.TLegend(0.6,0.99,0.89,0.8)
    legs.append(leg)
    
    try:
        if len(data1) == 0:
            raise IndexError
        (vsize_graph1,
         vsize_lstevt1,
         vsize_peak1,
         vsize_minim1,
         rss_graph1,
         rss_lstevt1,
         rss_peak1,
         rss_minim1) = createSimplMemGraphs(data1,npoints1,0,legs,prevrev=prevrev)
        #candFilename = CandFname[candle]
        #outputdir = "%s_%s_SimpleMemReport" % (candFilename,stepname1)
        #outputdir = os.path.join(outdir,outputdir)
        #print "Graph1"
        #vsize_graph1.Print()
    except IndexError:
        raise SimpMemParseErr(oldLogfile)
    
    try:
        if len(data2) == 0:
            raise IndexError
        (vsize_graph2,
         vsize_lstevt2,
         vsize_peak2,
         vsize_minim2,
         rss_graph2,
         rss_lstevt2,
         rss_peak2,
         rss_minim2) = createSimplMemGraphs(data2,npoints2,1,legs,prevrev=prevrev)
        #candFilename = CandFname[candle]
        #outputdir = "%s_%s_SimpleMemReport" % (candFilename,stepname1)
        #outputdir = os.path.join(outdir,outputdir)
        #print "Graph2"
        #vsize_graph2.Print()#       os.path.join(outputdir,"vsize_graph2.png"))
    except IndexError:
        raise SimpMemParseErr(newLogfile)  
    
    
    (vsize_lstevt, vsize_max_val, vsize_min_val) = getTwoGraphLimits(vsize_lstevt1, vsize_peak1, vsize_lstevt2, vsize_peak2, vsize_minim1, vsize_minim2)
    (rss_lstevt  , rss_max_val  , rss_min_val)   = getTwoGraphLimits(rss_lstevt1  , rss_peak1, rss_lstevt2  , rss_peak2, rss_minim1,   rss_minim2)    
    
    (vsize_min,vsize_max) = getMemOrigScale(vsize_minim1,vsize_minim2,vsize_peak1,vsize_peak2)
    (rss_min  ,rss_max  ) = getMemOrigScale(rss_minim1,rss_minim2,rss_peak1,rss_peak2)
    
    setupSuperimpose(vsize_graph1,
                     vsize_graph2,
                     vsize_lstevt,
                     0,
                     reporttype = 1,
                     title = "%s_vsize" % stepname1)
    setupSuperimpose(rss_graph1  ,
                     rss_graph2  ,
                     rss_lstevt  ,
                     0,
                     reporttype = 1,
                     title = "%s_rss"  %  stepname2)
    
    (vsizePerfDiffgraph, vsizeleg) = getMemDiff(data1,data2,npoints2,vsize_lstevt, (vsize_max - vsize_min), stepname1, rss=False)
    (rssPerfDiffgraph, rssleg)     = getMemDiff(data1,data2,npoints2,rss_lstevt  , (rss_max - rss_min)    , stepname1, rss=True )        
    
    vsize_canvas = drawMemGraphs(vsize_graph1, vsize_graph2, vsize_min_val, vsize_max_val, vsize_lstevt, legs[0], "vsize", stepname1)
    rss_canvas   = drawMemGraphs(rss_graph1  , rss_graph2  , rss_min_val, rss_max_val, rss_lstevt, legs[1], "rss"  , stepname1)
    vsize_change_canvas = drawMemChangeGraphs(vsizePerfDiffgraph, vsizeleg, "vsize", stepname1)         
    rss_change_canvas   = drawMemChangeGraphs(rssPerfDiffgraph  , rssleg  , "rss"  , stepname1)         
    
    if batch:
        
        
        logcandle = ""
        candname  = ""            
        found = candreg.search(os.path.basename(newLogfile))
        
        if found:
            logcandle = found.groups()[0]
            
        if   candle in CandFname:
            candFilename = CandFname[candle]
        elif logcandle in CandFname:
            candFilename = CandFname[logcandle]
        else:
            print("%s is an unknown candle!"%candle)
            candFilename = "Unknown-candle"
            
        outputdir = "%s_%s_SimpleMemReport" % (candFilename,stepname1)
        outputdir = os.path.join(outdir,outputdir)
        
        if not os.path.exists(outputdir):
            os.mkdir(outputdir)

            #print the graphs as files :)

        newrootfile = createROOT(outputdir,rootfilename)                
        
        vsize_canvas.Print(       os.path.join(outputdir,"vsize_graphs.png"), "png")
        rss_canvas.Print(         os.path.join(outputdir,"rss_graphs.png"  ), "png")
        vsize_change_canvas.Print(os.path.join(outputdir,"vsize_change.png"), "png")
        rss_change_canvas.Print(  os.path.join(outputdir,"rss_change.png"  ), "png")
        # write it on file
        map(lambda x: x.Write(), [vsize_graph1,vsize_graph2, rss_graph1, rss_graph2, vsizePerfDiffgraph, rssPerfDiffgraph])
        map(lambda x: x.Write(), [vsize_canvas,rss_canvas,vsize_change_canvas,rss_change_canvas])
        newrootfile.Close()
    else:
        # we have to do this if we are running the application standalone
        # For some reason it will not draw some graphs at all if there is more than
        # one step.
        # If we wait between iterations of this loop the graphs will be drawn correctly.
        # Perhaps a graphics buffer problem with ROOT?
        # Perhaps a garbage collection problem in python? (Doubt it)
        canvases.append(rss_canvas)
        canvases.append(vsize_canvas)
        canvases.append(vsize_change_canvas)
        canvases.append(rss_change_canvas)            
        time.sleep(5.0)

    #Eliminated the while loop!    
    #i += 1
        
    #
    # Create a one dimensional function and draw it
    #

    if batch:
        pass
    else:
        if len(canvases) > 0:
            while reduce(lambda x,y: x or y ,canvases):
                time.sleep(2.5)
    return 0            
        

def cmpTimingReport(rootfilename,outdir,oldLogfile,newLogfile,secsperbin,batch=True,prevrev=""):
    if batch:
        setBatch()
    
    data1 = getTimingLogData(oldLogfile)
    data2 = getTimingLogData(newLogfile)        
        
    try:
        (min_val1,max_val1,nbins1,npoints1,last_event1) = getLimits(data1,secsperbin)
    except IndexError as detail:
        raise TimingParseErr(oldLogfile)
    
    try:
        (min_val2,max_val2,nbins2,npoints2,last_event2) = getLimits(data2,secsperbin)
    except IndexError as detail:
        raise TimingParseErr(newLogfile)

    hsStack  = ROOT.THStack("hsStack","Histogram Comparison")
    leg      = ROOT.TLegend(0.6,0.99,0.89,0.8)
    histoleg = ROOT.TLegend(0.5,0.8,0.89,0.89)    

    #
    
    (graph1,histo1,mean1) = newGraphAndHisto(histoleg,leg,npoints1,nbins1,min_val1,max_val1,data1,0,prevrev)
    hsStack.Add(histo1)
    (graph2,histo2,mean2) = newGraphAndHisto(histoleg,leg,npoints2,nbins2,min_val2,max_val2,data2,1,prevrev)
    hsStack.Add(histo2)

    (biggestLastEvt,biggestMaxval, trashthis) = getTwoGraphLimits(last_event1,max_val1,last_event2,max_val2,min_val1,min_val2)
    
    (changegraph,chgleg) = getTimingDiff(data1,data2,npoints2,biggestLastEvt,biggestMaxval)
    setupSuperimpose(graph1,graph2,biggestLastEvt,biggestMaxval)
    avg_line1 = getMeanLines(mean1,last_event1,0)
    avg_line2 = getMeanLines(mean2,last_event2,1)

    #
    # Create a one dimensional function and draw it
    #
    histo1.GetXaxis().SetTitle("s")            
    graph_canvas   = drawGraphs(graph1,graph2,avg_line1,avg_line2,leg)
    changes_canvas = drawChanges(changegraph,chgleg)
    histo_canvas   = drawHistos(hsStack,histoleg)
    
    newrootfile = None
    if batch:

        newrootfile = createROOT(outdir,rootfilename)      

        cput = ROOT.TTree()
        #  array(typecode, initializer)
        #  typecode is i for int, f for float etc.
        tot_a1 = array( "f", [ 0 ] )
        tot_a2 = array( "f", [ 0 ] )

        tot_a1[0] = mean1
        tot_a2[0] = mean2

        cput.Branch("total1",tot_a1,"total1/F")
        cput.Branch("total2",tot_a2,"total2/F")
        cput.Fill()
        cput.Write("cpu_time_tuple",ROOT.TObject.kOverwrite)
        
        names = ["graphs.png","changes.png","histos.png"]
        
        graph_canvas.Print(  os.path.join(outdir,names[0]),"png")
        changes_canvas.Print(os.path.join(outdir,names[1]),"png")
        histo_canvas.Print(  os.path.join(outdir,names[2]),"png")
        
        map(lambda x:x.Write(),[graph1,graph2,changegraph,hsStack,histo1,histo2])
        
        graph_canvas.Write()    # to file
        changes_canvas.Write()
        histo_canvas.Write() 
        newrootfile.Close()   

        return names
    else:
        
        while graph_canvas or histo_canvas or changes_canvas:
            time.sleep(2.5)
        return 0
    
def rmtree(path):
    try:
        #os.remove(path)
        #Brute force solution:
        RemoveCmd="rm -Rf %s"%path
        os.system(RemoveCmd)
    except OSError as detail:
        if detail.errno == 39:
            try:
                gen = os.walk(path)
                nodes    = next(gen)
                nodes[0] = par
                nodes[1] = dirs
                nodes[2] = files
                for f in files:
                    os.remove(os.path.join(path,f))
                for d in dirs:
                    rmtree(os.path.join(path,d))
            except OSError as detail:
                print(detail)
            except IOError as detail:
                print(detail)
            os.remove(path)

def perfreport(perftype,file1,file2,outdir,IgProfMemopt=""):
    src = ""
    try:
        src = os.environ["CMSSW_SEARCH_PATH"]
    except KeyError as detail:
        print("ERROR: scramv1 environment could not be located", detail) 

    vars = src.split(":")
    loc  = vars[0]

    proftype = ""
    if   perftype == 0: # EdmSize
        proftype = "-fe"
    elif perftype == 1: # IgProf
        proftype = "-fi"
    else:               # Callgrind
        proftype = "-ff"

    cmssw_release_base = ""
    cmssw_data = ""
    try:
        cmssw_release_base = os.environ['CMSSW_RELEASE_BASE']
        cmssw_data = os.environ['CMSSW_DATA_PATH']
    except KeyError as detail:
        raise PerfReportErr

    xmlfile = os.path.join(cmssw_release_base,"src","Validation","Performance","doc","regress.xml")

    prRoot = "/afs/cern.ch/user/g/gbenelli/public/PerfReport2/2.0.1"

    # this might be useful at some point
    #cd %s ; eval `scramv1 runtime -csh`  ; source $CMSSW_DATA_PATH/perfreport/2.0.0/etc/profile.d/init.csh; cd - ; %s\"" % (loc,perfcmd)

    # Before adding Danilo's 2.1 we did this
    #perfcmd = "perfreport -tmp %s -i %s -r %s -o %s" % (proftype,file2,file1,outdir)    
    #cmd = "tcsh -c \"source %s/perfreport/2.0.0/etc/profile.d/init.csh; cd - ; %s\"" % (cmssw_data,perfcmd)

    # now we do

    tmpdir  = tmp.mkdtemp(prefix=os.path.join(outdir,"tmp"))

    perfcmd = "%s %s %s -c %s -t%s -i %s -r %s -o %s" % (os.path.join(prRoot,"bin","perfreport"),proftype,IgProfMemopt,xmlfile,tmpdir,file2,file1,outdir)            
    cmd     = "tcsh -c \"cd %s ; eval `scramv1 runtime -csh` ; cd - ;source %s/etc/profile.d/init.csh ; %s\"" % (loc,prRoot,perfcmd)
    #Obsolete popen4-> subprocess.Popen
    #process  = os.popen(cmd)
    process = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
    exitstat = process.wait()
    cmdout   = process.stdout.read()
    exitstat = process.returncode

    try:
        rmtree(tmpdir)        #Brute force solution rm -RF tmpdir done in rmtree()
        #os.rmdir(tmpdir)
    except IOError as detail:
        print("WARNING: Could not remove dir because IO%s" % detail)                
    except OSError as detail:
        print("WARNING: Could not remove dir because %s" % detail)                

    if True:
        print(cmd)
        print(cmdout)

    if not exitstat == None:
        sig     = exitstat >> 16    # Get the top 16 bits
        xstatus = exitstat & 0xffff # Mask out all bits except the bottom 16
        raise PerfReportErr("ERROR: PerfReport returned a non-zero exit status (%s, SIG_INT = %s) run %s. Dump follows: \n%s" % (perfcmd,xstatus,sig,cmdout))
    
    
def cmpEdmSizeReport(outdir,file1,file2):
    perfreport(0,file1,file2,outdir)

def ungzip(inf,outh):
    gzf = gzip.open(inf,"r")
    print("ungzipping")
    for char in gzf:
        os.write(outh,char)
    os.close(outh)
    print("finish ungzipping")
    gzf.close()

def ungzip2(inf,out):
    os.system("gzip -c -d %s > %s" % (inf,out)) 

def cmpIgProfReport(outdir,file1,file2,IgProfMemOpt=""):
    (tfile1, tfile2) = ("", "")
    try:
        # don't make temp files in /tmp because it's never bloody big enough
        (th1, tfile1) = tmp.mkstemp(prefix=os.path.join(outdir,"igprofRegressRep."))
        (th2, tfile2) = tmp.mkstemp(prefix=os.path.join(outdir,"igprofRegressRep."))
        os.close(th1)
        os.close(th2)
        os.remove(tfile1)
        os.remove(tfile2)
        ungzip2(file1,tfile1)
        ungzip2(file2,tfile2)        

        perfreport(1,tfile1,tfile2,outdir,IgProfMemOpt)

        os.remove(tfile1)
        os.remove(tfile2)
    except OSError as detail:
        raise PerfReportErr("WARNING: The OS returned the following error when comparing %s and %s\n%s" % (file1,file2,str(detail)))
        if os.path.exists(tfile1):
            os.remove(tfile1)
        if os.path.exists(tfile2):
            os.remove(tfile2)
    except IOError as detail:
        raise PerfReportErr("IOError: When comparing %s and %s using temporary files %s and %s. Error message:\n%s" % (file1,file2,tfile1,tfile2,str(detail)))
        if os.path.exists(tfile1):
            os.remove(tfile1)
        if os.path.exists(tfile2):
            os.remove(tfile2)        


def cmpCallgrindReport(outdir,file1,file2):
    perfreport(2,file1,file2,outdir)

def _main():
    outdir = os.getcwd()
    
    (file1,file2,secsperbin,reporttype,IgProfMemOptions)  = getParameters()

    try:
        if   reporttype == "timing":
            rootfilename = "timingrep-regression.root"
            cmpTimingReport(rootfilename ,outdir,file1,file2,secsperbin,True)
        elif reporttype == "simplememory":
            rootfilename = "simpmem-regression.root"
            cmpSimpMemReport(rootfilename,outdir,file1,file2,secsperbin,True)
        elif reporttype == "edmsize":
            cmpEdmSizeReport(outdir,file1,file2)
        elif reporttype == "callgrind":
            cmpCallgrindReport(outdir,file1,file2)
        elif reporttype == "igprof":
            cmpIgProfReport(outdir,file1,file2,IgProfMemOptions)            
    except TimingParseErr as detail:
        print("WARNING: Could not parse data from Timing report file %s; not performing regression" % detail.message)
    except SimpMemParseErr as detail:
        print("WARNING: Could not parse data from Memory report file %s; not performing regression" % detail.message)
    except PerfReportErr as detail:
        print("WARNING: Could not parse data from Edm file %s; not performing regression" % detail.message)
    except IOError as detail:
        print(detail)
    except OSError as detail:
        print(detail)

if __name__ == "__main__":
    _main()