Reference Manual

Go to the documentation of this file.
00001 #! /usr/bin/env python
00002 
00003 r'''
00004 cmsRelvalreport.py: a script to run performance tests and produce reports in a automated way.
00005 '''
00006 
00007 import glob,re #Used for IgProf Analyse work with IgProf Mem profile dumps
00008 # Configuration parameters:#############################################################
00009 
00010 # Perfreport 3 and 2 coordinates:
00011 PR3_BASE='/afs/cern.ch/user/d/dpiparo/w0/perfreport3installation/'
00012 PR3=PR3_BASE+'/bin/perfreport'# executable
00013 PERFREPORT3_PATH=PR3_BASE+'/share/perfreport' #path to xmls
00014 #PR3_PRODUCER_PLUGIN=PR3_BASE+'/lib/libcmssw_by_producer.so' #plugin for fpage
00015 PR3_PRODUCER_PLUGIN='/afs/cern.ch/user/d/dpiparo/w0/pr3/perfreport/plugins/cmssw_by_producer/libcmssw_by_producer.so'
00016 
00017 #PR2_BASE='/afs/cern.ch/user/d/dpiparo/w0/perfreport2.1installation/'
00018 PR2_BASE='/afs/cern.ch/user/g/gbenelli/public/PerfReport2/2.0.1/'
00019 PR2='%s' %(PR2_BASE+'/bin/perfreport')# executable
00020 PERFREPORT2_PATH=PR2_BASE+'/share/perfreport' #path to xmls
00021 
00022 import os
00023 cmssw_base=os.environ["CMSSW_BASE"]
00024 cmssw_release_base=os.environ["CMSSW_RELEASE_BASE"]
00025 pyrelvallocal=cmssw_base+"/src/Configuration/PyReleaseValidation"
00026 valperf=cmssw_base+"/src/Validation/Performance"
00027 #Set the path depending on the presence of a locally checked out version of PyReleaseValidation
00028 if os.path.exists(pyrelvallocal):
00029     RELEASE='CMSSW_BASE'
00030     print "Using LOCAL version of Configuration/PyReleaseValidation instead of the RELEASE version"
00031 elif not os.path.exists(pyrelvallocal):
00032     RELEASE='CMSSW_RELEASE_BASE'
00033     
00034 #Eliminating the full paths to all the scripts used they are assumed to be in the release by default, being in the scripts/ directory of their package    
00035 # Valgrind Memcheck Parser coordinates:
00036 #VMPARSER='%s/src/Utilities/ReleaseScripts/scripts/valgrindMemcheckParser.pl' %os.environ['CMSSW_RELEASE_BASE']#This has to always point to 'CMSSW_RELEASE_BASE'
00037 VMPARSER='valgrindMemcheckParser.pl'
00038 
00039 #Not a script... need to keep the full path to the release
00040 # Valgrind Memcheck Parser output style file coordinates:
00041 VMPARSERSTYLE='%s/src/Utilities/ReleaseScripts/data/valgrindMemcheckParser.css' %os.environ['CMSSW_RELEASE_BASE']#This has to always point to 'CMSSW_RELEASE_BASE'
00042 
00043 # IgProf_Analysis coordinates:
00044 #IGPROFANALYS='%s/src/Validation/Performance/scripts/cmsIgProf_Analysis.py'%os.environ[RELEASE]
00045 IGPROFANALYS='cmsIgProf_Analysis.py'
00046 
00047 # Timereport parser
00048 #TIMEREPORTPARSER='%s/src/Validation/Performance/scripts/cmsTimeReport.pl'%os.environ[RELEASE]
00049 TIMEREPORTPARSER='cmsTimeReport.pl'
00050 
00051 # Simple memory parser
00052 #SIMPLEMEMPARSER='%s/src/Validation/Performance/scripts/cmsSimplememchecker_parser.py' %os.environ[RELEASE]
00053 SIMPLEMEMPARSER='cmsSimplememchecker_parser.py'
00054 
00055 # Timing Parser
00056 #TIMINGPARSER='%s/src/Validation/Performance/scripts/cmsTiming_parser.py' %os.environ[RELEASE]
00057 TIMINGPARSER='cmsTiming_parser.py'
00058 
00059 # makeSkimDriver
00060 MAKESKIMDRIVERDIR='%s/src/Configuration/EventContent/test' %os.environ[RELEASE]
00061 MAKESKIMDRIVER='%s/makeSkimDriver.py'%MAKESKIMDRIVERDIR
00062 
00063 ########################################################################################
00064 
00065 
00066 # Library to include to run valgrind fce
00067 VFCE_LIB='/afs/cern.ch/user/m/moserro/public/vgfcelib' 
00068 PERL5_LIB='/afs/cern.ch/user/d/dpiparo/w0/PERLlibs/5.8.0'
00069 
00070 
00071 
00072 # the profilers that use the stout of the app..
00073 STDOUTPROFILERS=['Memcheck_Valgrind',
00074                  'Timereport_Parser',
00075                  'Timing_Parser',
00076                  'SimpleMem_Parser']
00077 # Profilers list
00078 PROFILERS=['ValgrindFCE',
00079            'IgProf_perf',
00080            'IgProf_mem',
00081            'Edm_Size']+STDOUTPROFILERS
00082                             
00083 
00084 # name of the executable to benchmark. It can be different from cmsRun in future           
00085 EXECUTABLE='cmsRun'
00086 
00087 # Command execution and debug switches
00088 EXEC=True
00089 DEBUG=True
00090 
00091 #Handy dictionaries to handle the mapping between IgProf Profiles and counters:
00092 IgProfCounters={'IgProfPerf':['PERF_TICKS'],
00093                 'IgProfMem':['MEM_TOTAL','MEM_LIVE','MEM_MAX']
00094                 }
00095 IgProfProfiles={'PERF_TICKS':'IgProfPerf',
00096                'MEM_TOTAL':'IgProfMem',
00097                'MEM_LIVE':'IgProfMem',
00098                'MEM_MAX':'IgProfMem'
00099                }
00100 import time   
00101 import optparse 
00102 import sys
00103 
00104 
00105 #######################################################################
00106 def red(string):
00107     return '%s%s%s' %('\033[1;31m',string,'\033[1;0m')    
00108 def green(string):
00109     return '%s%s%s' %('\033[1;32m',string,'\033[1;0m') 
00110 def yellow(string):
00111     return '%s%s%s' %('\033[1;33m',string,'\033[1;0m')     
00112 #######################################################################
00113 
00114 def clean_name(name):
00115     '''
00116     Trivially removes an underscore if present as last char of a string
00117     '''
00118     i=-1
00119     is_dirty=True
00120     while(is_dirty):
00121         if name[i]=='_':
00122             name=name[:-1]
00123         else:
00124             return name
00125         i-=1
00126 
00127 #######################################################################
00128 
00129 def execute(command):
00130         '''
00131         It executes command if the EXEC switch is True. 
00132         Catches exitcodes different from 0.
00133         '''
00134         logger('%s %s ' %(green('[execute]'),command))
00135         if EXEC:
00136             exit_code=os.system(command)
00137             if exit_code!=0:
00138                 logger(red('*** Seems like "%s" encountered problems.' %command))
00139             return exit_code
00140         else:
00141             return 0
00142             
00143 #######################################################################            
00144             
00145 def logger(message,level=0):
00146     '''
00147     level=0 output, level 1 debug.
00148     '''                  
00149     message='%s %s' %(yellow('[RelValreport]'),message)
00150     
00151     sys.stdout.flush()
00152     
00153     if level==0:
00154         print message
00155     if level==1 and DEBUG:
00156         print message    
00157     
00158     sys.stdout.flush()
00159 
00160 #######################################################################
00161 
00162 class Candles_file:
00163     '''
00164     Class to read the trivial ASCII file containing the candles
00165     '''
00166     def __init__(self, filename):
00167         
00168         self.commands_profilers_meta_list=[]    
00169     
00170         candlesfile=open(filename,'r')
00171         
00172         if filename[-3:]=='xml':
00173             command=''
00174             profiler=''
00175             meta=''
00176             db_meta=''
00177             reuse=False    
00178             
00179             from xml.dom import minidom
00180             
00181             # parse the config
00182             xmldoc = minidom.parse(filename)
00183             
00184             # get the candles
00185             candles_list = xmldoc.getElementsByTagName('candle')
00186             
00187             # a list of dictionaries to store the info
00188             candles_dict_list=[]
00189             
00190             for candle in candles_list:
00191                 info_dict={}
00192                 for child in candle.childNodes:# iteration over candle node children
00193                     if not child.__dict__.has_key('nodeName'):# if just a text node skip!
00194                         #print 'CONTINUE!'
00195                         continue
00196                     # We pick the info from the node
00197                     tag_name=child.tagName
00198                     #print 'Manipulating a %s ...'%tag_name
00199                     data=child.firstChild.data
00200                     #print 'Found the data: %s !' %data
00201                     # and we put it in the dictionary
00202                     info_dict[tag_name]=data
00203                 # to store it in a list
00204                 candles_dict_list.append(info_dict)
00205             
00206             # and now process what was parsed
00207                         
00208             for candle_dict in candles_dict_list:
00209                 # compulsory params!!
00210                 command=candle_dict['command']
00211                 profiler=candle_dict['profiler']
00212                 meta=candle_dict['meta']
00213                 # other params
00214                 try:
00215                     db_meta=candle_dict['db_meta']
00216                 except:
00217                     db_meta=None
00218                 try:
00219                     reuse=candle_dict['reuse']
00220                 except:
00221                     reuse=False    
00222                             
00223                 self.commands_profilers_meta_list.append([command,profiler,meta,reuse,db_meta])
00224         
00225         # The file is a plain ASCII
00226         else:
00227             for candle in candlesfile.readlines():
00228                 # Some parsing of the file
00229                 if candle[0]!='#' and candle.strip(' \n\t')!='': # if not a comment or an empty line
00230                     if candle[-1]=='\n': #remove trail \n if it's there
00231                         candle=candle[:-1] 
00232                     splitted_candle=candle.split('@@@') #separate
00233                     
00234                     # compulsory fields
00235                     command=splitted_candle[0]
00236                     profiler=splitted_candle[1].strip(' \t')
00237                     meta=splitted_candle[2].strip(' \t')        
00238                     info=[command,profiler,meta]
00239                     
00240                     # FIXME: AN .ini or xml config??
00241                     # do we have something more?
00242                     len_splitted_candle=len(splitted_candle)
00243                     reuse=False
00244                     if len_splitted_candle>3:
00245                         # is it a reuse statement?
00246                         if 'reuse' in splitted_candle[3]:
00247                             reuse=True
00248                         info.append(reuse)
00249                     else:
00250                         info.append(reuse)               
00251                     
00252                     # we have one more field or we hadn't a reuse in the last one    
00253                     if len_splitted_candle>4 or (len_splitted_candle>3 and not reuse):
00254                         cmssw_scram_version_string=splitted_candle[-1].strip(' \t')
00255                         info.append(cmssw_scram_version_string)
00256                     else:
00257                         info.append(None)
00258     
00259                         
00260                     self.commands_profilers_meta_list.append(info)
00261                     
00262     #----------------------------------------------------------------------
00263         
00264     def get_commands_profilers_meta_list(self):
00265         return self.commands_profilers_meta_list
00266             
00267 #######################################################################
00268 
00269 class Profile:
00270     '''
00271     Class that represents the procedure of performance report creation
00272     '''
00273     def __init__(self,command,profiler,profile_name):
00274         self.command=command
00275         self.profile_name=profile_name
00276         self.profiler=profiler
00277     
00278     #------------------------------------------------------------------
00279     # edit here if more profilers added
00280     def make_profile(self):
00281         '''
00282         Launch the right function according to the profiler name.
00283         '''  
00284         if self.profiler=='ValgrindFCE':
00285             return self._profile_valgrindfce()
00286         elif self.profiler.find('IgProf')!=-1:
00287             return self._profile_igprof()    
00288         elif self.profiler.find('Edm_Size')!=-1:
00289             return self._profile_edmsize()
00290         elif self.profiler=='Memcheck_Valgrind':
00291             return self._profile_Memcheck_Valgrind()
00292         elif self.profiler=='Timereport_Parser':
00293             return self._profile_Timereport_Parser()
00294         elif self.profiler=='Timing_Parser':
00295             return self._profile_Timing_Parser()        
00296         elif self.profiler=='SimpleMem_Parser':
00297             return self._profile_SimpleMem_Parser()
00298         elif self.profiler=='':
00299             return self._profile_None()
00300         elif self.profiler=='None': #adding this for the case of candle ASCII file non-profiling commands
00301             return self._profile_None()
00302         else:
00303             raise('No %s profiler found!' %self.profiler)
00304     #------------------------------------------------------------------
00305     def _profile_valgrindfce(self):
00306         '''
00307         Valgrind profile launcher.
00308         '''
00309         # ValgrindFCE needs a special library to run
00310         os.environ["VALGRIND_LIB"]=VFCE_LIB
00311         
00312         profiler_line=''
00313         valgrind_options= 'time valgrind '+\
00314                           '--tool=callgrind '+\
00315                           '--fce=%s ' %(self.profile_name)
00316         
00317         # If we are using cmsDriver we should use the prefix switch        
00318         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00319             profiler_line='%s --prefix "%s"' %(self.command,valgrind_options)
00320                             
00321         else:                          
00322             profiler_line='%s %s' %(valgrind_options,self.command)
00323                         #'--trace-children=yes '+\
00324         
00325         return execute(profiler_line)
00326     
00327     #------------------------------------------------------------------
00328     def _profile_igprof(self): 
00329         '''
00330         IgProf profiler launcher.
00331         '''
00332         profiler_line=''
00333         
00334         igprof_options='igprof -d -t %s ' \
00335                     %EXECUTABLE # IgProf profile not general only for CMSRUN!
00336         
00337         # To handle Igprof memory and performance profiler in one function
00338         if self.profiler=='IgProf_perf':
00339             igprof_options+='-pp '
00340         elif self.profiler=='IgProf_mem':
00341             igprof_options+='-mp '
00342         else:
00343             raise ('Unknown IgProf flavour: %s !'%self.profiler)
00344         igprof_options+='-z -o %s' %(self.profile_name)
00345         
00346         # If we are using cmsDriver we should use the prefix switch 
00347         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00348             profiler_line='%s --prefix "%s"' %(self.command,igprof_options) 
00349         else:
00350             profiler_line='%s %s' %(igprof_options, self.command)  
00351             
00352         return execute(profiler_line)
00353     
00354     #------------------------------------------------------------------
00355     
00356     def _profile_edmsize(self):
00357         '''
00358         Launch edm size profiler
00359         '''
00360         # In this case we replace the name to be clear
00361         input_rootfile=self.command
00362         
00363         # Skim the content if requested!
00364         if '.' in self.profiler:
00365             
00366             clean_profiler_name,options=self.profiler.split('.')
00367             content,nevts=options.split(',')
00368             outfilename='%s_%s.root'%(os.path.basename(self.command)[:-6],content)
00369             oldpypath=os.environ['PYTHONPATH']
00370             os.environ['PYTHONPATH']+=':%s' %MAKESKIMDRIVERDIR
00371             execute('%s -i %s -o %s --outputcommands %s -n %s' %(MAKESKIMDRIVER,
00372                                                                  self.command,
00373                                                                  outfilename,
00374                                                                  content,
00375                                                                  nevts)) 
00376             os.environ['PYTHONPATH']=oldpypath
00377             #execute('rm %s' %outfilename)
00378             self.command=outfilename
00379             self.profiler=clean_profiler_name
00380                                                                                                         
00381         
00382         profiler_line='edmEventSize -o %s -d %s'\
00383                             %(self.profile_name,self.command)
00384         
00385         return execute(profiler_line)
00386    
00387    #------------------------------------------------------------------
00388    
00389     def _profile_Memcheck_Valgrind(self):
00390         '''
00391         Valgrind Memcheck profile launcher
00392         '''
00393         profiler_line=''
00394         #Adding cms suppression of useless messages (cmsvgsupp)
00395         #Removing leak-checking (done with igprof)
00396         #'--leak-check=no '+\ (no is the default)
00397         #'--show-reachable=yes '+\
00398         #'--track-fds=yes '
00399         #Adding xml logging
00400         xmlFileName = self.profile_name.replace(",","-")[:-4] + ".xml"
00401         valgrind_options='time valgrind --track-origins=yes '+\
00402                                '--tool=memcheck `cmsvgsupp` '+\
00403                                '--num-callers=20 '+\
00404                                '--xml=yes '+\
00405                                '--xml-file=%s '%xmlFileName
00406         
00407         # If we are using cmsDriver we should use the prefix switch        
00408         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00409             #Replacing 2>&1 |tee with >& in the shell command to preserve the return code significance:
00410             # using tee return would be 0 even if the command failed before the pipe:
00411             profiler_line='%s --prefix "%s" >& %s' %(self.command,valgrind_options,self.profile_name)
00412                             
00413         else:                          
00414             profiler_line='%s %s >& %s' %(valgrind_options,self.command,self.profile_name)
00415                         #'--trace-children=yes '+\
00416         exec_status = execute(profiler_line)
00417 
00418         # compact the xml by removing the Leak_* errors
00419         if not exec_status:
00420             newFileName = xmlFileName.replace('valgrind.xml', 'vlgd.xml')
00421             compactCmd = 'xsltproc --output %s %s/test/filterOutValgrindLeakErrors.xsl %s' %(newFileName, valperf, xmlFileName)
00422             execute(compactCmd)
00423         
00424         return exec_status
00425     #-------------------------------------------------------------------
00426     
00427     def _profile_Timereport_Parser(self):
00428         return self._save_output()
00429         
00430     #-------------------------------------------------------------------
00431     
00432     def _profile_SimpleMem_Parser(self):
00433         return self._save_output()
00434 
00435     #-------------------------------------------------------------------
00436     
00437     def _profile_Timing_Parser(self):
00438         return self._save_output()     
00439     
00440     #-------------------------------------------------------------------
00441         
00442     def _save_output(self):
00443         '''
00444         Save the output of cmsRun on a file!
00445         '''               
00446 #         # a first maquillage about the profilename:
00447 #         if self.profile_name[-4:]!='.log':
00448 #             self.profile_name+='.log'
00449         #Replacing 2>&1 |tee with >& in the shell command to preserve the return code significance:
00450         # using tee return would be 0 even if the command failed before the pipe:
00451         profiler_line='%s  >& %s' %(self.command,self.profile_name)
00452         return execute(profiler_line)    
00453 
00454     #-------------------------------------------------------------------                    
00455     
00456     def _profile_None(self):
00457         '''
00458         Just Run the command!
00459         '''
00460         return execute(self.command)
00461     
00462     #-------------------------------------------------------------------
00463     
00464     def make_report(self,
00465                     fill_db=False,
00466                     db_name=None,
00467                     tmp_dir=None,
00468                     outdir=None,
00469                     IgProf_option=None,
00470                     metastring=None):
00471         '''
00472         Make a performance report with CMSSW scripts for CMSSW internal profiling (Timing/SimpleMemoryCheck) and Memcheck, PR2 for edmEventSize and Callgrind (NOTE PR2 is not supported anymore and is not currently in the CMSSW external, running froma privat AFS!), igprof-analyse for all IgProf profiling.
00473         '''
00474 
00475         if outdir==None or outdir==self.profile_name:
00476             outdir=self.profile_name+'_outdir'
00477             
00478         #Create the directory where the report will be stored:
00479         if not os.path.exists(outdir) and not fill_db and not IgProf_option:
00480             #Added an IgProf condition to avoid the creation of a directory that we will not need anymore, since we will put all info in the filenames
00481             execute('mkdir %s' %outdir)
00482         
00483         if fill_db:
00484             db_option='-a'
00485             if not os.path.exists(db_name):
00486                 db_option='-A'
00487         
00488         # temp in the local dir for PR
00489         tmp_switch=''    
00490         if tmp_dir!='':
00491             execute('mkdir %s' %tmp_dir)
00492             tmp_switch=' -t %s' %tmp_dir
00493 
00494         #Handle the various profilers:
00495         
00496         ##################################################################### 
00497         
00498         # Profiler is ValgrindFCE:
00499         if self.profiler=='ValgrindFCE':
00500             perfreport_command=''
00501             # Switch for filling the db
00502             if not fill_db:
00503                 os.environ["PERFREPORT_PATH"]='%s/' %PERFREPORT2_PATH
00504                 perfreport_command='%s %s -ff -i %s -o %s' %(PR2,
00505                                                              tmp_switch,
00506                                                              self.profile_name,
00507                                                              outdir)
00508             else:
00509                 os.environ["PERFREPORT_PATH"]='%s/' %PERFREPORT3_PATH
00510                 perfreport_command='%s %s -n5000 -u%s  -ff  -m \'scram_cmssw_version_string,%s\' -i %s %s -o %s' \
00511                                                     %(PR3,
00512                                                       tmp_switch,
00513                                                       PR3_PRODUCER_PLUGIN,
00514                                                       metastring,
00515                                                       self.profile_name,
00516                                                       db_option,
00517                                                       db_name)
00518             return execute(perfreport_command)
00519         
00520         #####################################################################            
00521             
00522         # Profiler is IgProf:
00523         if self.profiler.find('IgProf')!=-1:
00524             #First the case of IgProf PERF and MEM reporting:
00525             if not 'ANALYSE' in IgProf_option:
00526                 #Switch to the use of igprof-analyse instead of PerfReport!
00527                 #Will use the ANALYSE case for regressions between early event dumps and late event dumps of the profiles
00528                 #Following Andreas suggestion, add the number of events for the EndOfJob report
00529                 NumberOfEvents=self.command.split()[3] #FIXME: this is quite hardcoded... but should be stable...
00530                 sqlite_outputfile=self.profile_name.split(".")[0].replace(IgProfProfiles[IgProf_option[0]],IgProf_option[0])+'___'+NumberOfEvents+'_EndOfJob.sql3'
00531                 logger("Executing the report of the IgProf end of job profile")
00532                 exit=execute('igprof-analyse --sqlite -d -v -g -r %s %s | sqlite3 %s'%(IgProf_option[0],self.profile_name,sqlite_outputfile)) 
00533                 return exit
00534             #Then the "ANALYSE" case that we want to use to add to the same directories (Perf, MemTotal, MemLive)
00535             #also some other analyses and in particular:
00536             #1-the first 7 lines of the ASCII analysis of the IgProf profile dumps (total of the counters)
00537             #2-the dump at different event numbers,
00538             #3-the diff between the first and last dump,
00539             #4-the counters grouped by library using regexp at the last dump:
00540             else: #We use IgProf Analysis
00541                 #Set the IgProfCounter from the ANALYSE.MEM_TOT style IgProf_option
00542                 #print IgProf_option
00543                 IgProfCounter=IgProf_option[1]
00544                 #Add here the handling of the new IgProf.N.gz files so that they will get preserved and not overwritten:
00545                 logger("Looking for IgProf intermediate event profile dumps")
00546                 #Check if there are IgProf.N.gz dump files:
00547                 IgProfDumps=glob.glob("IgProf.*.gz")
00548                 #in case there are none check if they have already been mv'd to another name to avoid overwriting
00549                 #(MEM_LIVE usually re-uses MEM_TOTAL, so the IgProf.N.gz files will already have a MemTotal name...)
00550                 if not IgProfDumps:
00551                     localFiles=os.listdir('.')
00552                     IgProfDumpProfilesPrevious=re.compile(r"\w+.\d+.gz")
00553                     IgProfDumps=filter(lambda x: IgProfDumpProfilesPrevious.search(x),localFiles)
00554                 #Now if there are dumps execute the following analyses:
00555                 if IgProfDumps:
00556                     IgProfDumps.sort()
00557                     logger("Found the following IgProf intermediate event profile dumps:")
00558                     logger(IgProfDumps)
00559                     FirstDumpEvent=9999999
00560                     LastDumpEvent=0
00561                     exit=0
00562                     for dump in IgProfDumps:
00563                         if "___" in dump:
00564                             DumpEvent=dump.split(".")[0].split("___")[-1]
00565                         else:
00566                             DumpEvent=dump.split(".")[1]
00567                         #New naming convention using ___ as separator
00568                         DumpedProfileName=self.profile_name[:-3]+"___"+DumpEvent+".gz"
00569                         if dump.startswith("IgProf"):
00570                             execute('mv %s %s'%(dump,DumpedProfileName))
00571                         #Keep a tab of the first and last dump event for the diff analysis
00572                         if int(DumpEvent) < FirstDumpEvent:
00573                             FirstDumpEvent = int(DumpEvent)
00574                         if int(DumpEvent) > LastDumpEvent:
00575                             LastDumpEvent = int(DumpEvent)
00576                         #Eliminating the ASCII analysis to get the totals, Giulio will provide this information in igprof-navigator with a special query
00577                         #First type of analysis: dump first 7 lines of ASCII analysis:
00578                         #logger("Executing the igprof-analyse analysis to dump the ASCII 7 lines output with the totals for the IgProf counter")
00579                         #exit=execute('%s -o%s -i%s -t%s' %(IGPROFANALYS,outdir,DumpedProfileName,"ASCII"))
00580                         #Second type of analysis: dump the report in sqlite format to be ready to be browsed with igprof-navigator
00581                         logger("Executing the igprof-analyse analysis saving into igprof-navigator browseable SQLite3 format")
00582                         #exit=exit+execute('%s -o%s -i%s -t%s' %(IGPROFANALYS,outdir,DumpedProfileName,"SQLite3"))
00583                         #Execute all types of analyses available with the current profile (using the dictionary IgProfProfile):
00584                         #To avoid this we should use a further input in SimulationCandles.txt IgProfMem.ANALYSE.MEM_TOTAL maybe the cleanest solution.
00585                         #for IgProfile in IgProfCounters.keys():
00586                         #    if DumpedProfileName.find(IgProfile)>0:
00587                         #        for counter in IgProfCounters[IgProfile]:
00588                         #Check that the file does not exist:
00589                         #if not os.path.exists(DumpedProfileName.split(".")[0].replace(IgProfProfiles[counter],counter)+".sql3"):
00590                         exit=exit+execute('%s -c%s -i%s -t%s' %(IGPROFANALYS,IgProfCounter,DumpedProfileName,"SQLite3"))
00591                                     #else:
00592                                     #    print "File %s already exists will not process profile"%DumpedProfileName.split(".")[0].replace(IgProfProfiles[counter],counter)+".sql3"
00593                     #FIXME:
00594                     #Issue with multiple profiles in the same dir: assuming Perf and Mem will always be in separate dirs
00595                     #Potential ssue with multiple steps?
00596                     #Adapting to new igprof naming scheme:
00597                     FirstDumpProfile=self.profile_name[:-3]+"___"+str(FirstDumpEvent)+".gz"
00598                     LastDumpProfile=self.profile_name[:-3]+"___"+str(LastDumpEvent)+".gz"
00599                     #Third type of analysis: execute the diff analysis:
00600                     #Check there are at least 2 IgProf intermediate event dump profiles to do a regression!
00601                     if len(IgProfDumps)>1:
00602                         logger("Executing the igprof-analyse regression between the first IgProf profile dump and the last one")
00603                         #for IgProfile in IgProfCounters.keys():
00604                         #    if DumpedProfileName.find(IgProfile)>0:
00605                         #        IgProfCounter=IgProfCounters[IgProfile]
00606                         exit=exit+execute('%s -c%s -i%s -r%s' %(IGPROFANALYS,IgProfCounter,LastDumpProfile,FirstDumpProfile))
00607                     else:
00608                         logger("CANNOT execute any regressions: not enough IgProf intermediate event profile dumps!")
00609                     #Fourth type of analysis: execute the grouped by library igprof-analyse:
00610                     logger("Executing the igprof-analyse analysis merging the results by library via regexp and saving the result in igprof-navigator browseable SQLite3 format")
00611                     #for IgProfile in IgProfCounters.keys():
00612                     #        if DumpedProfileName.find(IgProfile)>0:
00613                     #            IgProfCounter=IgProfCounters[IgProfile]
00614                     exit=exit+execute('%s -c%s -i%s --library' %(IGPROFANALYS,IgProfCounter,LastDumpProfile))
00615                 #If they are not there at all (no dumps)
00616                 else:
00617                     logger("No IgProf intermediate event profile dumps found!")
00618                     exit=0
00619                 
00620                 return exit
00621                 
00622 
00623         #####################################################################                     
00624             
00625         # Profiler is EdmSize:        
00626         if 'Edm_Size' in self.profiler:
00627             perfreport_command=''
00628             if not fill_db:
00629                 os.environ["PERFREPORT_PATH"]='%s/' \
00630                                             %PERFREPORT2_PATH
00631                 perfreport_command='%s %s -fe -i %s -o %s' \
00632                                             %(PR2,
00633                                               tmp_switch,
00634                                               self.profile_name,
00635                                               outdir)
00636             else:
00637                 os.environ["PERFREPORT_PATH"]='%s/' \
00638                                             %PERFREPORT3_PATH
00639                 perfreport_command='%s %s -n5000 -u%s -fe -i %s -a -o %s' \
00640                                             %(PR3,
00641                                               tmp_switch,
00642                                               PR3_PRODUCER_PLUGIN,
00643                                               self.profile_name,
00644                                               db_name)             
00645 
00646             return execute(perfreport_command)    
00647 
00648         #FIXME: probably need to move this somewhere else now that we use return statements
00649         if tmp_dir!='':
00650             execute('rm -r %s' %tmp_dir)
00651             
00652         #####################################################################    
00653                             
00654         # Profiler is Valgrind Memcheck
00655         if self.profiler=='Memcheck_Valgrind':
00656             # Three pages will be produced:
00657             os.environ['PERL5LIB']=PERL5_LIB
00658             report_coordinates=(VMPARSER,self.profile_name,outdir)
00659             # Copy the Valgrind Memcheck parser style file in the outdir
00660             copyStyleFile='cp -pR %s %s'%(VMPARSERSTYLE,outdir)
00661             execute(copyStyleFile)
00662             report_commands=('%s --preset +prod,-prod1 %s > %s/edproduce.html'\
00663                                 %report_coordinates,
00664                              '%s --preset +prod1 %s > %s/esproduce.html'\
00665                                 %report_coordinates,
00666                              '%s -t beginJob %s > %s/beginjob.html'\
00667                                 %report_coordinates)
00668             exit=0
00669             for command in report_commands:
00670                 exit= exit + execute(command)      
00671             return exit
00672         #####################################################################                
00673                 
00674         # Profiler is TimeReport parser
00675         
00676         if self.profiler=='Timereport_Parser':
00677             return execute('%s %s %s' %(TIMEREPORTPARSER,self.profile_name,outdir))
00678 
00679         #####################################################################
00680         
00681         # Profiler is Timing Parser            
00682         
00683         if self.profiler=='Timing_Parser':
00684             return execute('%s -i %s -o %s' %(TIMINGPARSER,self.profile_name,outdir))
00685         
00686                     
00687         #####################################################################
00688         
00689         # Profiler is Simple memory parser
00690         
00691         if self.profiler=='SimpleMem_Parser':
00692             return execute('%s -i %s -o %s' %(SIMPLEMEMPARSER,self.profile_name,outdir))
00693             
00694         #####################################################################
00695         
00696         # no profiler
00697             
00698         if self.profiler=='' or self.profiler=='None': #Need to catch the None case, since otherwise we get no return code (crash for pre-requisite step running).
00699             return 0         #Used to be pass, but we need a return 0 to handle exit code properly!         
00700                                                                 
00701 #############################################################################################
00702 
00703 def principal(options):
00704     '''
00705     Here the objects of the Profile class are istantiated.
00706     '''
00707     #Add a global exit code variable, that is the sum of all exit codes, to return it at the end:
00708     exitCodeSum=0
00709     # Build a list of commands for programs to benchmark.
00710     # It will be only one if -c option is selected
00711     commands_profilers_meta_list=[]
00712     
00713     # We have only one
00714     if options.infile=='':
00715         logger('Single command found...')
00716         commands_profilers_meta_list.append([options.command,'','',False,''])
00717     
00718     # We have more: we parse the list of candles    
00719     else:
00720         logger('List of commands found. Processing %s ...' %options.infile)
00721         
00722         # an object that represents the candles file:
00723         candles_file = Candles_file(options.infile)
00724         
00725         commands_profilers_meta_list=candles_file.get_commands_profilers_meta_list()
00726        
00727         
00728     logger('Iterating through commands of executables to profile ...')
00729     
00730     # Cycle through the commands
00731     len_commands_profilers_meta_list=len(commands_profilers_meta_list)    
00732     
00733     commands_counter=1
00734     precedent_profile_name=''
00735     precedent_reuseprofile=False
00736     for command,profiler_opt,meta,reuseprofile,db_metastring in commands_profilers_meta_list:
00737                   
00738         exit_code=0
00739         
00740         logger('Processing command %d/%d' \
00741                     %(commands_counter,len_commands_profilers_meta_list))
00742         logger('Process started on %s' %time.asctime())
00743         
00744         # for multiple directories and outputs let's put the meta
00745         # just before the output profile and the outputdir
00746         profile_name=''
00747         profiler=''
00748         reportdir=options.output
00749         IgProf_counter=options.IgProf_counter
00750            
00751         
00752         if options.infile!='': # we have a list of commands
00753 
00754             reportdir='%s_%s' %(meta,options.output) #Usually options.output is not used
00755             reportdir=clean_name(reportdir)          #Remove _
00756          
00757             profile_name=clean_name('%s_%s'%(meta,options.profile_name)) #Also options.profile_name is usually not used... should clean up...
00758                     
00759             # profiler is igprof: we need to disentangle the profiler and the counter
00760             
00761             if profiler_opt.find('.')!=-1 and profiler_opt.find('IgProf')!=-1:
00762                 profiler_opt_split=profiler_opt.split('.')
00763                 profiler=profiler_opt_split[0]
00764                 IgProf_counter=profiler_opt_split[1:] #This way can handle IgProfMem.ANALYSE.MEM_TOT etc.
00765                 if profile_name[-3:]!='.gz':
00766                     profile_name+='.gz'
00767             
00768             # Profiler is Timereport_Parser
00769             elif profiler_opt in STDOUTPROFILERS:
00770                 # a first maquillage about the profilename:
00771                 if profile_name[:-4]!='.log':
00772                     profile_name+='.log'
00773                 profiler=profiler_opt
00774                         
00775             # profiler is not igprof
00776             else:
00777                 profiler=profiler_opt
00778             
00779             if precedent_reuseprofile:
00780                 profile_name=precedent_profile_name
00781             if reuseprofile:
00782                 precedent_profile_name=profile_name 
00783 
00784                                 
00785                         
00786         else: # we have a single command: easy job!
00787             profile_name=options.profile_name
00788             reportdir=options.output
00789             profiler=options.profiler
00790 
00791         
00792         
00793         # istantiate a Profile object    
00794         if precedent_profile_name!='':
00795             if os.path.exists(precedent_profile_name):
00796                 logger('Reusing precedent profile: %s ...' %precedent_profile_name)
00797                 if profile_name!=precedent_profile_name:
00798                     logger('Copying the old profile to the new name %s ...' %profile_name)
00799                     execute('cp %s %s' %(precedent_profile_name, profile_name))
00800                 
00801         performance_profile=Profile(command,
00802                                     profiler,
00803                                     profile_name)   
00804 
00805         # make profile if needed
00806         if options.profile:                                         
00807             if reuseprofile:
00808                 logger('Saving profile name to reuse it ...')
00809                 precedent_profile_name=profile_name
00810             else:
00811                 precedent_profile_name=''                                
00812             
00813             if not precedent_reuseprofile:
00814                 logger('Creating profile for command %d using %s ...' \
00815                                                 %(commands_counter,profiler))     
00816                 exit_code=performance_profile.make_profile()
00817                 print exit_code
00818                 logger('The exit code was %s'%exit_code)
00819                 exitCodeSum=exitCodeSum+exit_code #Add all exit codes into the global exitCodeSum in order to return it on cmsRelvareport.py exit.
00820                 logger('The exit code sum is %s'%exitCodeSum)
00821             
00822         
00823         # make report if needed   
00824         if options.report:
00825             if exit_code!=0:
00826                 logger('Halting report creation procedure: unexpected exit code %s from %s ...' \
00827                                             %(exit_code,profiler))
00828             else:   
00829                 logger('Creating report for command %d using %s ...' \
00830                                                 %(commands_counter,profiler))     
00831                                                
00832                 # Write into the db instead of producing html if this is the case:
00833                 if options.db:
00834                     exit_code=performance_profile.make_report(fill_db=True,
00835                                                     db_name=options.output,
00836                                                     metastring=db_metastring,
00837                                                     tmp_dir=options.pr_temp,
00838                                                     IgProf_option=IgProf_counter)
00839                     exitCodeSum=exitCodeSum+exit_code #this is to also check that the reporting works... a little more ambitious testing... could do without for release integration
00840                 else:
00841                     exit_code=performance_profile.make_report(outdir=reportdir,
00842                                                     tmp_dir=options.pr_temp,
00843                                                     IgProf_option=IgProf_counter)
00844                     exitCodeSum=exitCodeSum+exit_code #this is to also check that the reporting works... a little more ambitious testing... could do without for release integration
00845                     
00846         commands_counter+=1                                                
00847         precedent_reuseprofile=reuseprofile
00848         if not precedent_reuseprofile:
00849             precedent_profile_name=''
00850         
00851         logger('Process ended on %s\n' %time.asctime())
00852     
00853     logger('Procedure finished on %s' %time.asctime())
00854     logger("Exit code sum is %s"%exitCodeSum)
00855     return exitCodeSum
00856 
00857 ###################################################################################################    
00858         
00859 if __name__=="__main__":
00860 
00861     usage='\n'+\
00862           '----------------------------------------------------------------------------\n'+\
00863           ' RelValreport: a tool for automation of benchmarking and report generation. \n'+\
00864           '----------------------------------------------------------------------------\n\n'+\
00865           'relvalreport.py <options>\n'+\
00866           'relvalreport.py -i candles_150.txt -R -P -n 150.out -o 150_report\n'+\
00867           ' - Executes the candles contained in the file candles_150.txt, create\n'+\
00868           '   profiles, specified by -n, and reports, specified by -o.\n\n'+\
00869           'Candles file grammar:\n'+\
00870           'A candle is specified by the syntax:\n'+\
00871           'executable_name @@@ profiler_name @@@ meta\n'+\
00872           ' - executable_name: the name of the executable to benchmark.\n'+\
00873           ' - profiler_name: the name of the profiler to use. The available are: %s.\n' %str(PROFILERS)+\
00874           '   In case you want to use IgProf_mem or IgProf_perf, the counter (MEM_TOTAL,PERF_TICKS...)\n'+\
00875           '   must be added with a ".": IgProf_mem.MEM_TOTAL.\n'+\
00876           ' - meta: metastring that is used to change the name of the names specified in the command line\n'+\
00877           '   in case of batch execution.'+\
00878           'An example of candle file:\n\n'+\
00879           '  ># My list of candles:\n'+\
00880           '  >\n'+\
00881           '  >cmsDriver.py MU- -sSIM  -e10_20 @@@ IgProf_perf.PERF_TICKS @@@ QCD_sim_IgProfperf\n'+\
00882           '  >cmsDriver.py MU- -sRECO -e10_20 @@@ ValgrindFCE            @@@ QCD_reco_Valgrind\n'+\
00883           '  >cmsRun mycfg.cfg                @@@ IgProf_mem.MEM_TOTAL   @@@ Mycfg\n'
00884              
00885              
00886 
00887     parser = optparse.OptionParser(usage)
00888 
00889     parser.add_option('-p', '--profiler',
00890                       help='Profilers are: %s' %str(PROFILERS) ,
00891                       default='',
00892                       dest='profiler')
00893                                             
00894     parser.add_option('-c', '--command',
00895                       help='Command to profile. If specified the infile is ignored.' ,
00896                       default='',
00897                       dest='command') 
00898                       
00899     parser.add_option('-t',
00900                       help='The temp directory to store the PR service files. Default is PR_TEMP Ignored if PR is not used.',
00901                       default='',
00902                       dest='pr_temp')
00903     
00904     #Flags                      
00905                       
00906     parser.add_option('--db',
00907                       help='EXPERIMENTAL: Write results on the db.',
00908                       action='store_true',
00909                       default=False,
00910                       dest='db')               
00911 
00912     parser.add_option('-R','--Report',
00913                       help='Create a static html report. If db switch is on this is ignored.',
00914                       action='store_true',
00915                       default=False,
00916                       dest='report')                      
00917     
00918     parser.add_option('-P','--Profile',
00919                       help='Create a profile for the selected profiler.',
00920                       action='store_true',
00921                       default=False,
00922                       dest='profile')                        
00923     
00924     # Output location for profile and report                      
00925     
00926     parser.add_option('-n', '--profile_name',
00927                       help='Profile name' ,
00928                       default='',
00929                       dest='profile_name') 
00930                                                                                                                             
00931     parser.add_option('-o', '--output',
00932                       help='Outdir for the html report or db filename.' ,
00933                       default='',
00934                       dest='output')
00935                       
00936     #Batch functionality                       
00937                                                
00938     parser.add_option('-i', '--infile',
00939                       help='Name of the ASCII file containing the commands to profile.' ,
00940                       default='',
00941                       dest='infile')    
00942     
00943     # ig prof options
00944 
00945     parser.add_option('-y', 
00946                       help='Specify the IgProf counter or the CMSSW. '+\
00947                            'If a profiler different from '+\
00948                            'IgProf is selected this is ignored.' ,
00949                       default=None,
00950                       dest='IgProf_counter')                        
00951                       
00952     parser.add_option('--executable',
00953                       help='Specify executable to monitor if different from cmsRun. '+\
00954                            'Only valid for IgProf.',
00955                       default='',
00956                       dest='executable')                               
00957               
00958     # Debug options
00959     parser.add_option('--noexec',
00960                       help='Do not exec commands, just display them!',
00961                       action='store_true',
00962                       default=False,
00963                       dest='noexec')   
00964                                         
00965     (options,args) = parser.parse_args()
00966     
00967     # FAULT CONTROLS
00968     if options.infile=='' and options.command=='' and not (options.report and not options.profile):
00969         raise('Specify at least one command to profile!')
00970     if options.profile_name=='' and options.infile=='':
00971         raise('Specify a profile name!')
00972     if not options.db and options.output=='' and options.infile=='':
00973         raise('Specify a db name or an output dir for the static report!')
00974     
00975     if not options.profile:
00976         if not os.path.exists(options.profile_name) and options.infile=='':
00977             raise('Profile %s does not exist!' %options.profile_name)
00978         logger("WARNING: No profile will be generated. An existing one will be processed!")
00979     
00980     if options.command!='' and options.infile!='':
00981         raise('-c and -i options cannot coexist!')    
00982     
00983     if options.profiler=='Memcheck_Valgrind' and not os.path.exists(VMPARSER):
00984         raise('Couldn\'t find Valgrind Memcheck Parser Script! Please install it from Utilities/ReleaseScripts.')
00985             
00986     if options.executable!='':
00987         globals()['EXECUTABLE']=options.executable
00988     
00989     if options.noexec:
00990         globals()['EXEC']=False
00991         
00992     logger('Procedure started on %s' %time.asctime())                               
00993     
00994     if options.infile == '':
00995         logger('Script options:')
00996         for key,val in options.__dict__.items():
00997             if val!='':
00998                 logger ('\t\t|- %s = %s' %(key, str(val)))
00999                 logger ('\t\t|')
01000     exit=principal(options)
01001     logger("Exit code received from principal is: %s"%exit)
01002     #Mind you! exit codes in Linux are all 0 if they are even! We can easily make the code 1
01003     if exit: #This is different than 0 only if there have been at least one non-zero exit(return) code in the cmsRelvalreport.py
01004         exit=1
01005     sys.exit(exit)
01006
/afs/cern.ch/work/a/aaltunda/public/www/CMSSW_6_2_5/src/Validation/Performance/scripts/cmsRelvalreport.py
