Reference Manual

Go to the documentation of this file.
00001 #! /usr/bin/env python
00002 
00003 r'''
00004 cmsRelvalreport.py: a script to run performance tests and produce reports in a automated way.
00005 '''
00006 
00007 import glob,re #Used for IgProf Analyse work with IgProf Mem profile dumps
00008 # Configuration parameters:#############################################################
00009 
00010 # Perfreport 3 and 2 coordinates:
00011 PR3_BASE='/afs/cern.ch/user/d/dpiparo/w0/perfreport3installation/'
00012 PR3=PR3_BASE+'/bin/perfreport'# executable
00013 PERFREPORT3_PATH=PR3_BASE+'/share/perfreport' #path to xmls
00014 #PR3_PRODUCER_PLUGIN=PR3_BASE+'/lib/libcmssw_by_producer.so' #plugin for fpage
00015 PR3_PRODUCER_PLUGIN='/afs/cern.ch/user/d/dpiparo/w0/pr3/perfreport/plugins/cmssw_by_producer/libcmssw_by_producer.so'
00016 
00017 #PR2_BASE='/afs/cern.ch/user/d/dpiparo/w0/perfreport2.1installation/'
00018 PR2_BASE='/afs/cern.ch/user/g/gbenelli/public/PerfReport2/2.0.1/'
00019 PR2='%s' %(PR2_BASE+'/bin/perfreport')# executable
00020 PERFREPORT2_PATH=PR2_BASE+'/share/perfreport' #path to xmls
00021 
00022 import os
00023 cmssw_base=os.environ["CMSSW_BASE"]
00024 cmssw_release_base=os.environ["CMSSW_RELEASE_BASE"]
00025 pyrelvallocal=cmssw_base+"/src/Configuration/PyReleaseValidation"
00026 #Set the path depending on the presence of a locally checked out version of PyReleaseValidation
00027 if os.path.exists(pyrelvallocal):
00028     RELEASE='CMSSW_BASE'
00029     print "Using LOCAL version of Configuration/PyReleaseValidation instead of the RELEASE version"
00030 elif not os.path.exists(pyrelvallocal):
00031     RELEASE='CMSSW_RELEASE_BASE'
00032     
00033 #Eliminating the full paths to all the scripts used they are assumed to be in the release by default, being in the scripts/ directory of their package    
00034 # Valgrind Memcheck Parser coordinates:
00035 #VMPARSER='%s/src/Utilities/ReleaseScripts/scripts/valgrindMemcheckParser.pl' %os.environ['CMSSW_RELEASE_BASE']#This has to always point to 'CMSSW_RELEASE_BASE'
00036 VMPARSER='valgrindMemcheckParser.pl'
00037 
00038 #Not a script... need to keep the full path to the release
00039 # Valgrind Memcheck Parser output style file coordinates:
00040 VMPARSERSTYLE='%s/src/Utilities/ReleaseScripts/data/valgrindMemcheckParser.css' %os.environ['CMSSW_RELEASE_BASE']#This has to always point to 'CMSSW_RELEASE_BASE'
00041 
00042 # IgProf_Analysis coordinates:
00043 #IGPROFANALYS='%s/src/Validation/Performance/scripts/cmsIgProf_Analysis.py'%os.environ[RELEASE]
00044 IGPROFANALYS='cmsIgProf_Analysis.py'
00045 
00046 # Timereport parser
00047 #TIMEREPORTPARSER='%s/src/Validation/Performance/scripts/cmsTimeReport.pl'%os.environ[RELEASE]
00048 TIMEREPORTPARSER='cmsTimeReport.pl'
00049 
00050 # Simple memory parser
00051 #SIMPLEMEMPARSER='%s/src/Validation/Performance/scripts/cmsSimplememchecker_parser.py' %os.environ[RELEASE]
00052 SIMPLEMEMPARSER='cmsSimplememchecker_parser.py'
00053 
00054 # Timing Parser
00055 #TIMINGPARSER='%s/src/Validation/Performance/scripts/cmsTiming_parser.py' %os.environ[RELEASE]
00056 TIMINGPARSER='cmsTiming_parser.py'
00057 
00058 # makeSkimDriver
00059 MAKESKIMDRIVERDIR='%s/src/Configuration/EventContent/test' %os.environ[RELEASE]
00060 MAKESKIMDRIVER='%s/makeSkimDriver.py'%MAKESKIMDRIVERDIR
00061 
00062 ########################################################################################
00063 
00064 
00065 # Library to include to run valgrind fce
00066 VFCE_LIB='/afs/cern.ch/user/m/moserro/public/vgfcelib' 
00067 PERL5_LIB='/afs/cern.ch/user/d/dpiparo/w0/PERLlibs/5.8.0'
00068 
00069 
00070 
00071 # the profilers that use the stout of the app..
00072 STDOUTPROFILERS=['Memcheck_Valgrind',
00073                  'Timereport_Parser',
00074                  'Timing_Parser',
00075                  'SimpleMem_Parser']
00076 # Profilers list
00077 PROFILERS=['ValgrindFCE',
00078            'IgProf_perf',
00079            'IgProf_mem',
00080            'Edm_Size']+STDOUTPROFILERS
00081                             
00082 
00083 # name of the executable to benchmark. It can be different from cmsRun in future           
00084 EXECUTABLE='cmsRun'
00085 
00086 # Command execution and debug switches
00087 EXEC=True
00088 DEBUG=True
00089 
00090 #Handy dictionaries to handle the mapping between IgProf Profiles and counters:
00091 IgProfCounters={'IgProfPerf':['PERF_TICKS'],
00092                 'IgProfMem':['MEM_TOTAL','MEM_LIVE','MEM_MAX']
00093                 }
00094 IgProfProfiles={'PERF_TICKS':'IgProfPerf',
00095                'MEM_TOTAL':'IgProfMem',
00096                'MEM_LIVE':'IgProfMem',
00097                'MEM_MAX':'IgProfMem'
00098                }
00099 import time   
00100 import optparse 
00101 import sys
00102 
00103 
00104 #######################################################################
00105 def red(string):
00106     return '%s%s%s' %('\033[1;31m',string,'\033[1;0m')    
00107 def green(string):
00108     return '%s%s%s' %('\033[1;32m',string,'\033[1;0m') 
00109 def yellow(string):
00110     return '%s%s%s' %('\033[1;33m',string,'\033[1;0m')     
00111 #######################################################################
00112 
00113 def clean_name(name):
00114     '''
00115     Trivially removes an underscore if present as last char of a string
00116     '''
00117     i=-1
00118     is_dirty=True
00119     while(is_dirty):
00120         if name[i]=='_':
00121             name=name[:-1]
00122         else:
00123             return name
00124         i-=1
00125 
00126 #######################################################################
00127 
00128 def execute(command):
00129         '''
00130         It executes command if the EXEC switch is True. 
00131         Catches exitcodes different from 0.
00132         '''
00133         logger('%s %s ' %(green('[execute]'),command))
00134         if EXEC:
00135             exit_code=os.system(command)
00136             if exit_code!=0:
00137                 logger(red('*** Seems like "%s" encountered problems.' %command))
00138             return exit_code
00139         else:
00140             return 0
00141             
00142 #######################################################################            
00143             
00144 def logger(message,level=0):
00145     '''
00146     level=0 output, level 1 debug.
00147     '''                  
00148     message='%s %s' %(yellow('[RelValreport]'),message)
00149     
00150     sys.stdout.flush()
00151     
00152     if level==0:
00153         print message
00154     if level==1 and DEBUG:
00155         print message    
00156     
00157     sys.stdout.flush()
00158 
00159 #######################################################################
00160 
00161 class Candles_file:
00162     '''
00163     Class to read the trivial ASCII file containing the candles
00164     '''
00165     def __init__(self, filename):
00166         
00167         self.commands_profilers_meta_list=[]    
00168     
00169         candlesfile=open(filename,'r')
00170         
00171         if filename[-3:]=='xml':
00172             command=''
00173             profiler=''
00174             meta=''
00175             db_meta=''
00176             reuse=False    
00177             
00178             from xml.dom import minidom
00179             
00180             # parse the config
00181             xmldoc = minidom.parse(filename)
00182             
00183             # get the candles
00184             candles_list = xmldoc.getElementsByTagName('candle')
00185             
00186             # a list of dictionaries to store the info
00187             candles_dict_list=[]
00188             
00189             for candle in candles_list:
00190                 info_dict={}
00191                 for child in candle.childNodes:# iteration over candle node children
00192                     if not child.__dict__.has_key('nodeName'):# if just a text node skip!
00193                         #print 'CONTINUE!'
00194                         continue
00195                     # We pick the info from the node
00196                     tag_name=child.tagName
00197                     #print 'Manipulating a %s ...'%tag_name
00198                     data=child.firstChild.data
00199                     #print 'Found the data: %s !' %data
00200                     # and we put it in the dictionary
00201                     info_dict[tag_name]=data
00202                 # to store it in a list
00203                 candles_dict_list.append(info_dict)
00204             
00205             # and now process what was parsed
00206                         
00207             for candle_dict in candles_dict_list:
00208                 # compulsory params!!
00209                 command=candle_dict['command']
00210                 profiler=candle_dict['profiler']
00211                 meta=candle_dict['meta']
00212                 # other params
00213                 try:
00214                     db_meta=candle_dict['db_meta']
00215                 except:
00216                     db_meta=None
00217                 try:
00218                     reuse=candle_dict['reuse']
00219                 except:
00220                     reuse=False    
00221                             
00222                 self.commands_profilers_meta_list.append([command,profiler,meta,reuse,db_meta])
00223         
00224         # The file is a plain ASCII
00225         else:
00226             for candle in candlesfile.readlines():
00227                 # Some parsing of the file
00228                 if candle[0]!='#' and candle.strip(' \n\t')!='': # if not a comment or an empty line
00229                     if candle[-1]=='\n': #remove trail \n if it's there
00230                         candle=candle[:-1] 
00231                     splitted_candle=candle.split('@@@') #separate
00232                     
00233                     # compulsory fields
00234                     command=splitted_candle[0]
00235                     profiler=splitted_candle[1].strip(' \t')
00236                     meta=splitted_candle[2].strip(' \t')        
00237                     info=[command,profiler,meta]
00238                     
00239                     # FIXME: AN .ini or xml config??
00240                     # do we have something more?
00241                     len_splitted_candle=len(splitted_candle)
00242                     reuse=False
00243                     if len_splitted_candle>3:
00244                         # is it a reuse statement?
00245                         if 'reuse' in splitted_candle[3]:
00246                             reuse=True
00247                         info.append(reuse)
00248                     else:
00249                         info.append(reuse)               
00250                     
00251                     # we have one more field or we hadn't a reuse in the last one    
00252                     if len_splitted_candle>4 or (len_splitted_candle>3 and not reuse):
00253                         cmssw_scram_version_string=splitted_candle[-1].strip(' \t')
00254                         info.append(cmssw_scram_version_string)
00255                     else:
00256                         info.append(None)
00257     
00258                         
00259                     self.commands_profilers_meta_list.append(info)
00260                     
00261     #----------------------------------------------------------------------
00262         
00263     def get_commands_profilers_meta_list(self):
00264         return self.commands_profilers_meta_list
00265             
00266 #######################################################################
00267 
00268 class Profile:
00269     '''
00270     Class that represents the procedure of performance report creation
00271     '''
00272     def __init__(self,command,profiler,profile_name):
00273         self.command=command
00274         self.profile_name=profile_name
00275         self.profiler=profiler
00276     
00277     #------------------------------------------------------------------
00278     # edit here if more profilers added
00279     def make_profile(self):
00280         '''
00281         Launch the right function according to the profiler name.
00282         '''  
00283         if self.profiler=='ValgrindFCE':
00284             return self._profile_valgrindfce()
00285         elif self.profiler.find('IgProf')!=-1:
00286             return self._profile_igprof()    
00287         elif self.profiler.find('Edm_Size')!=-1:
00288             return self._profile_edmsize()
00289         elif self.profiler=='Memcheck_Valgrind':
00290             return self._profile_Memcheck_Valgrind()
00291         elif self.profiler=='Timereport_Parser':
00292             return self._profile_Timereport_Parser()
00293         elif self.profiler=='Timing_Parser':
00294             return self._profile_Timing_Parser()        
00295         elif self.profiler=='SimpleMem_Parser':
00296             return self._profile_SimpleMem_Parser()
00297         elif self.profiler=='':
00298             return self._profile_None()
00299         elif self.profiler=='None': #adding this for the case of candle ASCII file non-profiling commands
00300             return self._profile_None()
00301         else:
00302             raise('No %s profiler found!' %self.profiler)
00303     #------------------------------------------------------------------
00304     def _profile_valgrindfce(self):
00305         '''
00306         Valgrind profile launcher.
00307         '''
00308         # ValgrindFCE needs a special library to run
00309         os.environ["VALGRIND_LIB"]=VFCE_LIB
00310         
00311         profiler_line=''
00312         valgrind_options= 'time valgrind '+\
00313                           '--tool=callgrind '+\
00314                           '--fce=%s ' %(self.profile_name)
00315         
00316         # If we are using cmsDriver we should use the prefix switch        
00317         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00318             profiler_line='%s --prefix "%s"' %(self.command,valgrind_options)
00319                             
00320         else:                          
00321             profiler_line='%s %s' %(valgrind_options,self.command)
00322                         #'--trace-children=yes '+\
00323         
00324         return execute(profiler_line)
00325     
00326     #------------------------------------------------------------------
00327     def _profile_igprof(self): 
00328         '''
00329         IgProf profiler launcher.
00330         '''
00331         profiler_line=''
00332         
00333         igprof_options='igprof -d -t %s ' \
00334                     %EXECUTABLE # IgProf profile not general only for CMSRUN!
00335         
00336         # To handle Igprof memory and performance profiler in one function
00337         if self.profiler=='IgProf_perf':
00338             igprof_options+='-pp '
00339         elif self.profiler=='IgProf_mem':
00340             igprof_options+='-mp '
00341         else:
00342             raise ('Unknown IgProf flavour: %s !'%self.profiler)
00343         igprof_options+='-z -o %s' %(self.profile_name)
00344         
00345         # If we are using cmsDriver we should use the prefix switch 
00346         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00347             profiler_line='%s --prefix "%s"' %(self.command,igprof_options) 
00348         else:
00349             profiler_line='%s %s' %(igprof_options, self.command)  
00350             
00351         return execute(profiler_line)
00352     
00353     #------------------------------------------------------------------
00354     
00355     def _profile_edmsize(self):
00356         '''
00357         Launch edm size profiler
00358         '''
00359         # In this case we replace the name to be clear
00360         input_rootfile=self.command
00361         
00362         # Skim the content if requested!
00363         if '.' in self.profiler:
00364             
00365             clean_profiler_name,options=self.profiler.split('.')
00366             content,nevts=options.split(',')
00367             outfilename='%s_%s.root'%(os.path.basename(self.command)[:-6],content)
00368             oldpypath=os.environ['PYTHONPATH']
00369             os.environ['PYTHONPATH']+=':%s' %MAKESKIMDRIVERDIR
00370             execute('%s -i %s -o %s --outputcommands %s -n %s' %(MAKESKIMDRIVER,
00371                                                                  self.command,
00372                                                                  outfilename,
00373                                                                  content,
00374                                                                  nevts)) 
00375             os.environ['PYTHONPATH']=oldpypath
00376             #execute('rm %s' %outfilename)
00377             self.command=outfilename
00378             self.profiler=clean_profiler_name
00379                                                                                                         
00380         
00381         profiler_line='edmEventSize -o %s -d %s'\
00382                             %(self.profile_name,self.command)
00383         
00384         return execute(profiler_line)
00385    
00386    #------------------------------------------------------------------
00387    
00388     def _profile_Memcheck_Valgrind(self):
00389         '''
00390         Valgrind Memcheck profile launcher
00391         '''
00392         profiler_line=''
00393         #Adding cms suppression of useless messages (cmsvgsupp)
00394         #Removing leak-checking (done with igprof)
00395         #'--leak-check=no '+\ (no is the default)
00396         #'--show-reachable=yes '+\
00397         #'--track-fds=yes '
00398         #Adding xml logging
00399         valgrind_options='time valgrind --tool=memcheck `cmsvgsupp` '+\
00400                                '--num-callers=20 '+\
00401                                '--xml=yes '+\
00402                                '--xml-file=%s.xml '%self.profile_name.replace(",","-")[:-4]
00403         
00404         # If we are using cmsDriver we should use the prefix switch        
00405         if EXECUTABLE=='cmsRun' and self.command.find('cmsDriver.py')!=-1:
00406             #Replacing 2>&1 |tee with >& in the shell command to preserve the return code significance:
00407             # using tee return would be 0 even if the command failed before the pipe:
00408             profiler_line='%s --prefix "%s" >& %s' %(self.command,valgrind_options,self.profile_name)
00409                             
00410         else:                          
00411             profiler_line='%s %s >& %s' %(valgrind_options,self.command,self.profile_name)
00412                         #'--trace-children=yes '+\
00413         return execute(profiler_line)
00414     #-------------------------------------------------------------------
00415     
00416     def _profile_Timereport_Parser(self):
00417         return self._save_output()
00418         
00419     #-------------------------------------------------------------------
00420     
00421     def _profile_SimpleMem_Parser(self):
00422         return self._save_output()
00423 
00424     #-------------------------------------------------------------------
00425     
00426     def _profile_Timing_Parser(self):
00427         return self._save_output()     
00428     
00429     #-------------------------------------------------------------------
00430         
00431     def _save_output(self):
00432         '''
00433         Save the output of cmsRun on a file!
00434         '''               
00435 #         # a first maquillage about the profilename:
00436 #         if self.profile_name[-4:]!='.log':
00437 #             self.profile_name+='.log'
00438         #Replacing 2>&1 |tee with >& in the shell command to preserve the return code significance:
00439         # using tee return would be 0 even if the command failed before the pipe:
00440         profiler_line='%s  >& %s' %(self.command,self.profile_name)
00441         return execute(profiler_line)    
00442 
00443     #-------------------------------------------------------------------                    
00444     
00445     def _profile_None(self):
00446         '''
00447         Just Run the command!
00448         '''
00449         return execute(self.command)
00450     
00451     #-------------------------------------------------------------------
00452     
00453     def make_report(self,
00454                     fill_db=False,
00455                     db_name=None,
00456                     tmp_dir=None,
00457                     outdir=None,
00458                     IgProf_option=None,
00459                     metastring=None):
00460         '''
00461         Make a performance report with CMSSW scripts for CMSSW internal profiling (Timing/SimpleMemoryCheck) and Memcheck, PR2 for edmEventSize and Callgrind (NOTE PR2 is not supported anymore and is not currently in the CMSSW external, running froma privat AFS!), igprof-analyse for all IgProf profiling.
00462         '''
00463 
00464         if outdir==None or outdir==self.profile_name:
00465             outdir=self.profile_name+'_outdir'
00466             
00467         #Create the directory where the report will be stored:
00468         if not os.path.exists(outdir) and not fill_db and not IgProf_option:
00469             #Added an IgProf condition to avoid the creation of a directory that we will not need anymore, since we will put all info in the filenames
00470             execute('mkdir %s' %outdir)
00471         
00472         if fill_db:
00473             db_option='-a'
00474             if not os.path.exists(db_name):
00475                 db_option='-A'
00476         
00477         # temp in the local dir for PR
00478         tmp_switch=''    
00479         if tmp_dir!='':
00480             execute('mkdir %s' %tmp_dir)
00481             tmp_switch=' -t %s' %tmp_dir
00482 
00483         #Handle the various profilers:
00484         
00485         ##################################################################### 
00486         
00487         # Profiler is ValgrindFCE:
00488         if self.profiler=='ValgrindFCE':
00489             perfreport_command=''
00490             # Switch for filling the db
00491             if not fill_db:
00492                 os.environ["PERFREPORT_PATH"]='%s/' %PERFREPORT2_PATH
00493                 perfreport_command='%s %s -ff -i %s -o %s' %(PR2,
00494                                                              tmp_switch,
00495                                                              self.profile_name,
00496                                                              outdir)
00497             else:
00498                 os.environ["PERFREPORT_PATH"]='%s/' %PERFREPORT3_PATH
00499                 perfreport_command='%s %s -n5000 -u%s  -ff  -m \'scram_cmssw_version_string,%s\' -i %s %s -o %s' \
00500                                                     %(PR3,
00501                                                       tmp_switch,
00502                                                       PR3_PRODUCER_PLUGIN,
00503                                                       metastring,
00504                                                       self.profile_name,
00505                                                       db_option,
00506                                                       db_name)
00507             return execute(perfreport_command)
00508         
00509         #####################################################################            
00510             
00511         # Profiler is IgProf:
00512         if self.profiler.find('IgProf')!=-1:
00513             #First the case of IgProf PERF and MEM reporting:
00514             if not 'ANALYSE' in IgProf_option:
00515                 #Switch to the use of igprof-analyse instead of PerfReport!
00516                 #Will use the ANALYSE case for regressions between early event dumps and late event dumps of the profiles
00517                 #Following Andreas suggestion, add the number of events for the EndOfJob report
00518                 NumberOfEvents=self.command.split()[3] #FIXME: this is quite hardcoded... but should be stable...
00519                 sqlite_outputfile=self.profile_name.split(".")[0].replace(IgProfProfiles[IgProf_option[0]],IgProf_option[0])+'___'+NumberOfEvents+'_EndOfJob.sql3'
00520                 logger("Executing the report of the IgProf end of job profile")
00521                 exit=execute('igprof-analyse --sqlite -d -v -g -r %s %s | sqlite3 %s'%(IgProf_option[0],self.profile_name,sqlite_outputfile)) 
00522                 return exit
00523             #Then the "ANALYSE" case that we want to use to add to the same directories (Perf, MemTotal, MemLive)
00524             #also some other analyses and in particular:
00525             #1-the first 7 lines of the ASCII analysis of the IgProf profile dumps (total of the counters)
00526             #2-the dump at different event numbers,
00527             #3-the diff between the first and last dump,
00528             #4-the counters grouped by library using regexp at the last dump:
00529             else: #We use IgProf Analysis
00530                 #Set the IgProfCounter from the ANALYSE.MEM_TOT style IgProf_option
00531                 #print IgProf_option
00532                 IgProfCounter=IgProf_option[1]
00533                 #Add here the handling of the new IgProf.N.gz files so that they will get preserved and not overwritten:
00534                 logger("Looking for IgProf intermediate event profile dumps")
00535                 #Check if there are IgProf.N.gz dump files:
00536                 IgProfDumps=glob.glob("IgProf.*.gz")
00537                 #in case there are none check if they have already been mv'd to another name to avoid overwriting
00538                 #(MEM_LIVE usually re-uses MEM_TOTAL, so the IgProf.N.gz files will already have a MemTotal name...)
00539                 if not IgProfDumps:
00540                     localFiles=os.listdir('.')
00541                     IgProfDumpProfilesPrevious=re.compile(r"\w+.\d+.gz")
00542                     IgProfDumps=filter(lambda x: IgProfDumpProfilesPrevious.search(x),localFiles)
00543                 #Now if there are dumps execute the following analyses:
00544                 if IgProfDumps:
00545                     IgProfDumps.sort()
00546                     logger("Found the following IgProf intermediate event profile dumps:")
00547                     logger(IgProfDumps)
00548                     FirstDumpEvent=9999999
00549                     LastDumpEvent=0
00550                     exit=0
00551                     for dump in IgProfDumps:
00552                         if "___" in dump:
00553                             DumpEvent=dump.split(".")[0].split("___")[-1]
00554                         else:
00555                             DumpEvent=dump.split(".")[1]
00556                         #New naming convention using ___ as separator
00557                         DumpedProfileName=self.profile_name[:-3]+"___"+DumpEvent+".gz"
00558                         if dump.startswith("IgProf"):
00559                             execute('mv %s %s'%(dump,DumpedProfileName))
00560                         #Keep a tab of the first and last dump event for the diff analysis
00561                         if int(DumpEvent) < FirstDumpEvent:
00562                             FirstDumpEvent = int(DumpEvent)
00563                         if int(DumpEvent) > LastDumpEvent:
00564                             LastDumpEvent = int(DumpEvent)
00565                         #Eliminating the ASCII analysis to get the totals, Giulio will provide this information in igprof-navigator with a special query
00566                         #First type of analysis: dump first 7 lines of ASCII analysis:
00567                         #logger("Executing the igprof-analyse analysis to dump the ASCII 7 lines output with the totals for the IgProf counter")
00568                         #exit=execute('%s -o%s -i%s -t%s' %(IGPROFANALYS,outdir,DumpedProfileName,"ASCII"))
00569                         #Second type of analysis: dump the report in sqlite format to be ready to be browsed with igprof-navigator
00570                         logger("Executing the igprof-analyse analysis saving into igprof-navigator browseable SQLite3 format")
00571                         #exit=exit+execute('%s -o%s -i%s -t%s' %(IGPROFANALYS,outdir,DumpedProfileName,"SQLite3"))
00572                         #Execute all types of analyses available with the current profile (using the dictionary IgProfProfile):
00573                         #To avoid this we should use a further input in SimulationCandles.txt IgProfMem.ANALYSE.MEM_TOTAL maybe the cleanest solution.
00574                         #for IgProfile in IgProfCounters.keys():
00575                         #    if DumpedProfileName.find(IgProfile)>0:
00576                         #        for counter in IgProfCounters[IgProfile]:
00577                         #Check that the file does not exist:
00578                         #if not os.path.exists(DumpedProfileName.split(".")[0].replace(IgProfProfiles[counter],counter)+".sql3"):
00579                         exit=exit+execute('%s -c%s -i%s -t%s' %(IGPROFANALYS,IgProfCounter,DumpedProfileName,"SQLite3"))
00580                                     #else:
00581                                     #    print "File %s already exists will not process profile"%DumpedProfileName.split(".")[0].replace(IgProfProfiles[counter],counter)+".sql3"
00582                     #FIXME:
00583                     #Issue with multiple profiles in the same dir: assuming Perf and Mem will always be in separate dirs
00584                     #Potential ssue with multiple steps?
00585                     #Adapting to new igprof naming scheme:
00586                     FirstDumpProfile=self.profile_name[:-3]+"___"+str(FirstDumpEvent)+".gz"
00587                     LastDumpProfile=self.profile_name[:-3]+"___"+str(LastDumpEvent)+".gz"
00588                     #Third type of analysis: execute the diff analysis:
00589                     #Check there are at least 2 IgProf intermediate event dump profiles to do a regression!
00590                     if len(IgProfDumps)>1:
00591                         logger("Executing the igprof-analyse regression between the first IgProf profile dump and the last one")
00592                         #for IgProfile in IgProfCounters.keys():
00593                         #    if DumpedProfileName.find(IgProfile)>0:
00594                         #        IgProfCounter=IgProfCounters[IgProfile]
00595                         exit=exit+execute('%s -c%s -i%s -r%s' %(IGPROFANALYS,IgProfCounter,LastDumpProfile,FirstDumpProfile))
00596                     else:
00597                         logger("CANNOT execute any regressions: not enough IgProf intermediate event profile dumps!")
00598                     #Fourth type of analysis: execute the grouped by library igprof-analyse:
00599                     logger("Executing the igprof-analyse analysis merging the results by library via regexp and saving the result in igprof-navigator browseable SQLite3 format")
00600                     #for IgProfile in IgProfCounters.keys():
00601                     #        if DumpedProfileName.find(IgProfile)>0:
00602                     #            IgProfCounter=IgProfCounters[IgProfile]
00603                     exit=exit+execute('%s -c%s -i%s --library' %(IGPROFANALYS,IgProfCounter,LastDumpProfile))
00604                 #If they are not there at all (no dumps)
00605                 else:
00606                     logger("No IgProf intermediate event profile dumps found!")
00607                     exit=0
00608                 
00609                 return exit
00610                 
00611 
00612         #####################################################################                     
00613             
00614         # Profiler is EdmSize:        
00615         if 'Edm_Size' in self.profiler:
00616             perfreport_command=''
00617             if not fill_db:
00618                 os.environ["PERFREPORT_PATH"]='%s/' \
00619                                             %PERFREPORT2_PATH
00620                 perfreport_command='%s %s -fe -i %s -o %s' \
00621                                             %(PR2,
00622                                               tmp_switch,
00623                                               self.profile_name,
00624                                               outdir)
00625             else:
00626                 os.environ["PERFREPORT_PATH"]='%s/' \
00627                                             %PERFREPORT3_PATH
00628                 perfreport_command='%s %s -n5000 -u%s -fe -i %s -a -o %s' \
00629                                             %(PR3,
00630                                               tmp_switch,
00631                                               PR3_PRODUCER_PLUGIN,
00632                                               self.profile_name,
00633                                               db_name)             
00634 
00635             return execute(perfreport_command)    
00636 
00637         #FIXME: probably need to move this somewhere else now that we use return statements
00638         if tmp_dir!='':
00639             execute('rm -r %s' %tmp_dir)
00640             
00641         #####################################################################    
00642                             
00643         # Profiler is Valgrind Memcheck
00644         if self.profiler=='Memcheck_Valgrind':
00645             # Three pages will be produced:
00646             os.environ['PERL5LIB']=PERL5_LIB
00647             report_coordinates=(VMPARSER,self.profile_name,outdir)
00648             # Copy the Valgrind Memcheck parser style file in the outdir
00649             copyStyleFile='cp -pR %s %s'%(VMPARSERSTYLE,outdir)
00650             execute(copyStyleFile)
00651             report_commands=('%s --preset +prod,-prod1 %s > %s/edproduce.html'\
00652                                 %report_coordinates,
00653                              '%s --preset +prod1 %s > %s/esproduce.html'\
00654                                 %report_coordinates,
00655                              '%s -t beginJob %s > %s/beginjob.html'\
00656                                 %report_coordinates)
00657             exit=0
00658             for command in report_commands:
00659                 exit= exit + execute(command)      
00660             return exit
00661         #####################################################################                
00662                 
00663         # Profiler is TimeReport parser
00664         
00665         if self.profiler=='Timereport_Parser':
00666             return execute('%s %s %s' %(TIMEREPORTPARSER,self.profile_name,outdir))
00667 
00668         #####################################################################
00669         
00670         # Profiler is Timing Parser            
00671         
00672         if self.profiler=='Timing_Parser':
00673             return execute('%s -i %s -o %s' %(TIMINGPARSER,self.profile_name,outdir))
00674         
00675                     
00676         #####################################################################
00677         
00678         # Profiler is Simple memory parser
00679         
00680         if self.profiler=='SimpleMem_Parser':
00681             return execute('%s -i %s -o %s' %(SIMPLEMEMPARSER,self.profile_name,outdir))
00682             
00683         #####################################################################
00684         
00685         # no profiler
00686             
00687         if self.profiler=='' or self.profiler=='None': #Need to catch the None case, since otherwise we get no return code (crash for pre-requisite step running).
00688             return 0         #Used to be pass, but we need a return 0 to handle exit code properly!         
00689                                                                 
00690 #############################################################################################
00691 
00692 def principal(options):
00693     '''
00694     Here the objects of the Profile class are istantiated.
00695     '''
00696     #Add a global exit code variable, that is the sum of all exit codes, to return it at the end:
00697     exitCodeSum=0
00698     # Build a list of commands for programs to benchmark.
00699     # It will be only one if -c option is selected
00700     commands_profilers_meta_list=[]
00701     
00702     # We have only one
00703     if options.infile=='':
00704         logger('Single command found...')
00705         commands_profilers_meta_list.append([options.command,'','',False,''])
00706     
00707     # We have more: we parse the list of candles    
00708     else:
00709         logger('List of commands found. Processing %s ...' %options.infile)
00710         
00711         # an object that represents the candles file:
00712         candles_file = Candles_file(options.infile)
00713         
00714         commands_profilers_meta_list=candles_file.get_commands_profilers_meta_list()
00715        
00716         
00717     logger('Iterating through commands of executables to profile ...')
00718     
00719     # Cycle through the commands
00720     len_commands_profilers_meta_list=len(commands_profilers_meta_list)    
00721     
00722     commands_counter=1
00723     precedent_profile_name=''
00724     precedent_reuseprofile=False
00725     for command,profiler_opt,meta,reuseprofile,db_metastring in commands_profilers_meta_list:
00726                   
00727         exit_code=0
00728         
00729         logger('Processing command %d/%d' \
00730                     %(commands_counter,len_commands_profilers_meta_list))
00731         logger('Process started on %s' %time.asctime())
00732         
00733         # for multiple directories and outputs let's put the meta
00734         # just before the output profile and the outputdir
00735         profile_name=''
00736         profiler=''
00737         reportdir=options.output
00738         IgProf_counter=options.IgProf_counter
00739            
00740         
00741         if options.infile!='': # we have a list of commands
00742 
00743             reportdir='%s_%s' %(meta,options.output) #Usually options.output is not used
00744             reportdir=clean_name(reportdir)          #Remove _
00745          
00746             profile_name=clean_name('%s_%s'%(meta,options.profile_name)) #Also options.profile_name is usually not used... should clean up...
00747                     
00748             # profiler is igprof: we need to disentangle the profiler and the counter
00749             
00750             if profiler_opt.find('.')!=-1 and profiler_opt.find('IgProf')!=-1:
00751                 profiler_opt_split=profiler_opt.split('.')
00752                 profiler=profiler_opt_split[0]
00753                 IgProf_counter=profiler_opt_split[1:] #This way can handle IgProfMem.ANALYSE.MEM_TOT etc.
00754                 if profile_name[-3:]!='.gz':
00755                     profile_name+='.gz'
00756             
00757             # Profiler is Timereport_Parser
00758             elif profiler_opt in STDOUTPROFILERS:
00759                 # a first maquillage about the profilename:
00760                 if profile_name[:-4]!='.log':
00761                     profile_name+='.log'
00762                 profiler=profiler_opt
00763                         
00764             # profiler is not igprof
00765             else:
00766                 profiler=profiler_opt
00767             
00768             if precedent_reuseprofile:
00769                 profile_name=precedent_profile_name
00770             if reuseprofile:
00771                 precedent_profile_name=profile_name 
00772 
00773                                 
00774                         
00775         else: # we have a single command: easy job!
00776             profile_name=options.profile_name
00777             reportdir=options.output
00778             profiler=options.profiler
00779 
00780         
00781         
00782         # istantiate a Profile object    
00783         if precedent_profile_name!='':
00784             if os.path.exists(precedent_profile_name):
00785                 logger('Reusing precedent profile: %s ...' %precedent_profile_name)
00786                 if profile_name!=precedent_profile_name:
00787                     logger('Copying the old profile to the new name %s ...' %profile_name)
00788                     execute('cp %s %s' %(precedent_profile_name, profile_name))
00789                 
00790         performance_profile=Profile(command,
00791                                     profiler,
00792                                     profile_name)   
00793 
00794         # make profile if needed
00795         if options.profile:                                         
00796             if reuseprofile:
00797                 logger('Saving profile name to reuse it ...')
00798                 precedent_profile_name=profile_name
00799             else:
00800                 precedent_profile_name=''                                
00801             
00802             if not precedent_reuseprofile:
00803                 logger('Creating profile for command %d using %s ...' \
00804                                                 %(commands_counter,profiler))     
00805                 exit_code=performance_profile.make_profile()
00806                 print exit_code
00807                 logger('The exit code was %s'%exit_code)
00808                 exitCodeSum=exitCodeSum+exit_code #Add all exit codes into the global exitCodeSum in order to return it on cmsRelvareport.py exit.
00809                 logger('The exit code sum is %s'%exitCodeSum)
00810             
00811         
00812         # make report if needed   
00813         if options.report:
00814             if exit_code!=0:
00815                 logger('Halting report creation procedure: unexpected exit code %s from %s ...' \
00816                                             %(exit_code,profiler))
00817             else:   
00818                 logger('Creating report for command %d using %s ...' \
00819                                                 %(commands_counter,profiler))     
00820                                                
00821                 # Write into the db instead of producing html if this is the case:
00822                 if options.db:
00823                     exit_code=performance_profile.make_report(fill_db=True,
00824                                                     db_name=options.output,
00825                                                     metastring=db_metastring,
00826                                                     tmp_dir=options.pr_temp,
00827                                                     IgProf_option=IgProf_counter)
00828                     exitCodeSum=exitCodeSum+exit_code #this is to also check that the reporting works... a little more ambitious testing... could do without for release integration
00829                 else:
00830                     exit_code=performance_profile.make_report(outdir=reportdir,
00831                                                     tmp_dir=options.pr_temp,
00832                                                     IgProf_option=IgProf_counter)
00833                     exitCodeSum=exitCodeSum+exit_code #this is to also check that the reporting works... a little more ambitious testing... could do without for release integration
00834                     
00835         commands_counter+=1                                                
00836         precedent_reuseprofile=reuseprofile
00837         if not precedent_reuseprofile:
00838             precedent_profile_name=''
00839         
00840         logger('Process ended on %s\n' %time.asctime())
00841     
00842     logger('Procedure finished on %s' %time.asctime())
00843     logger("Exit code sum is %s"%exitCodeSum)
00844     return exitCodeSum
00845 
00846 ###################################################################################################    
00847         
00848 if __name__=="__main__":
00849 
00850     usage='\n'+\
00851           '----------------------------------------------------------------------------\n'+\
00852           ' RelValreport: a tool for automation of benchmarking and report generation. \n'+\
00853           '----------------------------------------------------------------------------\n\n'+\
00854           'relvalreport.py <options>\n'+\
00855           'relvalreport.py -i candles_150.txt -R -P -n 150.out -o 150_report\n'+\
00856           ' - Executes the candles contained in the file candles_150.txt, create\n'+\
00857           '   profiles, specified by -n, and reports, specified by -o.\n\n'+\
00858           'Candles file grammar:\n'+\
00859           'A candle is specified by the syntax:\n'+\
00860           'executable_name @@@ profiler_name @@@ meta\n'+\
00861           ' - executable_name: the name of the executable to benchmark.\n'+\
00862           ' - profiler_name: the name of the profiler to use. The available are: %s.\n' %str(PROFILERS)+\
00863           '   In case you want to use IgProf_mem or IgProf_perf, the counter (MEM_TOTAL,PERF_TICKS...)\n'+\
00864           '   must be added with a ".": IgProf_mem.MEM_TOTAL.\n'+\
00865           ' - meta: metastring that is used to change the name of the names specified in the command line\n'+\
00866           '   in case of batch execution.'+\
00867           'An example of candle file:\n\n'+\
00868           '  ># My list of candles:\n'+\
00869           '  >\n'+\
00870           '  >cmsDriver.py MU- -sSIM  -e10_20 @@@ IgProf_perf.PERF_TICKS @@@ QCD_sim_IgProfperf\n'+\
00871           '  >cmsDriver.py MU- -sRECO -e10_20 @@@ ValgrindFCE            @@@ QCD_reco_Valgrind\n'+\
00872           '  >cmsRun mycfg.cfg                @@@ IgProf_mem.MEM_TOTAL   @@@ Mycfg\n'
00873              
00874              
00875 
00876     parser = optparse.OptionParser(usage)
00877 
00878     parser.add_option('-p', '--profiler',
00879                       help='Profilers are: %s' %str(PROFILERS) ,
00880                       default='',
00881                       dest='profiler')
00882                                             
00883     parser.add_option('-c', '--command',
00884                       help='Command to profile. If specified the infile is ignored.' ,
00885                       default='',
00886                       dest='command') 
00887                       
00888     parser.add_option('-t',
00889                       help='The temp directory to store the PR service files. Default is PR_TEMP Ignored if PR is not used.',
00890                       default='',
00891                       dest='pr_temp')
00892     
00893     #Flags                      
00894                       
00895     parser.add_option('--db',
00896                       help='EXPERIMENTAL: Write results on the db.',
00897                       action='store_true',
00898                       default=False,
00899                       dest='db')               
00900 
00901     parser.add_option('-R','--Report',
00902                       help='Create a static html report. If db switch is on this is ignored.',
00903                       action='store_true',
00904                       default=False,
00905                       dest='report')                      
00906     
00907     parser.add_option('-P','--Profile',
00908                       help='Create a profile for the selected profiler.',
00909                       action='store_true',
00910                       default=False,
00911                       dest='profile')                        
00912     
00913     # Output location for profile and report                      
00914     
00915     parser.add_option('-n', '--profile_name',
00916                       help='Profile name' ,
00917                       default='',
00918                       dest='profile_name') 
00919                                                                                                                             
00920     parser.add_option('-o', '--output',
00921                       help='Outdir for the html report or db filename.' ,
00922                       default='',
00923                       dest='output')
00924                       
00925     #Batch functionality                       
00926                                                
00927     parser.add_option('-i', '--infile',
00928                       help='Name of the ASCII file containing the commands to profile.' ,
00929                       default='',
00930                       dest='infile')    
00931     
00932     # ig prof options
00933 
00934     parser.add_option('-y', 
00935                       help='Specify the IgProf counter or the CMSSW. '+\
00936                            'If a profiler different from '+\
00937                            'IgProf is selected this is ignored.' ,
00938                       default=None,
00939                       dest='IgProf_counter')                        
00940                       
00941     parser.add_option('--executable',
00942                       help='Specify executable to monitor if different from cmsRun. '+\
00943                            'Only valid for IgProf.',
00944                       default='',
00945                       dest='executable')                               
00946               
00947     # Debug options
00948     parser.add_option('--noexec',
00949                       help='Do not exec commands, just display them!',
00950                       action='store_true',
00951                       default=False,
00952                       dest='noexec')   
00953                                         
00954     (options,args) = parser.parse_args()
00955     
00956     # FAULT CONTROLS
00957     if options.infile=='' and options.command=='' and not (options.report and not options.profile):
00958         raise('Specify at least one command to profile!')
00959     if options.profile_name=='' and options.infile=='':
00960         raise('Specify a profile name!')
00961     if not options.db and options.output=='' and options.infile=='':
00962         raise('Specify a db name or an output dir for the static report!')
00963     
00964     if not options.profile:
00965         if not os.path.exists(options.profile_name) and options.infile=='':
00966             raise('Profile %s does not exist!' %options.profile_name)
00967         logger("WARNING: No profile will be generated. An existing one will be processed!")
00968     
00969     if options.command!='' and options.infile!='':
00970         raise('-c and -i options cannot coexist!')    
00971     
00972     if options.profiler=='Memcheck_Valgrind' and not os.path.exists(VMPARSER):
00973         raise('Couldn\'t find Valgrind Memcheck Parser Script! Please install it from Utilities/ReleaseScripts.')
00974             
00975     if options.executable!='':
00976         globals()['EXECUTABLE']=options.executable
00977     
00978     if options.noexec:
00979         globals()['EXEC']=False
00980         
00981     logger('Procedure started on %s' %time.asctime())                               
00982     
00983     if options.infile == '':
00984         logger('Script options:')
00985         for key,val in options.__dict__.items():
00986             if val!='':
00987                 logger ('\t\t|- %s = %s' %(key, str(val)))
00988                 logger ('\t\t|')
00989     exit=principal(options)
00990     logger("Exit code received from principal is: %s"%exit)
00991     #Mind you! exit codes in Linux are all 0 if they are even! We can easily make the code 1
00992     if exit: #This is different than 0 only if there have been at least one non-zero exit(return) code in the cmsRelvalreport.py
00993         exit=1
00994     sys.exit(exit)
00995
/data/refman/pasoursint/CMSSW_4_4_5_patch3/src/Validation/Performance/scripts/cmsRelvalreport.py
