Reference Manual

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 import os, sys, re, time
00004 
00005 import random
00006 from threading import Thread
00007         
00008 class WorkFlowRunner(Thread):
00009     def __init__(self, wf):
00010         Thread.__init__(self)
00011         self.wf = wf
00012 
00013         self.status=-1
00014         self.report=''
00015         self.nfail=0
00016         self.npass=0
00017 
00018         return
00019 
00020     def doCmd(self, cmd, dryRun=False):
00021 
00022         msg = "\n# in: " +os.getcwd()
00023         if dryRun: msg += " dryRun for '"
00024         else:      msg += " going to execute "
00025         msg += cmd.replace(';','\n')
00026         print msg
00027 
00028         cmdLog = open(self.wf.numId+'_'+self.wf.nameId+'/cmdLog','a')
00029         cmdLog.write(msg+'\n')
00030         cmdLog.close()
00031         
00032         ret = 0
00033         if not dryRun:
00034             ret = os.system(cmd)
00035             if ret != 0:
00036                 print "ERROR executing ",cmd,'ret=', ret
00037 
00038         return ret
00039     
00040     def run(self):
00041 
00042         startDir = os.getcwd()
00043 
00044         wfDir = self.wf.numId+'_'+self.wf.nameId
00045         if not os.path.exists(wfDir):
00046             os.makedirs(wfDir)
00047 
00048         preamble = ''
00049         if os.path.exists( os.path.join(os.environ["CMS_PATH"],'cmsset_default.sh') ) :
00050             preamble = 'source $CMS_PATH/cmsset_default.sh; '
00051         else:
00052             preamble = 'source $CMS_PATH/sw/cmsset_default.sh; '
00053         preamble += 'eval `scram run -sh`; '
00054         preamble += 'cd '+wfDir+'; '
00055         preamble += 'ulimit -v 4069000;' # make sure processes keep within limits ...
00056         
00057         startime='date %s' %time.asctime()
00058 
00059         # set defaults for the statuses
00060         stat1 = 'PASSED'
00061         stat2 = 'PASSED' 
00062         stat3 = 'PASSED'
00063         stat4 = 'PASSED'
00064         if not self.wf.cmdStep2: stat2 = 'NOSTEP'
00065         if not self.wf.cmdStep3: stat3 = 'NOSTEP'
00066         if not self.wf.cmdStep4: stat4 = 'NOSTEP'
00067         
00068         # run the first workflow:
00069         cmd = preamble
00070 
00071         inFile = 'file:raw.root'
00072         if 'REALDATA' in self.wf.cmdStep1:
00073             realDataRe = re.compile('REALDATA:\s*(/[A-Za-z].*?),(\s*RUN:\s*(?P<run>\d+),)?(\s*FILES:\s*(?P<files>\d+),)?(\s*EVENTS:\s*(?P<events>\d+))?,\s*LABEL:\s*(?P<label>.*),\s*LOCATION:\s*(?P<location>.*)\s*')
00074             realDataMatch = realDataRe.match(self.wf.cmdStep1)
00075             if realDataMatch:
00076                 run = None
00077                 if realDataMatch.group("run") : run = realDataMatch.group("run")
00078                 label  = realDataMatch.group("label")
00079                 location = realDataMatch.group("location").lower().strip()
00080                 if 'caf' in location:
00081                     print "ignoring workflow ",self.wf.numId, self.wf.nameId, ' as this is on CAF ...'
00082                     self.npass = [0,0,0,0]
00083                     self.nfail = [0,0,0,0]
00084 
00085                     logStat = 'Step1-NOTRUN Step2-NOTRUN Step3-NOTRUN Step4-NOTRUN ' 
00086                     self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, 0, 0,0,0,0)
00087                     return
00088                 
00089                 files  = None
00090                 events = None
00091                 if realDataMatch.group("files"): 
00092                   files  = realDataMatch.group("files")
00093                 if realDataMatch.group("events"): 
00094                   events = realDataMatch.group("events")
00095                   if self.wf.cmdStep2 and ' -n ' not in self.wf.cmdStep2: self.wf.cmdStep2 += ' -n ' + events
00096                   if self.wf.cmdStep3 and ' -n ' not in self.wf.cmdStep3: self.wf.cmdStep3 += ' -n ' + events
00097                   if self.wf.cmdStep4 and ' -n ' not in self.wf.cmdStep4: self.wf.cmdStep4 += ' -n ' + events
00098 
00099                 print "run, files, events, label", run, files, events, label 
00100                 cmd += 'dbs search --noheader --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet '
00101                 cmd += "--query='find file where dataset like "+realDataMatch.group(1)
00102                 if run: cmd += " and run=" + run
00103                 cmd += "' "
00104                 cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'-dbsquery.log',)
00105                 retStep1 = self.doCmd(cmd)
00106                 if retStep1 == 0:
00107                     lf = open(wfDir+'/step1_'+self.wf.nameId+'-dbsquery.log', 'r')
00108                     lines = lf.readlines()
00109                     lf.close()
00110                     if not lines or len(lines)==0 :
00111                         inFile = "NoFileFoundInDBS"
00112                         retStep1 = -95
00113                     else:
00114                         try:
00115                             inFile = lines[0].strip()
00116                         except Exception, e:
00117                             print "ERROR determining file from DBS query: ", str(e)
00118                             inFile = "NoFileFoundInDBS"
00119                             retStep1 = -90
00120             else:
00121                 print "ERROR: found REALDATA in '"+self.wf.cmdStep1+"' but not RE match !!??!"
00122                 retStep1 = -99
00123         else:
00124             cmd += self.wf.cmdStep1 + ' --fileout file:raw.root '
00125             cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'.log ',)
00126             retStep1 = self.doCmd(cmd)
00127 
00128         print " ... ret: " , retStep1
00129 
00130         # prepare and run the next workflows -- if the previous step was OK :
00131         # set some defaults
00132         retStep2 = 0
00133         retStep3 = 0
00134         retStep4 = 0
00135         if self.wf.cmdStep2 and retStep1 == 0:
00136             fullcmd = preamble
00137             fullcmd += self.wf.cmdStep2
00138             if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00139             fullcmd += ' --fileout file:reco.root '
00140             print '=====>>> ', self.wf.nameId, self.wf.numId
00141 
00142             # for HI B0 step2 use a file from a previous relval production as step1 doesn't write
00143             # any output in 1 hour. Add himix flag here as this is only needed when run on the mixed
00144             # input files (the relvals are OK)
00145             if ( '40.0' in str(self.wf.numId) ) :
00146                 fullcmd += ' --himix '
00147                 inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/102FF831-9B0F-E011-A3E9-003048678BC6.root'
00148             if ( '41.0' in str(self.wf.numId) ) : 
00149                 fullcmd += ' --himix '
00150                 inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_GammaJet_pt20_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/06B4F699-A50F-E011-AD62-0018F3D0962E.root'
00151 
00152             fullcmd += ' --filein '+inFile+ ' '
00153             fullcmd += ' > %s 2>&1; ' % ('step2_'+self.wf.nameId+'.log ',)
00154             # print fullcmd
00155             retStep2 = self.doCmd(fullcmd)
00156 #            if random.randint(0,100) < 20 : retStep2 = -42
00157 
00158             if self.wf.cmdStep3 and retStep2 == 0:
00159                 fullcmd = preamble
00160                 fullcmd += self.wf.cmdStep3
00161                 if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00162                 # FIXME: dirty hack for beam-spot dedicated relval
00163                 if not '134' in str(self.wf.numId):
00164                     if 'HARVESTING' in fullcmd and not 'filein' in fullcmd:
00165                         fullcmd += ' --filein file:reco_inDQM.root --fileout file:step3.root '
00166                     else:
00167                         fullcmd += ' --filein file:reco.root --fileout file:step3.root '
00168                 fullcmd += ' > %s 2>&1; ' % ('step3_'+self.wf.nameId+'.log ',)
00169                 # print fullcmd
00170                 retStep3 = self.doCmd(fullcmd)
00171 #                if random.randint(0,100) < 40 : retStep3 = -42
00172                 if self.wf.cmdStep4 and retStep3 == 0:
00173                     fullcmd = preamble
00174                     fullcmd += self.wf.cmdStep4
00175                     if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00176                     # FIXME: dirty hack for beam-spot dedicated relval
00177                     if not '134' in str(self.wf.numId):
00178                         fullcmd += ' --filein file:step3.root '
00179                     fullcmd += ' > %s 2>&1; ' % ('step4_'+self.wf.nameId+'.log ',)
00180                     # print fullcmd
00181                     retStep4 = self.doCmd(fullcmd)
00182 #                    if random.randint(0,100) < 40 : retStep4 = -42
00183 
00184         os.chdir(startDir)
00185 
00186         endtime='date %s' %time.asctime()
00187         tottime='%s-%s'%(endtime,startime)
00188 
00189         self.nfail = [0,0,0,0]
00190         self.npass = [1,1,1,1]
00191         if 'NOSTEP' in stat2: # don't say reco/alca is passed if we don't have to run them
00192             self.npass = [1,0,0,0]
00193         else: # we have a reco step, check for alca:
00194             if 'NOSTEP' in stat3 :
00195                 self.npass = [1,1,0,0]
00196                 if 'NOSTEP' in stat4 :
00197                     self.npass = [1,1,1,0]
00198         if retStep1 != 0 :
00199             stat1 = 'FAILED'
00200             stat2 = 'NOTRUN'
00201             stat3 = 'NOTRUN'
00202             stat4 = 'NOTRUN'
00203             self.npass = [0,0,0,0]
00204             self.nfail = [1,0,0,0]
00205 
00206         if retStep2 != 0 :
00207             stat2 = 'FAILED'
00208             stat3 = 'NOTRUN'
00209             stat4 = 'NOTRUN'
00210             self.npass = [1,0,0,0]
00211             self.nfail = [0,1,0,0]
00212 
00213         if retStep3 != 0 :
00214             stat3 = 'FAILED'
00215             stat4 = 'NOTRUN'
00216             self.npass = [1,1,0,0]
00217             self.nfail = [0,0,1,0]
00218 
00219         if retStep4 != 0 :
00220             stat4 = 'FAILED'
00221             self.npass = [1,1,1,0]
00222             self.nfail = [0,0,0,1]
00223 
00224         logStat = 'Step1-'+stat1+' Step2-'+stat2+' Step3-'+stat3+' '+' Step4-'+stat4+' ' 
00225         self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, tottime, retStep1,retStep2,retStep3, retStep4)
00226 
00227         return
00228 
00229 
00230 # ================================================================================
00231 
00232 class WorkFlow(object):
00233 
00234     def __init__(self, num, nameID, cmd1, cmd2=None, cmd3=None, cmd4=None, real=None):
00235         self.numId  = num.strip()
00236         self.nameId = nameID
00237         self.cmdStep1 = cmd1
00238         if self.cmdStep1: self.cmdStep1 = self.cmdStep1.replace('--no_exec', '') # make sure the commands execute
00239         self.cmdStep2 = cmd2
00240         if self.cmdStep2: self.cmdStep2 = self.cmdStep2.replace('--no_exec', '') # make sure the commands execute
00241         self.cmdStep3 = cmd3
00242         if self.cmdStep3: self.cmdStep3 = self.cmdStep3.replace('--no_exec', '') # make sure the commands execute
00243         self.cmdStep4 = cmd4
00244         if self.cmdStep4: self.cmdStep4 = self.cmdStep4.replace('--no_exec', '') # make sure the commands execute
00245 
00246         # run on real data requested:
00247         self.real = real
00248         return
00249 
00250 # ================================================================================
00251 
00252 class MatrixReader(object):
00253 
00254     def __init__(self):
00255 
00256         self.reset()
00257 
00258         return
00259 
00260     def reset(self):
00261 
00262         self.step1WorkFlows = {}
00263         self.step2WorkFlows = {}
00264         self.step3WorkFlows = {}
00265         self.step4WorkFlows = {}
00266 
00267         self.workFlows = []
00268         self.nameList  = {}
00269         
00270 
00271         self.filesPrefMap = {'cmsDriver_standard_hlt.txt' : 'std-' ,
00272                              'cmsDriver_highstats_hlt.txt': 'hi-'  ,
00273                              'cmsDriver_generator.txt'    : 'gen-'  ,
00274                              # 'cmsDriver_PileUp_hlt.txt'   : 'pu-'  ,
00275                              # 'cmsDriver_realData_hlt.txt' : 'data-'
00276                              }
00277 
00278         self.files = ['cmsDriver_standard_hlt.txt' ,
00279                       'cmsDriver_highstats_hlt.txt',
00280                       'cmsDriver_generator.txt'    ,
00281                       # 'cmsDriver_PileUp_hlt.txt'   ,
00282                       # 'cmsDriver_realData_hlt.txt' 
00283                       ]
00284         
00285         return
00286 
00287     def readMatrix(self, fileNameIn):
00288         
00289         prefix = self.filesPrefMap[fileNameIn]
00290 
00291         print "processing ", fileNameIn,
00292         lines = []
00293         try:
00294             try:
00295                 inFile = open(fileNameIn, 'r')
00296                 print ' from local developer area',
00297             except IOError:
00298                 baseRelPath = os.environ['CMSSW_BASE']
00299                 # print "trying fall-back to cmsDriver files from developer area at:", baseRelPath
00300                 try:
00301                     inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
00302                     print ' from ', baseRelPath,
00303                 except IOError:
00304                     baseRelPath = os.environ['CMSSW_RELEASE_BASE']
00305                     # print "trying fall back to cmsDriver files from base release at:", baseRelPath
00306                     inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
00307                     print ' from ', baseRelPath,
00308             lines = inFile.readlines()
00309             inFile.close()
00310         except Exception, e:
00311             print "ERROR reading in file ", fileNameIn, str(e)
00312             return
00313 
00314         print " found ", len(lines), 'entries.'
00315         
00316         realRe = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
00317         step1Re = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
00318         step2Re = re.compile('\s*STEP2\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00319         step3Re = re.compile('\s*STEP3\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00320         step4Re = re.compile('\s*STEP4\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
00321         for lineIn in lines:
00322             line = lineIn.strip()
00323 
00324             realMatch = realRe.match(line)
00325             if realMatch :
00326                 num  = realMatch.group(1).strip()
00327                 name = realMatch.group(2).strip().replace('<','').replace('>','').replace(':','')
00328                 next = realMatch.group(3).strip().replace('+','').replace(',', ' ')
00329                 cmd  = realMatch.group(4).strip()
00330 
00331                 step2 = "None"
00332                 step3 = "None"
00333                 step4 = "None"
00334 
00335                 steps = next.split()
00336                 if len(steps) > 0:
00337                     step2 = steps[0].strip()
00338                 if len(steps) > 1:
00339                     step3 = steps[1].strip()
00340                 if len(steps) > 2:
00341                     step4 = steps[2].strip()
00342                 
00343                 self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
00344                 continue
00345             
00346                 
00347             step1Match = step1Re.match(line)
00348             if step1Match :
00349                 num  = step1Match.group(1).strip()
00350                 name = step1Match.group(2).strip().replace('<','').replace('>','').replace(':','')
00351                 next = step1Match.group(3).strip().replace('+','').replace(',', ' ')
00352                 cmd  = step1Match.group(4).strip()
00353                 step2 = "None"
00354                 step3 = "None"
00355                 step4 = "None"
00356 
00357                 steps = next.split()
00358                 if len(steps) > 0:
00359                     step2 = steps[0].strip()
00360                 if len(steps) > 1:
00361                     step3 = steps[1].strip()
00362                 if len(steps) > 2:
00363                     step4 = steps[2].strip()
00364                 
00365                 self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
00366                 continue
00367             
00368             step2Match = step2Re.match(line)
00369             if step2Match :
00370                 name = step2Match.group(1).strip()
00371                 cmd  = step2Match.group(2).strip()
00372                 self.step2WorkFlows[name] = (cmd.replace('--no_exec','') ) # make sure the command is really run
00373                 continue
00374 
00375             step3Match = step3Re.match(line)
00376             if step3Match :
00377                 name = step3Match.group(1).strip()
00378                 cmd  = step3Match.group(2).strip()
00379                 self.step3WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
00380                 continue
00381 
00382             step4Match = step4Re.match(line)
00383             if step4Match :
00384                 name = step4Match.group(1).strip()
00385                 cmd  = step4Match.group(2).strip()
00386                 self.step4WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
00387                 continue
00388 
00389         return
00390 
00391     def showRaw(self):
00392 
00393         print "found ", len(self.step1WorkFlows.keys()), ' workflows for step1:'
00394         ids = self.step1WorkFlows.keys()
00395         ids.sort()
00396         for key in ids:
00397             val = self.step1WorkFlows[key]
00398             print key[0], ':', val
00399         
00400         print "found ", len(self.step2WorkFlows.keys()), ' workflows for step2:'
00401         for key, val in self.step2WorkFlows.items():
00402             print key, ':', val
00403         
00404         print "found ", len(self.step3WorkFlows.keys()), ' workflows for step3:'
00405         for key, val in self.step3WorkFlows.items():
00406             print key, ':', val
00407         
00408         print "found ", len(self.step4WorkFlows.keys()), ' workflows for step4:'
00409         for key, val in self.step4WorkFlows.items():
00410             print key, ':', val
00411         
00412         return
00413 
00414     def showWorkFlows(self, selected=None):
00415 
00416         maxLen = 100 # for summary, limit width of output
00417         fmt1   = "%-6s %-35s [1]: %s ..."
00418         fmt2   = "       %35s [%d]: %s ..."
00419         print "\nfound a total of ", len(self.workFlows), ' workflows:'
00420         if selected:
00421             print "      of which the following", len(selected), 'were selected:'
00422             maxLen = -1  # for individual listing, no limit on width
00423             fmt1   = "%-6s %-35s [1]: %s " 
00424             fmt2   = "       %35s [%d]: %s"
00425         n1 = 0
00426         n2 = 0
00427         n3 = 0
00428         n4 = 0
00429         for wf in self.workFlows:
00430             if selected and float(wf.numId) not in selected: continue
00431             n1+=1
00432             print fmt1 % (wf.numId, wf.nameId, (wf.cmdStep1+' ')[:maxLen])
00433             if wf.cmdStep2:
00434                 n2+=1
00435                 print fmt2 % ( ' ', 2, (wf.cmdStep2+' ')[:maxLen])
00436                 if wf.cmdStep3:
00437                     n3+=1
00438                     print fmt2 % ( ' ', 3, (wf.cmdStep3+' ')[:maxLen])
00439                     if wf.cmdStep4:
00440                         n4+=1
00441                         print fmt2 % ( ' ', 4, (wf.cmdStep4+' ')[:maxLen])
00442 
00443         print n1, 'workflows for step1,'
00444         print n2, 'workflows for step1 + step2,'
00445         print n3, 'workflows for step1 + step2 + step3'
00446         print n4, 'workflows for step1 + step2 + step3 + step4'
00447 
00448         return
00449     
00450     def createWorkFlows(self, fileNameIn):
00451 
00452         prefixIn = self.filesPrefMap[fileNameIn]
00453 
00454         # get through the list of items and update the requested workflows only
00455         keyList = self.step1WorkFlows.keys()
00456         ids = []
00457         for item in keyList:
00458             id, pref = item
00459             if pref != prefixIn : continue
00460             ids.append( float(id) )
00461             
00462         ids.sort()
00463         n1 = 0
00464         n2 = 0
00465         n3 = 0
00466         n4 = 0
00467         for key in ids:
00468             val = self.step1WorkFlows[(key,prefixIn)]
00469             num, name, step2, step3, step4, cmd, real = val
00470             nameId = num+'_'+name
00471             if step2.lower() != 'none':
00472                 name += '+'+step2
00473                 if step3.lower() != 'none':
00474                     name += '+'+step3
00475                     if step4.lower() != 'none':
00476                         name += '+'+step4
00477             if nameId in self.nameList.keys():
00478                 print "==> duplicate name found for ", nameId
00479                 print '    keeping  : ', self.nameList[nameId]
00480                 print '    ignoring : ', val
00481             else:
00482                 self.nameList[nameId] = val
00483 
00484             cmd2 = None
00485             cmd3 = None
00486             cmd4 = None
00487             
00488             n1 += 1
00489 
00490             if step2.lower() != 'none':
00491                 n2 += 1
00492                 cmd2 = self.step2WorkFlows[step2]
00493                 if step3.lower() != 'none':
00494                     n3 += 1
00495                     cmd3 = self.step3WorkFlows[step3]
00496                     if step4.lower() != 'none':
00497                         n4 += 1
00498                         cmd4 = self.step4WorkFlows[step4]
00499                     #print '\tstep3 : ', self.step3WorkFlows[step3]
00500             self.workFlows.append( WorkFlow(num, name, cmd, cmd2, cmd3, cmd4) )
00501 
00502         return
00503 
00504     def prepare(self):
00505         
00506         for matrixFile in self.files:
00507             try:
00508                 self.readMatrix(matrixFile)
00509             except Exception, e:
00510                 print "ERROR reading file:", matrixFile, str(e)
00511 
00512             try:
00513                 self.createWorkFlows(matrixFile)
00514             except Exception, e:
00515                 print "ERROR creating workflows :", str(e)
00516 
00517     def show(self, selected=None):    
00518         # self.showRaw()
00519         self.showWorkFlows(selected)
00520         print '\n','-'*80,'\n'
00521 
00522 
00523     def updateDB(self):
00524 
00525         import pickle
00526         pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
00527 
00528         return
00529 
00530 # ================================================================================
00531 
00532 class MatrixRunner(object):
00533 
00534     def __init__(self, wfIn=None, nThrMax=8):
00535 
00536         self.workFlows = wfIn
00537 
00538         self.threadList = []
00539         self.maxThreads = int(nThrMax) # make sure we get a number ...
00540 
00541 
00542     def activeThreads(self):
00543 
00544         nActive = 0
00545         for t in self.threadList:
00546             if t.isAlive() : nActive += 1
00547 
00548         return nActive
00549 
00550         
00551     def runTests(self, testList=None):
00552 
00553         startDir = os.getcwd()
00554 
00555         # make sure we have a way to set the environment in the threads ...
00556         if not os.environ.has_key('CMS_PATH'):
00557             cmsPath = '/afs/cern.ch/cms'
00558             print "setting default for CMS_PATH to", cmsPath
00559             os.environ['CMS_PATH'] = cmsPath
00560 
00561         report=''       
00562         print 'Running in %s thread(s)' % self.maxThreads
00563                 
00564         for wf in self.workFlows:
00565 
00566             if testList and float(wf.numId) not in [float(x) for x in testList]: continue
00567 
00568             # don't know yet how to treat real data WF ...
00569             if wf.real : continue
00570 
00571             item = wf.nameId
00572             if os.path.islink(item) : continue # ignore symlinks
00573             
00574             # make sure we don't run more than the allowed number of threads:
00575             while self.activeThreads() >= self.maxThreads:
00576                 time.sleep(10)
00577                 continue
00578             
00579             print '\nPreparing to run %s %s' % (wf.numId, item)
00580           
00581 ##            if testList: # if we only run a selection, run only 5 events instead of 10
00582 ##                wf.cmdStep1 = wf.cmdStep1.replace('-n 10', '-n 5')
00583                 
00584             current = WorkFlowRunner(wf)
00585             self.threadList.append(current)
00586             current.start()
00587             time.sleep(random.randint(1,5)) # try to avoid race cond by sleeping random amount of time [1,5] sec 
00588 
00589         # wait until all threads are finished
00590         while self.activeThreads() > 0:
00591             time.sleep(5)
00592             
00593         # all threads are done now, check status ...
00594         nfail1 = 0
00595         nfail2 = 0
00596         nfail3 = 0
00597         nfail4 = 0
00598         npass  = 0
00599         npass1 = 0
00600         npass2 = 0
00601         npass3 = 0
00602         npass4 = 0
00603         for pingle in self.threadList:
00604             pingle.join()
00605             try:
00606                 nfail1 += pingle.nfail[0]
00607                 nfail2 += pingle.nfail[1]
00608                 nfail3 += pingle.nfail[2]
00609                 nfail4 += pingle.nfail[3]
00610                 npass1 += pingle.npass[0]
00611                 npass2 += pingle.npass[1]
00612                 npass3 += pingle.npass[2]
00613                 npass4 += pingle.npass[3]
00614                 npass  += npass1+npass2+npass3+npass4
00615                 report += pingle.report
00616                 # print pingle.report
00617             except Exception, e:
00618                 msg = "ERROR retrieving info from thread: " + str(e)
00619                 nfail1 += 1
00620                 nfail2 += 1
00621                 nfail3 += 1
00622                 nfail4 += 1
00623                 report += msg
00624                 print msg
00625                 
00626         report+='\n %s %s %s %s tests passed, %s %s %s %s failed\n' %(npass1, npass2, npass3, npass4, nfail1, nfail2, nfail3, nfail4)
00627         print report
00628         
00629         runall_report_name='runall-report-step123-.log'
00630         runall_report=open(runall_report_name,'w')
00631         runall_report.write(report)
00632         runall_report.close()
00633 
00634         os.chdir(startDir)
00635         
00636         return
00637 
00638         
00639 # ================================================================================
00640 
00641 def runSelected(testList, nThreads=4, show=False) :
00642 
00643     stdList = ['5.2', # SingleMu10 FastSim
00644                '7',   # Cosmics+RECOCOS+ALCACOS
00645                '8',   # BeamHalo+RECOCOS+ALCABH
00646                '25',  # TTbar+RECO2+ALCATT2  STARTUP
00647                ]
00648     hiStatList = [
00649                   '121',   # TTbar_Tauola
00650                   '123.3', # TTBar FastSim
00651                    ]
00652 
00653     mrd = MatrixReader()
00654     mrd.prepare()
00655 
00656     if testList == []:
00657         testList = stdList+hiStatList
00658 
00659     ret = 0
00660     if show:
00661         mrd.show([float(x) for x in testList])
00662         print 'selected items:', testList
00663     else:
00664         mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
00665         ret = mRunnerHi.runTests(testList)
00666 
00667     return ret
00668 
00669 # ================================================================================
00670 
00671 def runData(testList, nThreads=4, show=False) :
00672 
00673     mrd = MatrixReader()
00674 
00675     mrd.prepare()
00676 
00677     ret = 0
00678     if show:
00679         if not testList or testList == ['all']:
00680             mrd.show()
00681         else:
00682             mrd.show([float(x) for x in testList])
00683         print 'selected items:', testList
00684     else:
00685         mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
00686         if not testList or testList == ['all']:
00687             ret = mRunnerHi.runTests()
00688         else:
00689             ret = mRunnerHi.runTests(testList)
00690 
00691     return ret
00692 
00693 # --------------------------------------------------------------------------------
00694 
00695 def runAll(testList=None, nThreads=4, show=False) :
00696 
00697     mrd = MatrixReader()
00698     mrd.prepare()
00699 
00700     ret = 0
00701     
00702     if show:
00703         mrd.show()
00704         print "nThreads = ",nThreads
00705     else:
00706         mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
00707         ret = mRunnerHi.runTests()
00708 
00709     return ret
00710 
00711 
00712 # --------------------------------------------------------------------------------
00713 
00714 def runOnly(only, show, nThreads=4):
00715 
00716     if not only: return
00717     
00718     for what in only:
00719         print "found request to run relvals only for ",what
00720 
00721 
00722 # --------------------------------------------------------------------------------
00723 
00724 def usage():
00725     print "Usage:", sys.argv[0], ' [options] '
00726     print """
00727 Where options is one of the following:
00728   -d, --data <list> comma-separated list of workflows to use from the realdata file.
00729                     <list> can be "all" to select all data workflows
00730   -l, --list <list> comma-separated list of workflows to use from the cmsDriver*.txt files
00731   -j, --nproc <n>   run <n> processes in parallel (default: 4 procs)
00732   -s, --selected    run a subset of 8 workflows (usually in the CustomIB)
00733   -n, -q, --show    show the (selected) workflows
00734 
00735 <list>s should be put in single- or double-quotes to avoid confusion with/by the shell
00736 """
00737 
00738 # ================================================================================
00739 
00740 if __name__ == '__main__':
00741 
00742     import getopt
00743     
00744     try:
00745         opts, args = getopt.getopt(sys.argv[1:], "hj:sl:nqo:d:", ['help',"nproc=",'selected','list=','showMatrix','only=','data='])
00746     except getopt.GetoptError, e:
00747         print "unknown option", str(e)
00748         sys.exit(2)
00749         
00750 # check command line parameter
00751 
00752     np=4 # default: four threads
00753     sel = None
00754     show = False
00755     only = None
00756     data = None
00757     for opt, arg in opts :
00758         if opt in ('-h','--help'):
00759             usage()
00760             sys.exit(0)
00761         if opt in ('-j', "--nproc" ):
00762             np=int(arg)
00763         if opt in ('-n','-q','--showMatrix', ):
00764             show = True
00765         if opt in ('-s','--selected',) :
00766             sel = []
00767         if opt in ('-o','--only',) :
00768             only = []
00769         if opt in ('-l','--list',) :
00770             sel = arg.split(',')
00771         if opt in ('-d','--data',) :
00772             data = arg.split(',')
00773 
00774     # print "sel",sel
00775     ret = 0
00776     if sel != None: # explicit distinguish from empty list (which is also false)
00777         ret = runSelected(testList=sel, nThreads=np, show=show)
00778     elif only != None:
00779         ret = runOnly(only=only, show=show, nThreads=np)
00780     elif data != None:
00781         ret = runData(testList=data, show=show, nThreads=np)
00782     else:
00783         ret = runAll(show=show, nThreads=np)
00784 
00785     sys.exit(ret)
/data/refman/pasoursint/CMSSW_4_1_8_patch13/src/Configuration/PyReleaseValidation/scripts/runTheMatrix.py
