Inherits threading::Thread.

Public Member Functions
def	__init__
def	doCmd
def	run
Public Attributes
	nfail
	npass
	report
	status
	wf

Detailed Description

Definition at line 8 of file runTheMatrix.py.

Constructor & Destructor Documentation

def runTheMatrix::WorkFlowRunner::__init__	(	self,
		wf
	)

Definition at line 9 of file runTheMatrix.py.

00010                           :
00011         Thread.__init__(self)
00012         self.wf = wf
00013 
00014         self.status=-1
00015         self.report=''
00016         self.nfail=0
00017         self.npass=0
00018 
00019         return

Member Function Documentation

def runTheMatrix::WorkFlowRunner::doCmd	(	self,
		cmd,
		dryRun = `False`
	)

Definition at line 20 of file runTheMatrix.py.

00021                                       :
00022 
00023         msg = "\n# in: " +os.getcwd()
00024         if dryRun: msg += " dryRun for '"
00025         else:      msg += " going to execute "
00026         msg += cmd.replace(';','\n')
00027         print msg
00028 
00029         cmdLog = open(self.wf.numId+'_'+self.wf.nameId+'/cmdLog','a')
00030         cmdLog.write(msg+'\n')
00031         cmdLog.close()
00032         
00033         ret = 0
00034         if not dryRun:
00035             ret = os.system(cmd)
00036             if ret != 0:
00037                 print "ERROR executing ",cmd,'ret=', ret
00038 
00039         return ret

def runTheMatrix::WorkFlowRunner::run ( self )

Definition at line 40 of file runTheMatrix.py.

00041                  :
00042 
00043         startDir = os.getcwd()
00044 
00045         wfDir = self.wf.numId+'_'+self.wf.nameId
00046         if not os.path.exists(wfDir):
00047             os.makedirs(wfDir)
00048 
00049         preamble = ''
00050         if os.path.exists( os.path.join(os.environ["CMS_PATH"],'cmsset_default.sh') ) :
00051             preamble = 'source $CMS_PATH/cmsset_default.sh; '
00052         else:
00053             preamble = 'source $CMS_PATH/sw/cmsset_default.sh; '
00054         preamble += 'eval `scram run -sh`; '
00055         preamble += 'cd '+wfDir+'; '
00056         preamble += 'ulimit -v 4069000;' # make sure processes keep within limits ...
00057         
00058         startime='date %s' %time.asctime()
00059 
00060         # set defaults for the statuses
00061         stat1 = 'PASSED'
00062         stat2 = 'PASSED' 
00063         stat3 = 'PASSED'
00064         stat4 = 'PASSED'
00065         if not self.wf.cmdStep2: stat2 = 'NOSTEP'
00066         if not self.wf.cmdStep3: stat3 = 'NOSTEP'
00067         if not self.wf.cmdStep4: stat4 = 'NOSTEP'
00068         
00069         # run the first workflow:
00070         cmd = preamble
00071 
00072         inFile = 'file:raw.root'
00073         if 'REALDATA' in self.wf.cmdStep1:
00074             realDataRe = re.compile('REALDATA:\s*(/[A-Za-z].*?),(\s*RUN:\s*(?P<run>\d+),)?(\s*FILES:\s*(?P<files>\d+),)?(\s*EVENTS:\s*(?P<events>\d+))?,\s*LABEL:\s*(?P<label>.*),\s*LOCATION:\s*(?P<location>.*)\s*')
00075             realDataMatch = realDataRe.match(self.wf.cmdStep1)
00076             if realDataMatch:
00077                 run = None
00078                 if realDataMatch.group("run") : run = realDataMatch.group("run")
00079                 label  = realDataMatch.group("label")
00080                 location = realDataMatch.group("location").lower().strip()
00081                 if 'caf' in location:
00082                     print "ignoring workflow ",self.wf.numId, self.wf.nameId, ' as this is on CAF ...'
00083                     self.npass = [0,0,0,0]
00084                     self.nfail = [0,0,0,0]
00085 
00086                     logStat = 'Step1-NOTRUN Step2-NOTRUN Step3-NOTRUN Step4-NOTRUN ' 
00087                     self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, 0, 0,0,0,0)
00088                     return
00089                 
00090                 files  = None
00091                 events = None
00092                 if realDataMatch.group("files"): 
00093                   files  = realDataMatch.group("files")
00094                 if realDataMatch.group("events"): 
00095                   events = realDataMatch.group("events")
00096                   if self.wf.cmdStep2 and ' -n ' not in self.wf.cmdStep2: self.wf.cmdStep2 += ' -n ' + events
00097                   if self.wf.cmdStep3 and ' -n ' not in self.wf.cmdStep3: self.wf.cmdStep3 += ' -n ' + events
00098                   if self.wf.cmdStep4 and ' -n ' not in self.wf.cmdStep4: self.wf.cmdStep4 += ' -n ' + events
00099 
00100                 print "run, files, events, label", run, files, events, label 
00101                 cmd += 'dbs search --noheader --url=https://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet '
00102                 cmd += "--query='find file where dataset like "+realDataMatch.group(1)
00103                 if run: cmd += " and run=" + run
00104                 cmd += "' "
00105                 cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'-dbsquery.log',)
00106                 retStep1 = self.doCmd(cmd)
00107                 if retStep1 == 0:
00108                     lf = open(wfDir+'/step1_'+self.wf.nameId+'-dbsquery.log', 'r')
00109                     lines = lf.readlines()
00110                     lf.close()
00111                     if not lines or len(lines)==0 :
00112                         inFile = "NoFileFoundInDBS"
00113                         retStep1 = -95
00114                     else:
00115                         try:
00116                             inFile = lines[0].strip()
00117                         except Exception, e:
00118                             print "ERROR determining file from DBS query: ", str(e)
00119                             inFile = "NoFileFoundInDBS"
00120                             retStep1 = -90
00121             else:
00122                 print "ERROR: found REALDATA in '"+self.wf.cmdStep1+"' but not RE match !!??!"
00123                 retStep1 = -99
00124         else:
00125             cmd += self.wf.cmdStep1 + ' --fileout file:raw.root '
00126             cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'.log ',)
00127             retStep1 = self.doCmd(cmd)
00128 
00129         print " ... ret: " , retStep1
00130 
00131         # prepare and run the next workflows -- if the previous step was OK :
00132         # set some defaults
00133         retStep2 = 0
00134         retStep3 = 0
00135         retStep4 = 0
00136         if self.wf.cmdStep2 and retStep1 == 0:
00137             fullcmd = preamble
00138             fullcmd += self.wf.cmdStep2
00139             if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00140             fullcmd += ' --fileout file:reco.root '
00141             print '=====>>> ', self.wf.nameId, self.wf.numId
00142 
00143             # for HI B0 step2 use a file from a previous relval production as step1 doesn't write
00144             # any output in 1 hour. Add himix flag here as this is only needed when run on the mixed
00145             # input files (the relvals are OK)
00146             if ( '40.0' in str(self.wf.numId) ) :
00147                 fullcmd += ' --himix '
00148                 inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/102FF831-9B0F-E011-A3E9-003048678BC6.root'
00149             if ( '41.0' in str(self.wf.numId) ) : 
00150                 fullcmd += ' --himix '
00151                 inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_GammaJet_pt20_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/06B4F699-A50F-E011-AD62-0018F3D0962E.root'
00152 
00153             fullcmd += ' --filein '+inFile+ ' '
00154             fullcmd += ' > %s 2>&1; ' % ('step2_'+self.wf.nameId+'.log ',)
00155             # print fullcmd
00156             retStep2 = self.doCmd(fullcmd)
00157 #            if random.randint(0,100) < 20 : retStep2 = -42
00158 
00159             if self.wf.cmdStep3 and retStep2 == 0:
00160                 fullcmd = preamble
00161                 fullcmd += self.wf.cmdStep3
00162                 if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00163                 # FIXME: dirty hack for beam-spot dedicated relval
00164                 if not '134' in str(self.wf.numId):
00165                     if 'HARVESTING' in fullcmd and not 'filein' in fullcmd:
00166                         fullcmd += ' --filein file:reco_inDQM.root --fileout file:step3.root '
00167                     else:
00168                         fullcmd += ' --filein file:reco.root --fileout file:step3.root '
00169                 fullcmd += ' > %s 2>&1; ' % ('step3_'+self.wf.nameId+'.log ',)
00170                 # print fullcmd
00171                 retStep3 = self.doCmd(fullcmd)
00172 #                if random.randint(0,100) < 40 : retStep3 = -42
00173                 if self.wf.cmdStep4 and retStep3 == 0:
00174                     fullcmd = preamble
00175                     fullcmd += self.wf.cmdStep4
00176                     if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00177                     # FIXME: dirty hack for beam-spot dedicated relval
00178                     if not '134' in str(self.wf.numId):
00179                         fullcmd += ' --filein file:step3.root '
00180                     fullcmd += ' > %s 2>&1; ' % ('step4_'+self.wf.nameId+'.log ',)
00181                     # print fullcmd
00182                     retStep4 = self.doCmd(fullcmd)
00183 #                    if random.randint(0,100) < 40 : retStep4 = -42
00184 
00185         os.chdir(startDir)
00186 
00187         endtime='date %s' %time.asctime()
00188         tottime='%s-%s'%(endtime,startime)
00189 
00190         self.nfail = [0,0,0,0]
00191         self.npass = [1,1,1,1]
00192         if 'NOSTEP' in stat2: # don't say reco/alca is passed if we don't have to run them
00193             self.npass = [1,0,0,0]
00194         else: # we have a reco step, check for alca:
00195             if 'NOSTEP' in stat3 :
00196                 self.npass = [1,1,0,0]
00197                 if 'NOSTEP' in stat4 :
00198                     self.npass = [1,1,1,0]
00199         if retStep1 != 0 :
00200             stat1 = 'FAILED'
00201             stat2 = 'NOTRUN'
00202             stat3 = 'NOTRUN'
00203             stat4 = 'NOTRUN'
00204             self.npass = [0,0,0,0]
00205             self.nfail = [1,0,0,0]
00206 
00207         if retStep2 != 0 :
00208             stat2 = 'FAILED'
00209             stat3 = 'NOTRUN'
00210             stat4 = 'NOTRUN'
00211             self.npass = [1,0,0,0]
00212             self.nfail = [0,1,0,0]
00213 
00214         if retStep3 != 0 :
00215             stat3 = 'FAILED'
00216             stat4 = 'NOTRUN'
00217             self.npass = [1,1,0,0]
00218             self.nfail = [0,0,1,0]
00219 
00220         if retStep4 != 0 :
00221             stat4 = 'FAILED'
00222             self.npass = [1,1,1,0]
00223             self.nfail = [0,0,0,1]
00224 
00225         logStat = 'Step1-'+stat1+' Step2-'+stat2+' Step3-'+stat3+' '+' Step4-'+stat4+' ' 
00226         self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, tottime, retStep1,retStep2,retStep3, retStep4)
00227 
00228         return
00229 
00230 
00231 # ================================================================================

Member Data Documentation

runTheMatrix::WorkFlowRunner::nfail

Definition at line 9 of file runTheMatrix.py.

runTheMatrix::WorkFlowRunner::npass

Definition at line 9 of file runTheMatrix.py.

runTheMatrix::WorkFlowRunner::report

Definition at line 9 of file runTheMatrix.py.

runTheMatrix::WorkFlowRunner::status

Definition at line 9 of file runTheMatrix.py.

runTheMatrix::WorkFlowRunner::wf

Definition at line 9 of file runTheMatrix.py.

runTheMatrix::WorkFlowRunner Class Reference

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation