Inherits threading::Thread.

Public Member Functions
def	__init__
def	doCmd
def	run
Public Attributes
	nfail
	npass
	report
	status
	wf

Detailed Description

Definition at line 8 of file runTheMatrix_dev.py.

Constructor & Destructor Documentation

def runTheMatrix_dev::WorkFlowRunner::__init__	(	self,
		wf
	)

Definition at line 9 of file runTheMatrix_dev.py.

00010                           :
00011         Thread.__init__(self)
00012         self.wf = wf
00013 
00014         self.status=-1
00015         self.report=''
00016         self.nfail=0
00017         self.npass=0
00018 
00019         return

Member Function Documentation

def runTheMatrix_dev::WorkFlowRunner::doCmd	(	self,
		cmd,
		dryRun = `False`
	)

Definition at line 20 of file runTheMatrix_dev.py.

00021                                       :
00022 
00023         msg = "\n# in: " +os.getcwd()
00024         if dryRun: msg += " dryRun for '"
00025         else:      msg += " going to execute "
00026         msg += cmd.replace(';','\n')
00027         print msg
00028 
00029         cmdLog = open(self.wf.numId+'_'+self.wf.nameId+'/cmdLog','a')
00030         cmdLog.write(msg+'\n')
00031         cmdLog.close()
00032         
00033         ret = 0
00034         if not dryRun:
00035             ret = os.system(cmd)
00036             if ret != 0:
00037                 print "ERROR executing ",cmd,'ret=', ret
00038 
00039         return ret

def runTheMatrix_dev::WorkFlowRunner::run ( self )

Definition at line 40 of file runTheMatrix_dev.py.

00041                  :
00042 
00043         startDir = os.getcwd()
00044 
00045         wfDir = self.wf.numId+'_'+self.wf.nameId
00046         if not os.path.exists(wfDir):
00047             os.makedirs(wfDir)
00048 
00049         preamble = ''
00050         if os.path.exists( os.path.join(os.environ["CMS_PATH"],'cmsset_default.sh') ) :
00051             preamble = 'source $CMS_PATH/cmsset_default.sh; '
00052         else:
00053             preamble = 'source $CMS_PATH/sw/cmsset_default.sh; '
00054         preamble += 'eval `scram run -sh`; '
00055         preamble += 'cd '+wfDir+'; '
00056         preamble += 'ulimit -v 4069000;' # make sure processes keep within limits ...
00057         
00058         startime='date %s' %time.asctime()
00059 
00060         # set defaults for the statuses
00061         stat1 = 'PASSED'
00062         stat2 = 'PASSED' 
00063         stat3 = 'PASSED'
00064         stat4 = 'PASSED'
00065         if not self.wf.cmdStep2: stat2 = 'NOSTEP'
00066         if not self.wf.cmdStep3: stat3 = 'NOSTEP'
00067         if not self.wf.cmdStep4: stat4 = 'NOSTEP'
00068         
00069         # run the first workflow:
00070         cmd = preamble
00071 
00072         inFile = 'file:raw.root'
00073         if self.wf.cmdStep1.startswith('DATAINPUT'):
00074             print "going to run with file input ... "
00075             if self.wf.input.run:
00076                 run      = str(self.wf.input.run)
00077             else:
00078                 run=None
00079 
00080             label    = self.wf.input.label
00081             location = self.wf.input.location.lower().strip()
00082             if 'caf' in location:
00083                 print "ignoring workflow ",self.wf.numId, self.wf.nameId, ' as this is on CAF ...'
00084                 self.npass = [0,0,0,0]
00085                 self.nfail = [0,0,0,0]
00086 
00087                 logStat = 'Step1-NOTRUN Step2-NOTRUN Step3-NOTRUN Step4-NOTRUN ' 
00088                 self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, 0, 0,0,0,0)
00089                 return
00090                 
00091             files  = str(self.wf.input.files)
00092             events = '10' # ignore the give number ...    str(self.wf.input.events)
00093             if self.wf.cmdStep2 and ' -n ' not in self.wf.cmdStep2: self.wf.cmdStep2 += ' -n ' + events
00094             if self.wf.cmdStep3 and ' -n ' not in self.wf.cmdStep3: self.wf.cmdStep3 += ' -n ' + events
00095             if self.wf.cmdStep4 and ' -n ' not in self.wf.cmdStep4: self.wf.cmdStep4 += ' -n ' + events
00096 
00097             print "run, files, events, label", run, files, events, label 
00098             cmd += 'dbs search --noheader --url=https://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet '
00099             cmd += "--query='find file where dataset like "+self.wf.input.dataSet
00100             if run: cmd += " and run=" + run
00101             cmd += "' "
00102             cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'-dbsquery.log',)
00103             retStep1 = self.doCmd(cmd)
00104             if retStep1 == 0:
00105                 lf = open(wfDir+'/step1_'+self.wf.nameId+'-dbsquery.log', 'r')
00106                 lines = lf.readlines()
00107                 lf.close()
00108                 if not lines or len(lines)==0 :
00109                     inFile = "NoFileFoundInDBS"
00110                     retStep1 = -95
00111                 else:
00112                     try:
00113                         inFile = lines[0].strip()
00114                     except Exception, e:
00115                         print "ERROR determining file from DBS query: ", str(e)
00116                         inFile = "NoFileFoundInDBS"
00117                         retStep1 = -90
00118         else:
00119             cmd += self.wf.cmdStep1 + ' --fileout file:raw.root '
00120             cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'.log ',)
00121             retStep1 = self.doCmd(cmd)
00122 
00123         print " ... ret: " , retStep1
00124 
00125         # prepare and run the next workflows -- if the previous step was OK :
00126         # set some defaults
00127         retStep2 = 0
00128         retStep3 = 0
00129         retStep4 = 0
00130         if self.wf.cmdStep2 and retStep1 == 0:
00131             fullcmd = preamble
00132             fullcmd += self.wf.cmdStep2
00133             if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00134             fullcmd += ' --fileout file:reco.root '
00135             print '=====>>> ', self.wf.nameId, self.wf.numId
00136 
00137             # for HI B0 step2 use a file from a previous relval production as step1 doesn't write
00138             # any output in 1 hour. Add himix flag here as this is only needed when run on the mixed
00139             # input files (the relvals are OK)
00140             if ( '40.0' in str(self.wf.numId) ) :
00141                 fullcmd += ' --himix '
00142                 inFile = '/store/relval/CMSSW_3_8_0_pre1/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM-RAW/MC_37Y_V5-v1/0001/E0DE7C01-2C6F-DF11-B61F-0026189438F4.root'
00143             if ( '41.0' in str(self.wf.numId) ) : 
00144                 fullcmd += ' --himix '
00145                 inFile = '/store/relval/CMSSW_3_8_0_pre1/RelValPyquen_GammaJet_pt20_2760GeV/GEN-SIM-RAW/MC_37Y_V5-v1/0001/F68A53A5-2B6F-DF11-8958-003048678FE6.root'
00146 
00147             fullcmd += ' --filein '+inFile+ ' '
00148             fullcmd += ' > %s 2>&1; ' % ('step2_'+self.wf.nameId+'.log ',)
00149             # print fullcmd
00150             retStep2 = self.doCmd(fullcmd)
00151 #            if random.randint(0,100) < 20 : retStep2 = -42
00152 
00153             if self.wf.cmdStep3 and retStep2 == 0:
00154                 fullcmd = preamble
00155                 fullcmd += self.wf.cmdStep3
00156                 if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00157                 # FIXME: dirty hack for beam-spot dedicated relval
00158                 if not '134' in str(self.wf.numId):
00159                     fullcmd += ' --filein file:reco.root --fileout file:step3.root '
00160                 fullcmd += ' > %s 2>&1; ' % ('step3_'+self.wf.nameId+'.log ',)
00161                 # print fullcmd
00162                 retStep3 = self.doCmd(fullcmd)
00163 #                if random.randint(0,100) < 40 : retStep3 = -42
00164                 if self.wf.cmdStep4 and retStep3 == 0:
00165                     fullcmd = preamble
00166                     fullcmd += self.wf.cmdStep4
00167                     if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
00168                     # FIXME: dirty hack for beam-spot dedicated relval
00169                     if not '134' in str(self.wf.numId):
00170                         fullcmd += ' --filein file:step3.root '
00171                     fullcmd += ' > %s 2>&1; ' % ('step4_'+self.wf.nameId+'.log ',)
00172                     # print fullcmd
00173                     retStep4 = self.doCmd(fullcmd)
00174 #                    if random.randint(0,100) < 40 : retStep4 = -42
00175 
00176         os.chdir(startDir)
00177 
00178         endtime='date %s' %time.asctime()
00179         tottime='%s-%s'%(endtime,startime)
00180 
00181         self.nfail = [0,0,0,0]
00182         self.npass = [1,1,1,1]
00183         if 'NOSTEP' in stat2: # don't say reco/alca is passed if we don't have to run them
00184             self.npass = [1,0,0,0]
00185         else: # we have a reco step, check for alca:
00186             if 'NOSTEP' in stat3 :
00187                 self.npass = [1,1,0,0]
00188                 if 'NOSTEP' in stat4 :
00189                     self.npass = [1,1,1,0]
00190         if retStep1 != 0 :
00191             stat1 = 'FAILED'
00192             stat2 = 'NOTRUN'
00193             stat3 = 'NOTRUN'
00194             stat4 = 'NOTRUN'
00195             self.npass = [0,0,0,0]
00196             self.nfail = [1,0,0,0]
00197 
00198         if retStep2 != 0 :
00199             stat2 = 'FAILED'
00200             stat3 = 'NOTRUN'
00201             stat4 = 'NOTRUN'
00202             self.npass = [1,0,0,0]
00203             self.nfail = [0,1,0,0]
00204 
00205         if retStep3 != 0 :
00206             stat3 = 'FAILED'
00207             stat4 = 'NOTRUN'
00208             self.npass = [1,1,0,0]
00209             self.nfail = [0,0,1,0]
00210 
00211         if retStep4 != 0 :
00212             stat4 = 'FAILED'
00213             self.npass = [1,1,1,0]
00214             self.nfail = [0,0,0,1]
00215 
00216         logStat = 'Step1-'+stat1+' Step2-'+stat2+' Step3-'+stat3+' '+' Step4-'+stat4+' ' 
00217         self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, tottime, retStep1,retStep2,retStep3, retStep4)
00218 
00219         return
00220 
00221 
00222 # ================================================================================

Member Data Documentation

runTheMatrix_dev::WorkFlowRunner::nfail

Definition at line 9 of file runTheMatrix_dev.py.

runTheMatrix_dev::WorkFlowRunner::npass

Definition at line 9 of file runTheMatrix_dev.py.

runTheMatrix_dev::WorkFlowRunner::report

Definition at line 9 of file runTheMatrix_dev.py.

runTheMatrix_dev::WorkFlowRunner::status

Definition at line 9 of file runTheMatrix_dev.py.

runTheMatrix_dev::WorkFlowRunner::wf

Definition at line 9 of file runTheMatrix_dev.py.

runTheMatrix_dev::WorkFlowRunner Class Reference

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation