CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
runTheMatrix.WorkFlowRunner Class Reference
Inheritance diagram for runTheMatrix.WorkFlowRunner:

Public Member Functions

def __init__
 
def doCmd
 
def run
 

Public Attributes

 nfail
 
 npass
 
 report
 
 status
 
 wf
 

Detailed Description

Definition at line 8 of file runTheMatrix.py.

Constructor & Destructor Documentation

def runTheMatrix.WorkFlowRunner.__init__ (   self,
  wf 
)

Definition at line 9 of file runTheMatrix.py.

9 
10  def __init__(self, wf):
11  Thread.__init__(self)
12  self.wf = wf
13 
14  self.status=-1
15  self.report=''
16  self.nfail=0
17  self.npass=0
18 
19  return

Member Function Documentation

def runTheMatrix.WorkFlowRunner.doCmd (   self,
  cmd,
  dryRun = False 
)

Definition at line 20 of file runTheMatrix.py.

Referenced by runTheMatrix.WorkFlowRunner.run(), and runTheMatrix_dev.WorkFlowRunner.run().

20 
21  def doCmd(self, cmd, dryRun=False):
22 
23  msg = "\n# in: " +os.getcwd()
24  if dryRun: msg += " dryRun for '"
25  else: msg += " going to execute "
26  msg += cmd.replace(';','\n')
27  print msg
28 
29  cmdLog = open(self.wf.numId+'_'+self.wf.nameId+'/cmdLog','a')
30  cmdLog.write(msg+'\n')
31  cmdLog.close()
32 
33  ret = 0
34  if not dryRun:
35  ret = os.system(cmd)
36  if ret != 0:
37  print "ERROR executing ",cmd,'ret=', ret
38 
39  return ret
def runTheMatrix.WorkFlowRunner.run (   self)

Definition at line 40 of file runTheMatrix.py.

References runTheMatrix.WorkFlowRunner.doCmd(), runall.testit.nfail, runTheMatrix.WorkFlowRunner.nfail, runall.testit.npass, runTheMatrix.WorkFlowRunner.npass, TShapeAnalysis.npass, runall.testit.report, runTheMatrix.WorkFlowRunner.report, ALIUtils.report, and strip().

Referenced by Types.LuminosityBlockID.cppID().

40 
41  def run(self):
42 
43  startDir = os.getcwd()
44 
45  wfDir = self.wf.numId+'_'+self.wf.nameId
46  if not os.path.exists(wfDir):
47  os.makedirs(wfDir)
48 
49  preamble = ''
50  if os.path.exists( os.path.join(os.environ["CMS_PATH"],'cmsset_default.sh') ) :
51  preamble = 'source $CMS_PATH/cmsset_default.sh; '
52  else:
53  preamble = 'source $CMS_PATH/sw/cmsset_default.sh; '
54  preamble += 'eval `scram run -sh`; '
55  preamble += 'cd '+wfDir+'; '
56  preamble += 'ulimit -v 4069000;' # make sure processes keep within limits ...
57 
58  startime='date %s' %time.asctime()
59 
60  # set defaults for the statuses
61  stat1 = 'PASSED'
62  stat2 = 'PASSED'
63  stat3 = 'PASSED'
64  stat4 = 'PASSED'
65  if not self.wf.cmdStep2: stat2 = 'NOSTEP'
66  if not self.wf.cmdStep3: stat3 = 'NOSTEP'
67  if not self.wf.cmdStep4: stat4 = 'NOSTEP'
68 
69  # run the first workflow:
70  cmd = preamble
71 
72  inFile = 'file:raw.root'
73  if 'REALDATA' in self.wf.cmdStep1:
74  realDataRe = re.compile('REALDATA:\s*(/[A-Za-z].*?),(\s*RUN:\s*(?P<run>\d+),)?(\s*FILES:\s*(?P<files>\d+),)?(\s*EVENTS:\s*(?P<events>\d+))?,\s*LABEL:\s*(?P<label>.*),\s*LOCATION:\s*(?P<location>.*)\s*')
75  realDataMatch = realDataRe.match(self.wf.cmdStep1)
76  if realDataMatch:
77  run = None
78  if realDataMatch.group("run") : run = realDataMatch.group("run")
79  label = realDataMatch.group("label")
80  location = realDataMatch.group("location").lower().strip()
81  if 'caf' in location:
82  print "ignoring workflow ",self.wf.numId, self.wf.nameId, ' as this is on CAF ...'
83  self.npass = [0,0,0,0]
84  self.nfail = [0,0,0,0]
85 
86  logStat = 'Step1-NOTRUN Step2-NOTRUN Step3-NOTRUN Step4-NOTRUN '
87  self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, 0, 0,0,0,0)
88  return
89 
90  files = None
91  events = None
92  if realDataMatch.group("files"):
93  files = realDataMatch.group("files")
94  if realDataMatch.group("events"):
95  events = realDataMatch.group("events")
96  if self.wf.cmdStep2 and ' -n ' not in self.wf.cmdStep2: self.wf.cmdStep2 += ' -n ' + events
97  if self.wf.cmdStep3 and ' -n ' not in self.wf.cmdStep3: self.wf.cmdStep3 += ' -n ' + events
98  if self.wf.cmdStep4 and ' -n ' not in self.wf.cmdStep4: self.wf.cmdStep4 += ' -n ' + events
99 
100  print "run, files, events, label", run, files, events, label
101  cmd += 'dbs search --noheader --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet '
102  cmd += "--query='find file where dataset like "+realDataMatch.group(1)
103  if run: cmd += " and run=" + run
104  cmd += "' "
105  cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'-dbsquery.log',)
106  retStep1 = self.doCmd(cmd)
107  if retStep1 == 0:
108  lf = open(wfDir+'/step1_'+self.wf.nameId+'-dbsquery.log', 'r')
109  lines = lf.readlines()
110  lf.close()
111  if not lines or len(lines)==0 :
112  inFile = "NoFileFoundInDBS"
113  retStep1 = -95
114  else:
115  try:
116  inFile = lines[0].strip()
117  except Exception, e:
118  print "ERROR determining file from DBS query: ", str(e)
119  inFile = "NoFileFoundInDBS"
120  retStep1 = -90
121  else:
122  print "ERROR: found REALDATA in '"+self.wf.cmdStep1+"' but not RE match !!??!"
123  retStep1 = -99
124  else:
125  cmd += self.wf.cmdStep1 + ' --fileout file:raw.root '
126  cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'.log ',)
127  retStep1 = self.doCmd(cmd)
128 
129  print " ... ret: " , retStep1
130 
131  # prepare and run the next workflows -- if the previous step was OK :
132  # set some defaults
133  retStep2 = 0
134  retStep3 = 0
135  retStep4 = 0
136  if self.wf.cmdStep2 and retStep1 == 0:
137  fullcmd = preamble
138  fullcmd += self.wf.cmdStep2
139  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
140  fullcmd += ' --fileout file:reco.root '
141  print '=====>>> ', self.wf.nameId, self.wf.numId
142 
143  # for HI B0 step2 use a file from a previous relval production as step1 doesn't write
144  # any output in 1 hour. Add himix flag here as this is only needed when run on the mixed
145  # input files (the relvals are OK)
146  if ( '40.0' in str(self.wf.numId) ) :
147  fullcmd += ' --himix '
148  inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/102FF831-9B0F-E011-A3E9-003048678BC6.root'
149  if ( '41.0' in str(self.wf.numId) ) :
150  fullcmd += ' --himix '
151  inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_GammaJet_pt20_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/06B4F699-A50F-E011-AD62-0018F3D0962E.root'
152 
153  fullcmd += ' --filein '+inFile+ ' '
154  fullcmd += ' > %s 2>&1; ' % ('step2_'+self.wf.nameId+'.log ',)
155  # print fullcmd
156  retStep2 = self.doCmd(fullcmd)
157 # if random.randint(0,100) < 20 : retStep2 = -42
158 
159  if self.wf.cmdStep3 and retStep2 == 0:
160  fullcmd = preamble
161  fullcmd += self.wf.cmdStep3
162  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
163  # FIXME: dirty hack for beam-spot dedicated relval
164  if not '134' in str(self.wf.numId):
165  if 'HARVESTING' in fullcmd and not 'filein' in fullcmd:
166  fullcmd += ' --filein file:reco_inDQM.root --fileout file:step3.root '
167  else:
168  fullcmd += ' --filein file:reco.root --fileout file:step3.root '
169  fullcmd += ' > %s 2>&1; ' % ('step3_'+self.wf.nameId+'.log ',)
170  # print fullcmd
171  retStep3 = self.doCmd(fullcmd)
172 # if random.randint(0,100) < 40 : retStep3 = -42
173  if self.wf.cmdStep4 and retStep3 == 0:
174  fullcmd = preamble
175  fullcmd += self.wf.cmdStep4
176  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
177  # FIXME: dirty hack for beam-spot dedicated relval
178  if not '134' in str(self.wf.numId):
179  fullcmd += ' --filein file:step3.root '
180  fullcmd += ' > %s 2>&1; ' % ('step4_'+self.wf.nameId+'.log ',)
181  # print fullcmd
182  retStep4 = self.doCmd(fullcmd)
183 # if random.randint(0,100) < 40 : retStep4 = -42
184 
185  os.chdir(startDir)
186 
187  endtime='date %s' %time.asctime()
188  tottime='%s-%s'%(endtime,startime)
189 
190  self.nfail = [0,0,0,0]
191  self.npass = [1,1,1,1]
192  if 'NOSTEP' in stat2: # don't say reco/alca is passed if we don't have to run them
193  self.npass = [1,0,0,0]
194  else: # we have a reco step, check for alca:
195  if 'NOSTEP' in stat3 :
196  self.npass = [1,1,0,0]
197  if 'NOSTEP' in stat4 :
198  self.npass = [1,1,1,0]
199  if retStep1 != 0 :
200  stat1 = 'FAILED'
201  stat2 = 'NOTRUN'
202  stat3 = 'NOTRUN'
203  stat4 = 'NOTRUN'
204  self.npass = [0,0,0,0]
205  self.nfail = [1,0,0,0]
206 
207  if retStep2 != 0 :
208  stat2 = 'FAILED'
209  stat3 = 'NOTRUN'
210  stat4 = 'NOTRUN'
211  self.npass = [1,0,0,0]
212  self.nfail = [0,1,0,0]
213 
214  if retStep3 != 0 :
215  stat3 = 'FAILED'
216  stat4 = 'NOTRUN'
217  self.npass = [1,1,0,0]
218  self.nfail = [0,0,1,0]
219 
220  if retStep4 != 0 :
221  stat4 = 'FAILED'
222  self.npass = [1,1,1,0]
223  self.nfail = [0,0,0,1]
224 
225  logStat = 'Step1-'+stat1+' Step2-'+stat2+' Step3-'+stat3+' '+' Step4-'+stat4+' '
226  self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, tottime, retStep1,retStep2,retStep3, retStep4)
227 
228  return
229 
230 
231 # ================================================================================
strip
void strip(std::string &input, const std::string &blanks=" \n\t")
Definition: stringTools.cc:16

Member Data Documentation

runTheMatrix.WorkFlowRunner.nfail

Definition at line 15 of file runTheMatrix.py.

Referenced by runTheMatrix.WorkFlowRunner.run(), and runTheMatrix_dev.WorkFlowRunner.run().

runTheMatrix.WorkFlowRunner.npass

Definition at line 16 of file runTheMatrix.py.

Referenced by runTheMatrix.WorkFlowRunner.run(), and runTheMatrix_dev.WorkFlowRunner.run().

runTheMatrix.WorkFlowRunner.report

Definition at line 14 of file runTheMatrix.py.

Referenced by addOnTests.testit.run(), runTheMatrix.WorkFlowRunner.run(), and runTheMatrix_dev.WorkFlowRunner.run().

runTheMatrix.WorkFlowRunner.status

Definition at line 13 of file runTheMatrix.py.

runTheMatrix.WorkFlowRunner.wf

Definition at line 11 of file runTheMatrix.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5