CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
runTheMatrix.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import os, sys, re, time
4 
5 import random
6 from threading import Thread
7 
8 class WorkFlowRunner(Thread):
9  def __init__(self, wf):
10  Thread.__init__(self)
11  self.wf = wf
12 
13  self.status=-1
14  self.report=''
15  self.nfail=0
16  self.npass=0
17 
18  return
19 
20  def doCmd(self, cmd, dryRun=False):
21 
22  msg = "\n# in: " +os.getcwd()
23  if dryRun: msg += " dryRun for '"
24  else: msg += " going to execute "
25  msg += cmd.replace(';','\n')
26  print msg
27 
28  cmdLog = open(self.wf.numId+'_'+self.wf.nameId+'/cmdLog','a')
29  cmdLog.write(msg+'\n')
30  cmdLog.close()
31 
32  ret = 0
33  if not dryRun:
34  ret = os.system(cmd)
35  if ret != 0:
36  print "ERROR executing ",cmd,'ret=', ret
37 
38  return ret
39 
40  def run(self):
41 
42  startDir = os.getcwd()
43 
44  wfDir = self.wf.numId+'_'+self.wf.nameId
45  if not os.path.exists(wfDir):
46  os.makedirs(wfDir)
47 
48  preamble = ''
49  if os.path.exists( os.path.join(os.environ["CMS_PATH"],'cmsset_default.sh') ) :
50  preamble = 'source $CMS_PATH/cmsset_default.sh; '
51  else:
52  preamble = 'source $CMS_PATH/sw/cmsset_default.sh; '
53  preamble += 'eval `scram run -sh`; '
54  preamble += 'cd '+wfDir+'; '
55  preamble += 'ulimit -v 4069000;' # make sure processes keep within limits ...
56 
57  startime='date %s' %time.asctime()
58 
59  # set defaults for the statuses
60  stat1 = 'PASSED'
61  stat2 = 'PASSED'
62  stat3 = 'PASSED'
63  stat4 = 'PASSED'
64  if not self.wf.cmdStep2: stat2 = 'NOSTEP'
65  if not self.wf.cmdStep3: stat3 = 'NOSTEP'
66  if not self.wf.cmdStep4: stat4 = 'NOSTEP'
67 
68  # run the first workflow:
69  cmd = preamble
70 
71  inFile = 'file:raw.root'
72  if 'REALDATA' in self.wf.cmdStep1:
73  realDataRe = re.compile('REALDATA:\s*(/[A-Za-z].*?),(\s*RUN:\s*(?P<run>\d+),)?(\s*FILES:\s*(?P<files>\d+),)?(\s*EVENTS:\s*(?P<events>\d+))?,\s*LABEL:\s*(?P<label>.*),\s*LOCATION:\s*(?P<location>.*)\s*')
74  realDataMatch = realDataRe.match(self.wf.cmdStep1)
75  if realDataMatch:
76  run = None
77  if realDataMatch.group("run") : run = realDataMatch.group("run")
78  label = realDataMatch.group("label")
79  location = realDataMatch.group("location").lower().strip()
80  if 'caf' in location:
81  print "ignoring workflow ",self.wf.numId, self.wf.nameId, ' as this is on CAF ...'
82  self.npass = [0,0,0,0]
83  self.nfail = [0,0,0,0]
84 
85  logStat = 'Step1-NOTRUN Step2-NOTRUN Step3-NOTRUN Step4-NOTRUN '
86  self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, 0, 0,0,0,0)
87  return
88 
89  files = None
90  events = None
91  if realDataMatch.group("files"):
92  files = realDataMatch.group("files")
93  if realDataMatch.group("events"):
94  events = realDataMatch.group("events")
95  if self.wf.cmdStep2 and ' -n ' not in self.wf.cmdStep2: self.wf.cmdStep2 += ' -n ' + events
96  if self.wf.cmdStep3 and ' -n ' not in self.wf.cmdStep3: self.wf.cmdStep3 += ' -n ' + events
97  if self.wf.cmdStep4 and ' -n ' not in self.wf.cmdStep4: self.wf.cmdStep4 += ' -n ' + events
98 
99  print "run, files, events, label", run, files, events, label
100  cmd += 'dbs search --noheader --url=http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet '
101  cmd += "--query='find file where dataset like "+realDataMatch.group(1)
102  if run: cmd += " and run=" + run
103  cmd += "' "
104  cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'-dbsquery.log',)
105  retStep1 = self.doCmd(cmd)
106  if retStep1 == 0:
107  lf = open(wfDir+'/step1_'+self.wf.nameId+'-dbsquery.log', 'r')
108  lines = lf.readlines()
109  lf.close()
110  if not lines or len(lines)==0 :
111  inFile = "NoFileFoundInDBS"
112  retStep1 = -95
113  else:
114  try:
115  inFile = lines[0].strip()
116  except Exception, e:
117  print "ERROR determining file from DBS query: ", str(e)
118  inFile = "NoFileFoundInDBS"
119  retStep1 = -90
120  else:
121  print "ERROR: found REALDATA in '"+self.wf.cmdStep1+"' but not RE match !!??!"
122  retStep1 = -99
123  else:
124  cmd += self.wf.cmdStep1 + ' --fileout file:raw.root '
125  cmd += ' > %s 2>&1; ' % ('step1_'+self.wf.nameId+'.log ',)
126  retStep1 = self.doCmd(cmd)
127 
128  print " ... ret: " , retStep1
129 
130  # prepare and run the next workflows -- if the previous step was OK :
131  # set some defaults
132  retStep2 = 0
133  retStep3 = 0
134  retStep4 = 0
135  if self.wf.cmdStep2 and retStep1 == 0:
136  fullcmd = preamble
137  fullcmd += self.wf.cmdStep2
138  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
139  fullcmd += ' --fileout file:reco.root '
140  print '=====>>> ', self.wf.nameId, self.wf.numId
141 
142  # for HI B0 step2 use a file from a previous relval production as step1 doesn't write
143  # any output in 1 hour. Add himix flag here as this is only needed when run on the mixed
144  # input files (the relvals are OK)
145  if ( '40.0' in str(self.wf.numId) ) :
146  fullcmd += ' --himix '
147  inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_ZeemumuJets_pt10_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/102FF831-9B0F-E011-A3E9-003048678BC6.root'
148  if ( '41.0' in str(self.wf.numId) ) :
149  fullcmd += ' --himix '
150  inFile = '/store/relval/CMSSW_3_9_7/RelValPyquen_GammaJet_pt20_2760GeV/GEN-SIM-DIGI-RAW-HLTDEBUG/START39_V7HI-v1/0054/06B4F699-A50F-E011-AD62-0018F3D0962E.root'
151 
152  fullcmd += ' --filein '+inFile+ ' '
153  fullcmd += ' > %s 2>&1; ' % ('step2_'+self.wf.nameId+'.log ',)
154  # print fullcmd
155  retStep2 = self.doCmd(fullcmd)
156 # if random.randint(0,100) < 20 : retStep2 = -42
157 
158  if self.wf.cmdStep3 and retStep2 == 0:
159  fullcmd = preamble
160  fullcmd += self.wf.cmdStep3
161  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
162  # FIXME: dirty hack for beam-spot dedicated relval
163  if not '134' in str(self.wf.numId):
164  if 'HARVESTING' in fullcmd and not 'filein' in fullcmd:
165  fullcmd += ' --filein file:reco_inDQM.root --fileout file:step3.root '
166  else:
167  fullcmd += ' --filein file:reco.root --fileout file:step3.root '
168  fullcmd += ' > %s 2>&1; ' % ('step3_'+self.wf.nameId+'.log ',)
169  # print fullcmd
170  retStep3 = self.doCmd(fullcmd)
171 # if random.randint(0,100) < 40 : retStep3 = -42
172  if self.wf.cmdStep4 and retStep3 == 0:
173  fullcmd = preamble
174  fullcmd += self.wf.cmdStep4
175  if ' -n ' not in fullcmd : fullcmd += ' -n -1 '
176  # FIXME: dirty hack for beam-spot dedicated relval
177  if not '134' in str(self.wf.numId):
178  fullcmd += ' --filein file:step3.root '
179  fullcmd += ' > %s 2>&1; ' % ('step4_'+self.wf.nameId+'.log ',)
180  # print fullcmd
181  retStep4 = self.doCmd(fullcmd)
182 # if random.randint(0,100) < 40 : retStep4 = -42
183 
184  os.chdir(startDir)
185 
186  endtime='date %s' %time.asctime()
187  tottime='%s-%s'%(endtime,startime)
188 
189  self.nfail = [0,0,0,0]
190  self.npass = [1,1,1,1]
191  if 'NOSTEP' in stat2: # don't say reco/alca is passed if we don't have to run them
192  self.npass = [1,0,0,0]
193  else: # we have a reco step, check for alca:
194  if 'NOSTEP' in stat3 :
195  self.npass = [1,1,0,0]
196  if 'NOSTEP' in stat4 :
197  self.npass = [1,1,1,0]
198  if retStep1 != 0 :
199  stat1 = 'FAILED'
200  stat2 = 'NOTRUN'
201  stat3 = 'NOTRUN'
202  stat4 = 'NOTRUN'
203  self.npass = [0,0,0,0]
204  self.nfail = [1,0,0,0]
205 
206  if retStep2 != 0 :
207  stat2 = 'FAILED'
208  stat3 = 'NOTRUN'
209  stat4 = 'NOTRUN'
210  self.npass = [1,0,0,0]
211  self.nfail = [0,1,0,0]
212 
213  if retStep3 != 0 :
214  stat3 = 'FAILED'
215  stat4 = 'NOTRUN'
216  self.npass = [1,1,0,0]
217  self.nfail = [0,0,1,0]
218 
219  if retStep4 != 0 :
220  stat4 = 'FAILED'
221  self.npass = [1,1,1,0]
222  self.nfail = [0,0,0,1]
223 
224  logStat = 'Step1-'+stat1+' Step2-'+stat2+' Step3-'+stat3+' '+' Step4-'+stat4+' '
225  self.report+='%s_%s %s - time %s; exit: %s %s %s %s \n' % (self.wf.numId, self.wf.nameId, logStat, tottime, retStep1,retStep2,retStep3, retStep4)
226 
227  return
228 
229 
230 # ================================================================================
231 
232 class WorkFlow(object):
233 
234  def __init__(self, num, nameID, cmd1, cmd2=None, cmd3=None, cmd4=None, real=None):
235  self.numId = num.strip()
236  self.nameId = nameID
237  self.cmdStep1 = cmd1
238  if self.cmdStep1: self.cmdStep1 = self.cmdStep1.replace('--no_exec', '') # make sure the commands execute
239  self.cmdStep2 = cmd2
240  if self.cmdStep2: self.cmdStep2 = self.cmdStep2.replace('--no_exec', '') # make sure the commands execute
241  self.cmdStep3 = cmd3
242  if self.cmdStep3: self.cmdStep3 = self.cmdStep3.replace('--no_exec', '') # make sure the commands execute
243  self.cmdStep4 = cmd4
244  if self.cmdStep4: self.cmdStep4 = self.cmdStep4.replace('--no_exec', '') # make sure the commands execute
245 
246  # run on real data requested:
247  self.real = real
248  return
249 
250 # ================================================================================
251 
252 class MatrixReader(object):
253 
254  def __init__(self):
255 
256  self.reset()
257 
258  return
259 
260  def reset(self):
261 
262  self.step1WorkFlows = {}
263  self.step2WorkFlows = {}
264  self.step3WorkFlows = {}
265  self.step4WorkFlows = {}
266 
267  self.workFlows = []
268  self.nameList = {}
269 
270 
271  self.filesPrefMap = {'cmsDriver_standard_hlt.txt' : 'std-' ,
272  'cmsDriver_highstats_hlt.txt': 'hi-' ,
273  'cmsDriver_generator.txt' : 'gen-' ,
274  # 'cmsDriver_PileUp_hlt.txt' : 'pu-' ,
275  # 'cmsDriver_realData_hlt.txt' : 'data-'
276  }
277 
278  self.files = ['cmsDriver_standard_hlt.txt' ,
279  'cmsDriver_highstats_hlt.txt',
280  'cmsDriver_generator.txt' ,
281  # 'cmsDriver_PileUp_hlt.txt' ,
282  # 'cmsDriver_realData_hlt.txt'
283  ]
284 
285  return
286 
287  def readMatrix(self, fileNameIn):
288 
289  prefix = self.filesPrefMap[fileNameIn]
290 
291  print "processing ", fileNameIn,
292  lines = []
293  try:
294  try:
295  inFile = open(fileNameIn, 'r')
296  print ' from local developer area',
297  except IOError:
298  baseRelPath = os.environ['CMSSW_BASE']
299  # print "trying fall-back to cmsDriver files from developer area at:", baseRelPath
300  try:
301  inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
302  print ' from ', baseRelPath,
303  except IOError:
304  baseRelPath = os.environ['CMSSW_RELEASE_BASE']
305  # print "trying fall back to cmsDriver files from base release at:", baseRelPath
306  inFile = open( os.path.join(baseRelPath, 'src/Configuration/PyReleaseValidation/data' ,fileNameIn), 'r')
307  print ' from ', baseRelPath,
308  lines = inFile.readlines()
309  inFile.close()
310  except Exception, e:
311  print "ERROR reading in file ", fileNameIn, str(e)
312  return
313 
314  print " found ", len(lines), 'entries.'
315 
316  realRe = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
317  step1Re = re.compile('\s*([1-9][0-9]*\.*[0-9]*)\s*\+\+\s*(.*?)\s*\+\+\s*(.*?)\s*@@@\s*(.*)\s*')
318  step2Re = re.compile('\s*STEP2\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
319  step3Re = re.compile('\s*STEP3\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
320  step4Re = re.compile('\s*STEP4\s*\+\+\s*(\S*)\s*@@@\s*(.*)\s*')
321  for lineIn in lines:
322  line = lineIn.strip()
323 
324  realMatch = realRe.match(line)
325  if realMatch :
326  num = realMatch.group(1).strip()
327  name = realMatch.group(2).strip().replace('<','').replace('>','').replace(':','')
328  next = realMatch.group(3).strip().replace('+','').replace(',', ' ')
329  cmd = realMatch.group(4).strip()
330 
331  step2 = "None"
332  step3 = "None"
333  step4 = "None"
334 
335  steps = next.split()
336  if len(steps) > 0:
337  step2 = steps[0].strip()
338  if len(steps) > 1:
339  step3 = steps[1].strip()
340  if len(steps) > 2:
341  step4 = steps[2].strip()
342 
343  self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
344  continue
345 
346 
347  step1Match = step1Re.match(line)
348  if step1Match :
349  num = step1Match.group(1).strip()
350  name = step1Match.group(2).strip().replace('<','').replace('>','').replace(':','')
351  next = step1Match.group(3).strip().replace('+','').replace(',', ' ')
352  cmd = step1Match.group(4).strip()
353  step2 = "None"
354  step3 = "None"
355  step4 = "None"
356 
357  steps = next.split()
358  if len(steps) > 0:
359  step2 = steps[0].strip()
360  if len(steps) > 1:
361  step3 = steps[1].strip()
362  if len(steps) > 2:
363  step4 = steps[2].strip()
364 
365  self.step1WorkFlows[(float(num),prefix)] = (str(float(num)), name, step2, step3, step4, cmd, None)
366  continue
367 
368  step2Match = step2Re.match(line)
369  if step2Match :
370  name = step2Match.group(1).strip()
371  cmd = step2Match.group(2).strip()
372  self.step2WorkFlows[name] = (cmd.replace('--no_exec','') ) # make sure the command is really run
373  continue
374 
375  step3Match = step3Re.match(line)
376  if step3Match :
377  name = step3Match.group(1).strip()
378  cmd = step3Match.group(2).strip()
379  self.step3WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
380  continue
381 
382  step4Match = step4Re.match(line)
383  if step4Match :
384  name = step4Match.group(1).strip()
385  cmd = step4Match.group(2).strip()
386  self.step4WorkFlows[name] = ( cmd.replace('--no_exec','') ) # make sure the command is really run
387  continue
388 
389  return
390 
391  def showRaw(self):
392 
393  print "found ", len(self.step1WorkFlows.keys()), ' workflows for step1:'
394  ids = self.step1WorkFlows.keys()
395  ids.sort()
396  for key in ids:
397  val = self.step1WorkFlows[key]
398  print key[0], ':', val
399 
400  print "found ", len(self.step2WorkFlows.keys()), ' workflows for step2:'
401  for key, val in self.step2WorkFlows.items():
402  print key, ':', val
403 
404  print "found ", len(self.step3WorkFlows.keys()), ' workflows for step3:'
405  for key, val in self.step3WorkFlows.items():
406  print key, ':', val
407 
408  print "found ", len(self.step4WorkFlows.keys()), ' workflows for step4:'
409  for key, val in self.step4WorkFlows.items():
410  print key, ':', val
411 
412  return
413 
414  def showWorkFlows(self, selected=None):
415 
416  maxLen = 100 # for summary, limit width of output
417  fmt1 = "%-6s %-35s [1]: %s ..."
418  fmt2 = " %35s [%d]: %s ..."
419  print "\nfound a total of ", len(self.workFlows), ' workflows:'
420  if selected:
421  print " of which the following", len(selected), 'were selected:'
422  maxLen = -1 # for individual listing, no limit on width
423  fmt1 = "%-6s %-35s [1]: %s "
424  fmt2 = " %35s [%d]: %s"
425  n1 = 0
426  n2 = 0
427  n3 = 0
428  n4 = 0
429  for wf in self.workFlows:
430  if selected and float(wf.numId) not in selected: continue
431  n1+=1
432  print fmt1 % (wf.numId, wf.nameId, (wf.cmdStep1+' ')[:maxLen])
433  if wf.cmdStep2:
434  n2+=1
435  print fmt2 % ( ' ', 2, (wf.cmdStep2+' ')[:maxLen])
436  if wf.cmdStep3:
437  n3+=1
438  print fmt2 % ( ' ', 3, (wf.cmdStep3+' ')[:maxLen])
439  if wf.cmdStep4:
440  n4+=1
441  print fmt2 % ( ' ', 4, (wf.cmdStep4+' ')[:maxLen])
442 
443  print n1, 'workflows for step1,'
444  print n2, 'workflows for step1 + step2,'
445  print n3, 'workflows for step1 + step2 + step3'
446  print n4, 'workflows for step1 + step2 + step3 + step4'
447 
448  return
449 
450  def createWorkFlows(self, fileNameIn):
451 
452  prefixIn = self.filesPrefMap[fileNameIn]
453 
454  # get through the list of items and update the requested workflows only
455  keyList = self.step1WorkFlows.keys()
456  ids = []
457  for item in keyList:
458  id, pref = item
459  if pref != prefixIn : continue
460  ids.append( float(id) )
461 
462  ids.sort()
463  n1 = 0
464  n2 = 0
465  n3 = 0
466  n4 = 0
467  for key in ids:
468  val = self.step1WorkFlows[(key,prefixIn)]
469  num, name, step2, step3, step4, cmd, real = val
470  nameId = num+'_'+name
471  if step2.lower() != 'none':
472  name += '+'+step2
473  if step3.lower() != 'none':
474  name += '+'+step3
475  if step4.lower() != 'none':
476  name += '+'+step4
477  if nameId in self.nameList.keys():
478  print "==> duplicate name found for ", nameId
479  print ' keeping : ', self.nameList[nameId]
480  print ' ignoring : ', val
481  else:
482  self.nameList[nameId] = val
483 
484  cmd2 = None
485  cmd3 = None
486  cmd4 = None
487 
488  n1 += 1
489 
490  if step2.lower() != 'none':
491  n2 += 1
492  cmd2 = self.step2WorkFlows[step2]
493  if step3.lower() != 'none':
494  n3 += 1
495  cmd3 = self.step3WorkFlows[step3]
496  if step4.lower() != 'none':
497  n4 += 1
498  cmd4 = self.step4WorkFlows[step4]
499  #print '\tstep3 : ', self.step3WorkFlows[step3]
500  self.workFlows.append( WorkFlow(num, name, cmd, cmd2, cmd3, cmd4) )
501 
502  return
503 
504  def prepare(self):
505 
506  for matrixFile in self.files:
507  try:
508  self.readMatrix(matrixFile)
509  except Exception, e:
510  print "ERROR reading file:", matrixFile, str(e)
511 
512  try:
513  self.createWorkFlows(matrixFile)
514  except Exception, e:
515  print "ERROR creating workflows :", str(e)
516 
517  def show(self, selected=None):
518  # self.showRaw()
519  self.showWorkFlows(selected)
520  print '\n','-'*80,'\n'
521 
522 
523  def updateDB(self):
524 
525  import pickle
526  pickle.dump(self.workFlows, open('theMatrix.pkl', 'w') )
527 
528  return
529 
530 # ================================================================================
531 
532 class MatrixRunner(object):
533 
534  def __init__(self, wfIn=None, nThrMax=8):
535 
536  self.workFlows = wfIn
537 
538  self.threadList = []
539  self.maxThreads = int(nThrMax) # make sure we get a number ...
540 
541 
542  def activeThreads(self):
543 
544  nActive = 0
545  for t in self.threadList:
546  if t.isAlive() : nActive += 1
547 
548  return nActive
549 
550 
551  def runTests(self, testList=None):
552 
553  startDir = os.getcwd()
554 
555  # make sure we have a way to set the environment in the threads ...
556  if not os.environ.has_key('CMS_PATH'):
557  cmsPath = '/afs/cern.ch/cms'
558  print "setting default for CMS_PATH to", cmsPath
559  os.environ['CMS_PATH'] = cmsPath
560 
561  report=''
562  print 'Running in %s thread(s)' % self.maxThreads
563 
564  for wf in self.workFlows:
565 
566  if testList and float(wf.numId) not in [float(x) for x in testList]: continue
567 
568  # don't know yet how to treat real data WF ...
569  if wf.real : continue
570 
571  item = wf.nameId
572  if os.path.islink(item) : continue # ignore symlinks
573 
574  # make sure we don't run more than the allowed number of threads:
575  while self.activeThreads() >= self.maxThreads:
576  time.sleep(10)
577  continue
578 
579  print '\nPreparing to run %s %s' % (wf.numId, item)
580 
581 ## if testList: # if we only run a selection, run only 5 events instead of 10
582 ## wf.cmdStep1 = wf.cmdStep1.replace('-n 10', '-n 5')
583 
584  current = WorkFlowRunner(wf)
585  self.threadList.append(current)
586  current.start()
587  time.sleep(random.randint(1,5)) # try to avoid race cond by sleeping random amount of time [1,5] sec
588 
589  # wait until all threads are finished
590  while self.activeThreads() > 0:
591  time.sleep(5)
592 
593  # all threads are done now, check status ...
594  nfail1 = 0
595  nfail2 = 0
596  nfail3 = 0
597  nfail4 = 0
598  npass = 0
599  npass1 = 0
600  npass2 = 0
601  npass3 = 0
602  npass4 = 0
603  for pingle in self.threadList:
604  pingle.join()
605  try:
606  nfail1 += pingle.nfail[0]
607  nfail2 += pingle.nfail[1]
608  nfail3 += pingle.nfail[2]
609  nfail4 += pingle.nfail[3]
610  npass1 += pingle.npass[0]
611  npass2 += pingle.npass[1]
612  npass3 += pingle.npass[2]
613  npass4 += pingle.npass[3]
614  npass += npass1+npass2+npass3+npass4
615  report += pingle.report
616  # print pingle.report
617  except Exception, e:
618  msg = "ERROR retrieving info from thread: " + str(e)
619  nfail1 += 1
620  nfail2 += 1
621  nfail3 += 1
622  nfail4 += 1
623  report += msg
624  print msg
625 
626  report+='\n %s %s %s %s tests passed, %s %s %s %s failed\n' %(npass1, npass2, npass3, npass4, nfail1, nfail2, nfail3, nfail4)
627  print report
628 
629  runall_report_name='runall-report-step123-.log'
630  runall_report=open(runall_report_name,'w')
631  runall_report.write(report)
632  runall_report.close()
633 
634  os.chdir(startDir)
635 
636  return
637 
638 
639 # ================================================================================
640 
641 def runSelected(testList, nThreads=4, show=False) :
642 
643  stdList = ['5.2', # SingleMu10 FastSim
644  '7', # Cosmics+RECOCOS+ALCACOS
645  '8', # BeamHalo+RECOCOS+ALCABH
646  '25', # TTbar+RECO2+ALCATT2 STARTUP
647  ]
648  hiStatList = [
649  '121', # TTbar_Tauola
650  '123.3', # TTBar FastSim
651  ]
652 
653  mrd = MatrixReader()
654  mrd.prepare()
655 
656  if testList == []:
657  testList = stdList+hiStatList
658 
659  ret = 0
660  if show:
661  mrd.show([float(x) for x in testList])
662  print 'selected items:', testList
663  else:
664  mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
665  ret = mRunnerHi.runTests(testList)
666 
667  return ret
668 
669 # ================================================================================
670 
671 def runData(testList, nThreads=4, show=False) :
672 
673  mrd = MatrixReader()
674 
675  mrd.prepare()
676 
677  ret = 0
678  if show:
679  if not testList or testList == ['all']:
680  mrd.show()
681  else:
682  mrd.show([float(x) for x in testList])
683  print 'selected items:', testList
684  else:
685  mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
686  if not testList or testList == ['all']:
687  ret = mRunnerHi.runTests()
688  else:
689  ret = mRunnerHi.runTests(testList)
690 
691  return ret
692 
693 # --------------------------------------------------------------------------------
694 
695 def runAll(testList=None, nThreads=4, show=False) :
696 
697  mrd = MatrixReader()
698  mrd.prepare()
699 
700  ret = 0
701 
702  if show:
703  mrd.show()
704  print "nThreads = ",nThreads
705  else:
706  mRunnerHi = MatrixRunner(mrd.workFlows, nThreads)
707  ret = mRunnerHi.runTests()
708 
709  return ret
710 
711 
712 # --------------------------------------------------------------------------------
713 
714 def runOnly(only, show, nThreads=4):
715 
716  if not only: return
717 
718  for what in only:
719  print "found request to run relvals only for ",what
720 
721 
722 # --------------------------------------------------------------------------------
723 
724 def usage():
725  print "Usage:", sys.argv[0], ' [options] '
726  print """
727 Where options is one of the following:
728  -d, --data <list> comma-separated list of workflows to use from the realdata file.
729  <list> can be "all" to select all data workflows
730  -l, --list <list> comma-separated list of workflows to use from the cmsDriver*.txt files
731  -j, --nproc <n> run <n> processes in parallel (default: 4 procs)
732  -s, --selected run a subset of 8 workflows (usually in the CustomIB)
733  -n, -q, --show show the (selected) workflows
734 
735 <list>s should be put in single- or double-quotes to avoid confusion with/by the shell
736 """
737 
738 # ================================================================================
739 
740 if __name__ == '__main__':
741 
742  import getopt
743 
744  try:
745  opts, args = getopt.getopt(sys.argv[1:], "hj:sl:nqo:d:", ['help',"nproc=",'selected','list=','showMatrix','only=','data='])
746  except getopt.GetoptError, e:
747  print "unknown option", str(e)
748  sys.exit(2)
749 
750 # check command line parameter
751 
752  np=4 # default: four threads
753  sel = None
754  show = False
755  only = None
756  data = None
757  for opt, arg in opts :
758  if opt in ('-h','--help'):
759  usage()
760  sys.exit(0)
761  if opt in ('-j', "--nproc" ):
762  np=int(arg)
763  if opt in ('-n','-q','--showMatrix', ):
764  show = True
765  if opt in ('-s','--selected',) :
766  sel = []
767  if opt in ('-o','--only',) :
768  only = []
769  if opt in ('-l','--list',) :
770  sel = arg.split(',')
771  if opt in ('-d','--data',) :
772  data = arg.split(',')
773 
774  # print "sel",sel
775  ret = 0
776  if sel != None: # explicit distinguish from empty list (which is also false)
777  ret = runSelected(testList=sel, nThreads=np, show=show)
778  elif only != None:
779  ret = runOnly(only=only, show=show, nThreads=np)
780  elif data != None:
781  ret = runData(testList=data, show=show, nThreads=np)
782  else:
783  ret = runAll(show=show, nThreads=np)
784 
785  sys.exit(ret)
strip
void strip(std::string &input, const std::string &blanks=" \n\t")
Definition: stringTools.cc:16
dbtoconf.object
list object
Definition: dbtoconf.py:77
Generated for CMSSW Reference Manual by doxygen 1.8.5