CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
WorkFlowRunner.WorkFlowRunner Class Reference
Inheritance diagram for WorkFlowRunner.WorkFlowRunner:

Public Member Functions

def __init__
 
def doCmd
 
def run
 

Public Attributes

 cafVeto
 
 dasOptions
 
 dryRun
 
 jobReport
 
 nfail
 
 noRun
 
 npass
 needs to set self.report More...
 
 report
 wrap up #### More...
 
 retStep
 
 stat
 
 status
 
 wf
 
 wfDir
 

Detailed Description

Definition at line 9 of file WorkFlowRunner.py.

Constructor & Destructor Documentation

def WorkFlowRunner.WorkFlowRunner.__init__ (   self,
  wf,
  noRun = False,
  dryRun = False,
  cafVeto = True,
  dasOptions = "",
  jobReport = False 
)

Definition at line 10 of file WorkFlowRunner.py.

10 
11  def __init__(self, wf, noRun=False,dryRun=False,cafVeto=True,dasOptions="",jobReport=False):
12  Thread.__init__(self)
13  self.wf = wf
14 
15  self.status=-1
16  self.report=''
17  self.nfail=0
18  self.npass=0
19  self.noRun=noRun
20  self.dryRun=dryRun
21  self.cafVeto=cafVeto
22  self.dasOptions=dasOptions
23  self.jobReport=jobReport
24 
25  self.wfDir=str(self.wf.numId)+'_'+self.wf.nameId
26  return
WorkFlowRunner.WorkFlowRunner.npass
npass
needs to set self.report
Definition: WorkFlowRunner.py:17

Member Function Documentation

def WorkFlowRunner.WorkFlowRunner.doCmd (   self,
  cmd 
)

Definition at line 27 of file WorkFlowRunner.py.

References WorkFlowRunner.WorkFlowRunner.dryRun, and WorkFlowRunner.WorkFlowRunner.wfDir.

27 
28  def doCmd(self, cmd):
29 
30  msg = "\n# in: " +os.getcwd()
31  if self.dryRun: msg += " dryRun for '"
32  else: msg += " going to execute "
33  msg += cmd.replace(';','\n')
34  print msg
35 
36  cmdLog = open(self.wfDir+'/cmdLog','a')
37  cmdLog.write(msg+'\n')
38  cmdLog.close()
39 
40  ret = 0
41  if not self.dryRun:
42  p = Popen(cmd, shell=True)
43  ret = os.waitpid(p.pid, 0)[1]
44  if ret != 0:
45  print "ERROR executing ",cmd,'ret=', ret
46 
47  return ret
def WorkFlowRunner.WorkFlowRunner.run (   self)

Definition at line 48 of file WorkFlowRunner.py.

References WorkFlowRunner.WorkFlowRunner.dryRun, runall.testit.nfail, WorkFlowRunner.WorkFlowRunner.nfail, runall.testit.npass, WorkFlowRunner.WorkFlowRunner.npass, TShapeAnalysis.npass, and WorkFlowRunner.WorkFlowRunner.wfDir.

Referenced by Types.EventID.cppID(), and Types.LuminosityBlockID.cppID().

48 
49  def run(self):
50 
51  startDir = os.getcwd()
52 
53  if not os.path.exists(self.wfDir):
54  os.makedirs(self.wfDir)
55  elif not self.dryRun: # clean up to allow re-running in the same overall devel area, then recreate the dir to make sure it exists
56  print "cleaning up ", self.wfDir, ' in ', os.getcwd()
57  shutil.rmtree(self.wfDir)
58  os.makedirs(self.wfDir)
59 
60  preamble = 'cd '+self.wfDir+'; '
61 
62  startime='date %s' %time.asctime()
63 
64  # check where we are running:
65  onCAF = False
66  if 'cms/caf/cms' in os.environ['CMS_PATH']:
67  onCAF = True
68 
69  ##needs to set
70  #self.report
71  self.npass = []
72  self.nfail = []
73  self.stat = []
74  self.retStep = []
75 
76  def closeCmd(i,ID):
77  return ' > %s 2>&1; ' % ('step%d_'%(i,)+ID+'.log ',)
78 
79  inFile=None
80  lumiRangeFile=None
81  aborted=False
82  for (istepmone,com) in enumerate(self.wf.cmds):
83  istep=istepmone+1
84  cmd = preamble
85  if aborted:
86  self.npass.append(0)
87  self.nfail.append(0)
88  self.retStep.append(0)
89  self.stat.append('NOTRUN')
90  continue
91  if not isinstance(com,str):
92  if self.cafVeto and (com.location == 'CAF' and not onCAF):
93  print "You need to be no CAF to run",self.wf.numId
94  self.npass.append(0)
95  self.nfail.append(0)
96  self.retStep.append(0)
97  self.stat.append('NOTRUN')
98  aborted=True
99  continue
100  #create lumiRange file first so if das fails we get its error code
101  cmd2 = com.lumiRanges()
102  if cmd2:
103  cmd2 =cmd+cmd2+closeCmd(istep,'lumiRanges')
104  lumiRangeFile='step%d_lumiRanges.log'%(istep,)
105  retStep = self.doCmd(cmd2)
106  cmd+=com.das(self.dasOptions)
107  cmd+=closeCmd(istep,'dasquery')
108  retStep = self.doCmd(cmd)
109  #don't use the file list executed, but use the das command of cmsDriver for next step
110  inFile='filelist:step%d_dasquery.log'%(istep,)
111  print "---"
112  else:
113  #chaining IO , which should be done in WF object already and not using stepX.root but <stepName>.root
114  cmd += com
115  if self.noRun:
116  cmd +=' --no_exec'
117  if inFile: #in case previous step used DAS query (either filelist of das:)
118  cmd += ' --filein '+inFile
119  inFile=None
120  if lumiRangeFile: #DAS query can also restrict lumi range
121  cmd += ' --lumiToProcess '+lumiRangeFile
122  lumiRangeFile=None
123  # 134 is an existing workflow where harvesting has to operate on AlcaReco and NOT on DQM; hard-coded..
124  if 'HARVESTING' in cmd and not 134==self.wf.numId and not '--filein' in cmd:
125  cmd+=' --filein file:step%d_inDQM.root --fileout file:step%d.root '%(istep-1,istep)
126  else:
127  if istep!=1 and not '--filein' in cmd:
128  cmd+=' --filein file:step%s.root '%(istep-1,)
129  if not '--fileout' in com:
130  cmd+=' --fileout file:step%s.root '%(istep,)
131  if self.jobReport:
132  cmd += ' --suffix "-j JobReport%s.xml " ' % istep
133  cmd+=closeCmd(istep,self.wf.nameId)
134  retStep = self.doCmd(cmd)
135 
136 
137 
138  self.retStep.append(retStep)
139  if (retStep!=0):
140  #error occured
141  self.npass.append(0)
142  self.nfail.append(1)
143  self.stat.append('FAILED')
144  #to skip processing
145  aborted=True
146  else:
147  #things went fine
148  self.npass.append(1)
149  self.nfail.append(0)
150  self.stat.append('PASSED')
151 
152 
153  os.chdir(startDir)
154 
155  endtime='date %s' %time.asctime()
156  tottime='%s-%s'%(endtime,startime)
157 
158 
159  #### wrap up ####
160 
161  logStat=''
162  for i,s in enumerate(self.stat):
163  logStat+='Step%d-%s '%(i,s)
164  self.report='%s_%s %s - time %s; exit: '%(self.wf.numId,self.wf.nameId,logStat,tottime)+' '.join(map(str,self.retStep))+'\n'
165 
166  return
167 
168 
169 
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
WorkFlowRunner.WorkFlowRunner.npass
npass
needs to set self.report
Definition: WorkFlowRunner.py:17

Member Data Documentation

WorkFlowRunner.WorkFlowRunner.cafVeto

Definition at line 20 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.dasOptions

Definition at line 21 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.dryRun

Definition at line 19 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.doCmd(), and WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.jobReport

Definition at line 22 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.nfail

Definition at line 16 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.noRun

Definition at line 18 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.npass

needs to set self.report

Definition at line 17 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.report

wrap up ####

Definition at line 15 of file WorkFlowRunner.py.

Referenced by addOnTests.testit.run().

WorkFlowRunner.WorkFlowRunner.retStep

Definition at line 73 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.stat

Definition at line 72 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.status

Definition at line 14 of file WorkFlowRunner.py.

Referenced by dirstructure.Comparison.__make_image(), and dirstructure.Comparison.__repr__().

WorkFlowRunner.WorkFlowRunner.wf

Definition at line 12 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.wfDir

Definition at line 24 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.doCmd(), and WorkFlowRunner.WorkFlowRunner.run().

Generated for CMSSW Reference Manual by doxygen 1.8.5