CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes
WorkFlowRunner.WorkFlowRunner Class Reference
Inheritance diagram for WorkFlowRunner.WorkFlowRunner:

Public Member Functions

def __init__
 
def doCmd
 
def run
 

Public Attributes

 cafVeto
 
 dryRun
 
 nfail
 
 noRun
 
 npass
 needs to set self.report More...
 
 report
 wrap up #### More...
 
 retStep
 
 stat
 
 status
 
 wf
 
 wfDir
 

Detailed Description

Definition at line 9 of file WorkFlowRunner.py.

Constructor & Destructor Documentation

def WorkFlowRunner.WorkFlowRunner.__init__ (   self,
  wf,
  noRun = False,
  dryRun = False,
  cafVeto = True 
)

Definition at line 10 of file WorkFlowRunner.py.

10 
11  def __init__(self, wf, noRun=False,dryRun=False,cafVeto=True):
12  Thread.__init__(self)
13  self.wf = wf
14 
15  self.status=-1
16  self.report=''
17  self.nfail=0
18  self.npass=0
19  self.noRun=noRun
20  self.dryRun=dryRun
21  self.cafVeto=cafVeto
22 
23  self.wfDir=str(self.wf.numId)+'_'+self.wf.nameId
24  return
npass
needs to set self.report

Member Function Documentation

def WorkFlowRunner.WorkFlowRunner.doCmd (   self,
  cmd 
)

Definition at line 25 of file WorkFlowRunner.py.

References WorkFlowRunner.WorkFlowRunner.dryRun, and WorkFlowRunner.WorkFlowRunner.wfDir.

25 
26  def doCmd(self, cmd):
27 
28  msg = "\n# in: " +os.getcwd()
29  if self.dryRun: msg += " dryRun for '"
30  else: msg += " going to execute "
31  msg += cmd.replace(';','\n')
32  print msg
33 
34  cmdLog = open(self.wfDir+'/cmdLog','a')
35  cmdLog.write(msg+'\n')
36  cmdLog.close()
37 
38  ret = 0
39  if not self.dryRun:
40  p = Popen(cmd, shell=True)
41  ret = os.waitpid(p.pid, 0)[1]
42  if ret != 0:
43  print "ERROR executing ",cmd,'ret=', ret
44 
45  return ret
def WorkFlowRunner.WorkFlowRunner.run (   self)

Definition at line 46 of file WorkFlowRunner.py.

References WorkFlowRunner.WorkFlowRunner.dryRun, runall.testit.nfail, WorkFlowRunner.WorkFlowRunner.nfail, runall.testit.npass, WorkFlowRunner.WorkFlowRunner.npass, TShapeAnalysis.npass, and WorkFlowRunner.WorkFlowRunner.wfDir.

Referenced by Types.LuminosityBlockID.cppID().

46 
47  def run(self):
48 
49  startDir = os.getcwd()
50 
51  if not os.path.exists(self.wfDir):
52  os.makedirs(self.wfDir)
53  elif not self.dryRun: # clean up to allow re-running in the same overall devel area, then recreate the dir to make sure it exists
54  print "cleaning up ", self.wfDir, ' in ', os.getcwd()
55  shutil.rmtree(self.wfDir)
56  os.makedirs(self.wfDir)
57 
58  preamble = 'cd '+self.wfDir+'; '
59 
60  startime='date %s' %time.asctime()
61 
62  # check where we are running:
63  onCAF = False
64  if 'cms/caf/cms' in os.environ['CMS_PATH']:
65  onCAF = True
66 
67  ##needs to set
68  #self.report
69  self.npass = []
70  self.nfail = []
71  self.stat = []
72  self.retStep = []
73 
74  def closeCmd(i,ID):
75  return ' > %s 2>&1; ' % ('step%d_'%(i,)+ID+'.log ',)
76 
77  inFile=None
78  lumiRangeFile=None
79  aborted=False
80  for (istepmone,com) in enumerate(self.wf.cmds):
81  istep=istepmone+1
82  cmd = preamble
83  if aborted:
84  self.npass.append(0)
85  self.nfail.append(0)
86  self.retStep.append(0)
87  self.stat.append('NOTRUN')
88  continue
89  if not isinstance(com,str):
90  if self.cafVeto and (com.location == 'CAF' and not onCAF):
91  print "You need to be no CAF to run",self.wf.numId
92  self.npass.append(0)
93  self.nfail.append(0)
94  self.retStep.append(0)
95  self.stat.append('NOTRUN')
96  aborted=True
97  continue
98  #create lumiRange file first so if das fails we get its error code
99  cmd2 = com.lumiRanges()
100  if cmd2:
101  cmd2 =cmd+cmd2+closeCmd(istep,'lumiRanges')
102  lumiRangeFile='step%d_lumiRanges.log'%(istep,)
103  retStep = self.doCmd(cmd2)
104  cmd+=com.das()
105  cmd+=closeCmd(istep,'dasquery')
106  retStep = self.doCmd(cmd)
107  #don't use the file list executed, but use the das command of cmsDriver for next step
108  inFile='filelist:step%d_dasquery.log'%(istep,)
109  print "---"
110  else:
111  #chaining IO , which should be done in WF object already and not using stepX.root but <stepName>.root
112  cmd += com
113  if self.noRun:
114  cmd +=' --no_exec'
115  if inFile: #in case previous step used DAS query (either filelist of das:)
116  cmd += ' --filein '+inFile
117  inFile=None
118  if lumiRangeFile: #DAS query can also restrict lumi range
119  cmd += ' --lumiToProcess '+lumiRangeFile
120  lumiRangeFile=None
121  if 'HARVESTING' in cmd and not '134' in str(self.wf.numId) and not '--filein' in cmd:
122  cmd+=' --filein file:step%d_inDQM.root --fileout file:step%d.root '%(istep-1,istep)
123  else:
124  if istep!=1 and not '--filein' in cmd:
125  cmd+=' --filein file:step%s.root '%(istep-1,)
126  if not '--fileout' in com:
127  cmd+=' --fileout file:step%s.root '%(istep,)
128 
129 
130 
131  cmd+=closeCmd(istep,self.wf.nameId)
132  retStep = self.doCmd(cmd)
133 
134  self.retStep.append(retStep)
135  if (retStep!=0):
136  #error occured
137  self.npass.append(0)
138  self.nfail.append(1)
139  self.stat.append('FAILED')
140  #to skip processing
141  aborted=True
142  else:
143  #things went fine
144  self.npass.append(1)
145  self.nfail.append(0)
146  self.stat.append('PASSED')
147 
148 
149  os.chdir(startDir)
150 
151  endtime='date %s' %time.asctime()
152  tottime='%s-%s'%(endtime,startime)
153 
154 
155  #### wrap up ####
156 
157  logStat=''
158  for i,s in enumerate(self.stat):
159  logStat+='Step%d-%s '%(i,s)
160  self.report='%s_%s %s - time %s; exit: '%(self.wf.numId,self.wf.nameId,logStat,tottime)+' '.join(map(str,self.retStep))+'\n'
161 
162  return
163 
164 
165 
dictionary map
Definition: Association.py:205
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
npass
needs to set self.report

Member Data Documentation

WorkFlowRunner.WorkFlowRunner.cafVeto

Definition at line 20 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.dryRun

Definition at line 19 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.doCmd(), and WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.nfail

Definition at line 16 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.noRun

Definition at line 18 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.npass

needs to set self.report

Definition at line 17 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.run().

WorkFlowRunner.WorkFlowRunner.report

wrap up ####

Definition at line 15 of file WorkFlowRunner.py.

Referenced by addOnTests.testit.run().

WorkFlowRunner.WorkFlowRunner.retStep

Definition at line 71 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.stat

Definition at line 70 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.status

Definition at line 14 of file WorkFlowRunner.py.

Referenced by dirstructure.Comparison.__make_image(), and dirstructure.Comparison.__repr__().

WorkFlowRunner.WorkFlowRunner.wf

Definition at line 12 of file WorkFlowRunner.py.

WorkFlowRunner.WorkFlowRunner.wfDir

Definition at line 22 of file WorkFlowRunner.py.

Referenced by WorkFlowRunner.WorkFlowRunner.doCmd(), and WorkFlowRunner.WorkFlowRunner.run().