test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Functions | Variables
crabWrap Namespace Reference

Functions

def checkStatus
 
def computeSummaryCRAB251
 
def computeSummaryCRAB260
 
def convertStatus
 
def crabAction
 
def crabActionCRAB251
 
def crabCreate
 
def crabStatus
 
def crabSubmit
 
def crabWatch
 
def getOutput
 
def initCrabEnvironment
 
def run
 
def summaryServer
 
def summaryStandAlone
 

Variables

 computeSummary = computeSummaryCRAB260
 
 project = None
 
float threshold = 95.0
 

Function Documentation

def crabWrap.checkStatus (   project,
  threshold = 95.0 
)

Definition at line 205 of file crabWrap.py.

References convertStatus(), crabStatus(), and bookConverter.max.

Referenced by crabWatch(), and production_tasks.MonitorJobs.run().

205 
206 def checkStatus(project, threshold = 95.0):
207 
208  status = crabStatus(project)
209 
210  print "Percentage of jobs per status:"
211  maxLength = max( [len(x) for x in status] )
212  for item in status:
213  print "%*s: %.0f%%" % (maxLength,item,status[item])
214 
215 
216  statusNew = convertStatus(status)
217 
218  print "Relative percentage finished: %.0f%%" % statusNew['Finished']
219  print "Relative percentage failed : %.0f%%" % statusNew['Failed']
220  print "Relative percentage running : %.0f%%" % statusNew['Running']
221 
222  finished = False
223  # Condition for stopping
224  #if fracFailed > 50.0: raise RuntimeError,'Too many jobs have failed (%.0f%%).' % fracFailed
225 
226  # Condition for considering it finished
227  if statusNew['Finished'] >= threshold: finished = True
228 
229  return finished
crabWrap.convertStatus
def convertStatus
Definition: crabWrap.py:174
crabWrap.crabStatus
def crabStatus
Definition: crabWrap.py:157
crabWrap.checkStatus
def checkStatus
Definition: crabWrap.py:205
def crabWrap.computeSummaryCRAB251 (   up_task)

Definition at line 38 of file crabWrap.py.

References split.

38 
39 def computeSummaryCRAB251(up_task):
40  "Computes jobs summary for given task"
41 
42  taskId = str(up_task['name'])
43  task_unique_name = str(up_task['name'])
44  ended = None
45 
46  summary = {}
47  nJobs = 0
48  for job in up_task.jobs :
49  id = str(job.runningJob['jobId'])
50  jobStatus = str(job.runningJob['statusScheduler'])
51  dest = str(job.runningJob['destination']).split(':')[0]
52  exe_exit_code = str(job.runningJob['applicationReturnCode'])
53  job_exit_code = str(job.runningJob['wrapperReturnCode'])
54  ended = str(job['standardInput'])
55  printline=''
56  if dest == 'None' : dest = ''
57  if exe_exit_code == 'None' : exe_exit_code = ''
58  if job_exit_code == 'None' : job_exit_code = ''
59  #printline+="%-6s %-18s %-36s %-13s %-16s %-4s" % (id,jobStatus,dest,exe_exit_code,job_exit_code,ended)
60  #print printline
61  if jobStatus in summary: summary[jobStatus] += 1
62  else: summary[jobStatus] = 1
63  nJobs += 1
64 
65  for item in summary: summary[item] = 100.*summary[item]/nJobs
66 
67  return summary
crabWrap.computeSummaryCRAB251
def computeSummaryCRAB251
Definition: crabWrap.py:38
split
double split
Definition: MVATrainer.cc:139
def crabWrap.computeSummaryCRAB260 (   up_task) 
Computes jobs summary for given task

Definition at line 5 of file crabWrap.py.

References split.

5 
6 def computeSummaryCRAB260(up_task):
7  """
8  Computes jobs summary for given task
9  """
10  taskId = str(up_task['name'])
11  task_unique_name = str(up_task['name'])
12  ended = None
13 
14  summary = {}
15  nJobs = 0
16  for job in up_task.jobs :
17  id = str(job.runningJob['jobId'])
18  jobStatus = str(job.runningJob['statusScheduler'])
19  jobState = str(job.runningJob['state'])
20  dest = str(job.runningJob['destination']).split(':')[0]
21  exe_exit_code = str(job.runningJob['applicationReturnCode'])
22  job_exit_code = str(job.runningJob['wrapperReturnCode'])
23  ended = str(job['closed'])
24  printline=''
25  if dest == 'None' : dest = ''
26  if exe_exit_code == 'None' : exe_exit_code = ''
27  if job_exit_code == 'None' : job_exit_code = ''
28  if job.runningJob['state'] == 'SubRequested' : jobStatus = 'Submitting'
29  if job.runningJob['state'] == 'Terminated': jobStatus = 'Done'
30 
31  if jobStatus in summary: summary[jobStatus] += 1
32  else: summary[jobStatus] = 1
33  nJobs += 1
34 
35  for item in summary: summary[item] = 100.*summary[item]/nJobs
36 
37  return summary
crabWrap.computeSummaryCRAB260
def computeSummaryCRAB260
Definition: crabWrap.py:5
split
double split
Definition: MVATrainer.cc:139
def crabWrap.convertStatus (   status) 
doneStatus = ['Done','Done (success)','Cleared','Retrieved']
failedStatus = ['Aborted','Done (failed)','Killed','Cancelled']
ignoreStatus = ['Created']

Definition at line 174 of file crabWrap.py.

Referenced by gen::HydjetHadronizer.build_hyjet(), checkStatus(), gen::Hydjet2Hadronizer.generatePartonsAndHadronize(), and CrabWatch.CrabWatch.run().

174 
175 def convertStatus(status):
176  """
177  doneStatus = ['Done','Done (success)','Cleared','Retrieved']
178  failedStatus = ['Aborted','Done (failed)','Killed','Cancelled']
179  ignoreStatus = ['Created']
180  """
181  doneStatus = ['SD','E']
182  failedStatus = ['A','DA','K']
183  runningStatus = ['R']
184  ignoreStatus = ['C']
185  sumDone = 0.0
186  sumFailed = 0.0
187  sumRunning = 0.0
188  sumIgnore = 0.0
189  for key in status:
190  if key in doneStatus: sumDone += status[key]
191  if key in failedStatus: sumFailed += status[key]
192  if key in runningStatus: sumRunning += status[key]
193  if key in ignoreStatus: sumIgnore += status[key]
194 
195  # frac(done)' = N*frac(done)/(N - N*frac(ignore)) = frac(done)/(1 - frac(ignore))
196  fracDone = 100.0*sumDone/(100.0 - sumIgnore)
197  fracFailed = 100.0*sumFailed/(100.0 - sumIgnore)
198  fracRun = 100.0*sumRunning/(100.0 - sumIgnore)
199 
200  result = {'Finished':fracDone,
201  'Failed':fracFailed,
202  'Running':fracRun}
203 
204  return result
crabWrap.convertStatus
def convertStatus
Definition: crabWrap.py:174
def crabWrap.crabAction (   options,
  action = None 
)

Definition at line 95 of file crabWrap.py.

References mps_fire.action.

Referenced by crabCreate(), crabStatus(), crabSubmit(), and getOutput().

95 
96 def crabAction(options, action = None):
97 
98  options = parseOptions(options)
99 
100  crab = Crab()
101  result = None
102  try:
103  crab.initialize_(options)
104  crab.run()
105  if action: result = action(crab)
106  del crab
107  print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)
108  except CrabException as e:
109  del crab
110  #print '\n' + common.prog_name + ': ' + str(e) + '\n'
111  raise
112 
113  if (common.logger): common.logger.delete()
114 
115  if result: return result
crabWrap.crabAction
def crabAction
Definition: crabWrap.py:95
mps_fire.action
string action
Definition: mps_fire.py:28
def crabWrap.crabActionCRAB251 (   options,
  action = None 
)

Definition at line 116 of file crabWrap.py.

References mps_fire.action.

116 
117 def crabActionCRAB251(options, action = None):
118 
119  options = parseOptions(options)
120 
121  result = None
122  try:
123  crab = Crab(options)
124  crab.run()
125  common.apmon.free()
126  if action: result = action(crab)
127  del crab
128  #print 'Log file is %s%s.log'%(common.work_space.logDir(),common.prog_name)
129  #print '\n############################## E N D ####################################\n'
130  except CrabException as e:
131  print '\n' + common.prog_name + ': ' + str(e) + '\n'
132  pass
133  pass
134  #if (common.logger): common.logger.delete()
135 
136  if result: return result
crabWrap.crabActionCRAB251
def crabActionCRAB251
Definition: crabWrap.py:116
mps_fire.action
string action
Definition: mps_fire.py:28
def crabWrap.crabCreate (   dir = '.',
  crabCfg_name = 'crab.cfg' 
)

Definition at line 137 of file crabWrap.py.

References crabAction().

137 
138 def crabCreate(dir = '.', crabCfg_name = 'crab.cfg'):
139 
140  cwd = os.getcwd()
141  os.chdir(dir)
142 
143  options = ['-create','-cfg',crabCfg_name]
144 
145  project = crabAction(options,lambda crab: common.work_space.topDir())
146 
147  os.chdir(cwd)
148 
149  return project
crabWrap.crabAction
def crabAction
Definition: crabWrap.py:95
crabWrap.crabCreate
def crabCreate
Definition: crabWrap.py:137
def crabWrap.crabStatus (   project)

Definition at line 157 of file crabWrap.py.

References mps_fire.action, and crabAction().

Referenced by checkStatus(), and CrabWatch.CrabWatch.run().

157 
158 def crabStatus(project):
159  options = ['-status']
160  if project:
161  options.append('-c')
162  options.append(project)
163 
164  def action(crab):
165  #act = '-status'
166  #return crab.actions[act].summary()
167  xml = crab.cfg_params.get("USER.xml_report",'')
168  return common.work_space.shareDir() + xml
169 
170  xmlreport = crabAction(options,action)
171  status = crabStatusFromReport(xmlreport)
172 
173  return status
crabWrap.crabAction
def crabAction
Definition: crabWrap.py:95
crabWrap.crabStatus
def crabStatus
Definition: crabWrap.py:157
mps_fire.action
string action
Definition: mps_fire.py:28
def crabWrap.crabSubmit (   project)

Definition at line 150 of file crabWrap.py.

References crabAction().

Referenced by CrabTask.CrabTask.submit().

150 
151 def crabSubmit(project):
152  options = ['-submit','-c',project]
153 
154  crabAction(options)
155 
156  return
crabWrap.crabAction
def crabAction
Definition: crabWrap.py:95
crabWrap.crabSubmit
def crabSubmit
Definition: crabWrap.py:150
def crabWrap.crabWatch (   action,
  project = None,
  threshold = 95.0 
)

Definition at line 240 of file crabWrap.py.

References mps_fire.action, and checkStatus().

Referenced by run().

240 
241 def crabWatch(action,project = None, threshold = 95.0):
242  #for i in range(5):
243  while True:
244  if checkStatus(project,threshold): break
245  time.sleep(180)
246 
247  print "Finished..."
248 
249  action(project)
250 
251  return
mps_fire.action
string action
Definition: mps_fire.py:28
crabWrap.checkStatus
def checkStatus
Definition: crabWrap.py:205
crabWrap.crabWatch
def crabWatch
Definition: crabWrap.py:240
def crabWrap.getOutput (   project)

Definition at line 230 of file crabWrap.py.

References crabAction().

Referenced by CrabTask.CrabTask.getoutput().

230 
231 def getOutput(project):
232  options = ['-getoutput']
233  if project:
234  options.append('-c')
235  options.append(project)
236 
237  crabAction(options)
238 
239  return
crabWrap.crabAction
def crabAction
Definition: crabWrap.py:95
crabWrap.getOutput
def getOutput
Definition: crabWrap.py:230
def crabWrap.initCrabEnvironment ( )

Definition at line 252 of file crabWrap.py.

References alcazmumu_cfi.filter, spr.find(), join(), and split.

252 
253 def initCrabEnvironment():
254  pythonpathenv = os.environ['PYTHONPATH']
255  pythonpathbegin = pythonpathenv.split(':')[0].rstrip('/')
256  pythonpathend = pythonpathenv.split(':')[-1].rstrip('/')
257 
258  indexBegin = sys.path.index(pythonpathbegin)
259  if 'CRABPSETPYTHON' in os.environ: sys.path.insert( indexBegin, os.environ['CRABPSETPYTHON'] )
260  if 'CRABDLSAPIPYTHON' in os.environ: sys.path.insert( indexBegin, os.environ['CRABDLSAPIPYTHON'] )
261  if 'CRABDBSAPIPYTHON' in os.environ: sys.path.insert( indexBegin, os.environ['CRABDBSAPIPYTHON'] )
262 
263  if os.environ['SCRAM_ARCH'].find('32') != -1 and 'CRABPYSQLITE' in os.environ:
264  sys.path.insert( indexBegin, os.environ['CRABPYSQLITE'] )
265  elif os.environ['SCRAM_ARCH'].find('64') != -1 and 'CRABPYSQLITE64' in os.environ:
266  sys.path.insert( indexBegin, os.environ['CRABPYSQLITE64'] )
267 
268  indexEnd = sys.path.index(pythonpathend) + 1
269  if 'CRABPYTHON' in os.environ:
270  if indexEnd >= len(sys.path): sys.path.append( os.environ['CRABPYTHON'] )
271  else: sys.path.insert( indexEnd, os.environ['CRABPYTHON'] )
272 
273  #print sys.path
274 
275  #os.environ['LD_LIBRARY_PATH'] = os.environ['GLITE_LOCATION'] + '/lib' + ':' + os.environ['LD_LIBRARY_PATH']
276  os.environ['VOMS_PROXY_INFO_DONT_VERIFY_AC'] = '1'
277  #print os.environ['LD_LIBRARY_PATH']
278  #print os.environ['VOMS_PROXY_INFO_DONT_VERIFY_AC']
279 
280  """
281  export LD_LIBRARY_PATH=${GLITE_LOCATION}/lib:${LD_LIBRARY_PATH}
282  export VOMS_PROXY_INFO_DONT_VERIFY_AC=1
283  """
284 
285  ## Get rid of some useless warning
286  try:
287  import warnings
288  warnings.simplefilter("ignore", RuntimeWarning)
289  # import socket
290  # socket.setdefaulttimeout(15) # Default timeout in seconds
291  except ImportError:
292  pass # too bad, you'll get the warning
293 
294  # Remove libraries which over-ride CRAB libs and DBS_CONFIG setting
295  badPaths = []
296  if 'DBSCMD_HOME' in os.environ: # CMSSW's DBS, remove last bit of path
297  badPaths.append('/'.join(os.environ['DBSCMD_HOME'].split('/')[:-1]))
298  if 'DBS_CLIENT_CONFIG' in os.environ:
299  del os.environ['DBS_CLIENT_CONFIG']
300 
301  def pathIsGood(checkPath):
302  """
303  Filter function for badPaths
304  """
305  for badPath in badPaths:
306  if checkPath.find(badPath) != -1:
307  return False
308  return True
309 
310  sys.path = filter(pathIsGood, sys.path)
crabWrap.initCrabEnvironment
def initCrabEnvironment
Definition: crabWrap.py:252
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
spr::find
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:7
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
split
double split
Definition: MVATrainer.cc:139
def crabWrap.run (   project = None,
  threshold = 95.0 
)

Definition at line 311 of file crabWrap.py.

References crabWatch().

311 
312 def run(project = None, threshold = 95.0):
313 
314  crabWatch(getOutput,project,threshold)
315 
316  return
crabWrap.run
def run
Definition: crabWrap.py:311
crabWrap.crabWatch
def crabWatch
Definition: crabWrap.py:240
def crabWrap.summaryServer (   self) 
Returns jobs summary

Definition at line 78 of file crabWrap.py.

References computeSummary.

78 
79 def summaryServer(self):
80  """
81  Returns jobs summary
82  """
83  #self.resynchClientSide()
84 
85  upTask = common._db.getTask()
86  return computeSummary(upTask)
87 
88 """
89 # Add method to Status classes
90 import Status
91 import StatusServer
92 Status.Status.summary = summaryStandAlone
93 StatusServer.StatusServer.summary = summaryServer
94 """
crabWrap.computeSummary
computeSummary
Definition: crabWrap.py:68
crabWrap.summaryServer
def summaryServer
Definition: crabWrap.py:78
def crabWrap.summaryStandAlone (   self) 
Returns jobs summary

Definition at line 70 of file crabWrap.py.

References computeSummary.

70 
71 def summaryStandAlone(self):
72  """
73  Returns jobs summary
74  """
75  task = common._db.getTask()
76  upTask = common.scheduler.queryEverything(task['id'])
77  return computeSummary(upTask)
crabWrap.computeSummary
computeSummary
Definition: crabWrap.py:68
crabWrap.summaryStandAlone
def summaryStandAlone
Definition: crabWrap.py:70

Variable Documentation

crabWrap.computeSummary = computeSummaryCRAB260

Definition at line 68 of file crabWrap.py.

Referenced by summaryServer(), and summaryStandAlone().

list crabWrap.project = None

Definition at line 318 of file crabWrap.py.

tuple crabWrap.threshold = 95.0

Definition at line 319 of file crabWrap.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5