CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
DTCalibrationWorker.py
Go to the documentation of this file.
1 from CalibMuon.DTCalibration.Workflow.DTTTrigProd import DTTTrigProd
2 from CalibMuon.DTCalibration.Workflow.DTTTrigTimeBoxesWriter import DTTTrigTimeBoxesWriter
3 from CalibMuon.DTCalibration.Workflow.DTResidualCalibration import DTResidualCalibration
4 from CalibMuon.DTCalibration.Workflow.DTTTrigResidualCorr import DTTTrigResidualCorr
5 from CalibMuon.DTCalibration.Workflow.DTTTrigValid import DTTTrigValid
6 from CalibMuon.DTCalibration.Workflow.DTValidSummary import DTValidSummary
7 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentCalibration import DTVDriftSegmentCalibration
8 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentWriter import DTVDriftSegmentWriter
9 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerCalibration import DTVDriftMeanTimerCalibration
10 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerWriter import DTVDriftMeanTimerWriter
11 from CalibMuon.DTCalibration.Workflow.DTNoiseCalibration import DTNoiseCalibration
12 from CalibMuon.DTCalibration.Workflow.DTDQMValidation import DTDQMValidation
13 from CalibMuon.DTCalibration.Workflow.DTDQMMerge import DTDQMMerge
14 from CalibMuon.DTCalibration.Workflow.DTDQMHarvesting import DTDQMHarvesting
15 from CalibMuon.DTCalibration.Workflow.DTDqm import DTDqm
16 from CalibMuon.DTCalibration.Workflow.DTT0DBValidation import DTT0DBValidation
17 from CalibMuon.DTCalibration.Workflow.DTAnalysisResiduals import DTAnalysisResiduals
18 from CalibMuon.DTCalibration.Workflow.CrabWatch import CrabWatch
19 from CalibMuon.DTCalibration.Workflow.tools import listFilesInCastor,haddInCastor,listFilesLocal,haddLocal,copyFilesFromCastor,copyFilesLocal,parseInput,getDatasetStr
20 import sys,os,time,optparse
21 
22 class DTCalibrationWorker:
23  def __init__(self,run,config):
24  self.config = config
25  self.refRun = run
26 
27  def run(self,type,mode,execute):
28  config = None
29  if self.config: config = self.config
30  refRun = 1
31  if self.refRun: refRun = self.refRun
32 
33  if type == 'ttrig': self.runTtrigWorkflow(mode,refRun,config,execute)
34  elif type == 'vdrift': self.runVDriftWorkflow(mode,refRun,config,execute)
35  elif type == 'noise': self.runNoiseWorkflow(mode,refRun,config,execute)
36  elif type == 't0': self.runT0Workflow(mode,refRun,config,execute)
37  elif type == 'validation': self.runValidationWorkflow(mode,refRun,config,execute)
38  elif type == 'analysis': self.runAnalysisWorkflow(mode,refRun,config,execute)
39  elif type == 'dbvalidation':
40  inputFiles = []
41  if config.dbFiles: inputFiles = config.dbFiles
42  self.runDBValidationWorkflow(mode,refRun,inputFiles,config,execute)
43 
44  return 0
45 
46  def dqmOutputDir(self,type,dirLabel,config):
47  dqm_output_dir = ''
48  if config.stageOutLocal:
49  from crab_util import findLastWorkDir
50  cwd = os.getcwd()
51  crab_task_dir = config.base_dir + '/' + dirLabel
52  os.chdir(crab_task_dir)
53  crabdir = findLastWorkDir('crab_0_')
54  if not crabdir: raise RuntimeError('Could not find CRAB dir in %s' % crab_task_dir)
55  os.chdir(cwd)
56  dqm_output_dir = crabdir + "/res"
57  elif config.stageOutCAF:
58  datasetstr = getDatasetStr(config.datasetpath)
59  dqm_output_dir = config.castorpath + '/DTCalibration/' + datasetstr + '/Run' + str(run) + '/' + type + '/' + dirLabel + '/' + 'v' + str(trial)
60 
61  return dqm_output_dir
62 
63  def runTtrigProd(self,run,runselection,trial,config,runStep=True):
64 
65  print "Processing tTrig production"
66  #config.runselection = runselection
67  datasetstr = getDatasetStr(config.datasetpath)
68  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + 'TimeBoxes' + '/' + 'v' + str(trial)
69 
70  task_dir = config.base_dir + '/TimeBoxes'
71  dtTtrigProd = DTTTrigProd(run,task_dir,config)
72  dtTtrigProd.writeCfg()
73 
74  if runStep:
75  project_prod = dtTtrigProd.run()
76 
77  print "Sent calibration jobs with project",project_prod
78  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
79 
80  crabProd = CrabWatch(project_prod)
81  crabProd.setThreshold(config.jobsFinishedThreshold)
82  crabProd.start()
83  crabProd.join()
84 
85  result_file = config.result_dir + '/DTTimeBoxes_%s.root'%run
86  if config.stageOutLocal:
87  output_dir = project_prod + "/res"
88  haddLocal(output_dir,result_file)
89  elif config.stageOutCAF:
90  castor_dir = config.castorpath + "/" + config.userdircaf
91  haddInCastor(castor_dir,result_file,'DTTimeBoxes','rfio://castorcms/','?svcClass=cmscafuser')
92 
93  return project_prod
94 
95  return None
96 
97  def runTtrigWriter(self,run,config,runStep=True):
98 
99  print "Processing tTrig correction"
100  dtTtrigWriter = DTTTrigTimeBoxesWriter(run,config.run_dir,config.result_dir,config)
101  dtTtrigWriter.writeCfg()
102 
103  if runStep:
104  dtTtrigWriter.run()
105 
106  print "Finished processing:"
107  for pset in dtTtrigWriter.configs: print "--->",pset
108 
109  def runResidualCalib(self,run,runselection,trial,input_db,label,result_file,config,runStep=True):
110 
111  print "Processing tTrig calibration"
112  #config.runselection = runselection
113  datasetstr = getDatasetStr(config.datasetpath)
114  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + label + '/' + 'v' + str(trial)
115 
116  task_dir = config.base_dir + '/' + label
117  dtResidualCalib = DTResidualCalibration(run=run,
118  dir=task_dir,
119  input_db=input_db,
120  config=config)
121  dtResidualCalib.writeCfg()
122 
123  if runStep:
124  project_residualCalib = dtResidualCalib.run()
125 
126  print "Sent calibration jobs with project",project_residualCalib
127  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
128 
129  crabResidualCalib = CrabWatch(project_residualCalib)
130  crabResidualCalib.setThreshold(config.jobsFinishedThreshold)
131  crabResidualCalib.start()
132  crabResidualCalib.join()
133 
134  if config.stageOutLocal:
135  output_dir = project_residualCalib + "/res"
136  haddLocal(output_dir,result_file,'residuals')
137  elif config.stageOutCAF:
138  castor_dir = config.castorpath + "/" + config.userdircaf
139  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
140 
141  return project_residualCalib
142 
143  return None
144 
145  def runAnalysisResiduals(self,run,runselection,trial,label,result_file,config,runStep=True):
146 
147  print "Processing residuals analysis"
148  datasetstr = getDatasetStr(config.datasetpath)
149  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/AnalysisResiduals/' + label + '/' + 'v' + str(trial)
150 
151  task_dir = config.base_dir + '/' + label
152  dtAnalysisResiduals = DTAnalysisResiduals(run=run,
153  dir=task_dir,
154  config=config)
155  dtAnalysisResiduals.writeCfg()
156 
157  if runStep:
158  project_analysisResiduals = dtAnalysisResiduals.run()
159 
160  print "Sent jobs with project",project_analysisResiduals
161  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
162 
163  crabAnalysisResiduals = CrabWatch(project_analysisResiduals)
164  crabAnalysisResiduals.setThreshold(config.jobsFinishedThreshold)
165  crabAnalysisResiduals.start()
166  crabAnalysisResiduals.join()
167 
168  if config.stageOutLocal:
169  output_dir = project_analysisResiduals + "/res"
170  haddLocal(output_dir,result_file,'residuals')
171  elif config.stageOutCAF:
172  castor_dir = config.castorpath + "/" + config.userdircaf
173  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
174 
175  return project_analysisResiduals
176 
177  return None
178 
179  def runTtrigResidualCorr(self,run,input_db,root_file,config,runStep=True):
180 
181  print "Processing tTrig residual correction"
182  dtTtrigResidualCorr = DTTTrigResidualCorr(run=run,
183  dir=config.run_dir,
184  input_db=input_db,
185  residuals=root_file,
186  result_dir=config.result_dir,
187  config=config)
188  dtTtrigResidualCorr.writeCfg()
189 
190  if runStep:
191  dtTtrigResidualCorr.run()
192 
193  print "Finished processing:"
194  for pset in dtTtrigResidualCorr.configs: print "--->",pset
195 
196  def runTtrigValid(self,run,runselection,trial,input_db,label,config,runStep=True):
197 
198  print "Processing tTrig validation"
199  #config.runselection = runselection
200  datasetstr = getDatasetStr(config.datasetpath)
201  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigValidation/' + label + '/' + 'v' + str(trial)
202 
203  task_dir = config.base_dir + '/' + label
204  dtTtrigValid = DTTTrigValid(run=run,
205  dir=task_dir,
206  input_db=input_db,
207  config=config)
208  dtTtrigValid.writeCfg()
209 
210  if runStep:
211  project_valid = dtTtrigValid.run()
212 
213  print "Sent validation jobs with project",project_valid
214  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
215 
216  crabValid = CrabWatch(project_valid)
217  crabValid.setThreshold(config.jobsFinishedThreshold)
218  crabValid.start()
219  crabValid.join()
220 
221  """
222  if config.stageOutLocal:
223  output_dir = project_valid + "/res"
224  haddLocal(output_dir,result_file,'residuals')
225  elif config.stageOutCAF:
226  castor_dir = config.castorpath + "/" + config.userdircaf
227  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
228  """
229 
230  return project_valid
231 
232  return None
233 
234  def runTtrigValidSummary(self,run,input_file,output_file,config,runStep=True):
235 
236  print "Processing Validation Summary"
237  dtTtrigValidSummary = DTValidSummary(run,config.run_dir,input_file,output_file,config)
238  dtTtrigValidSummary.writeCfg()
239 
240  if runStep:
241  dtTtrigValidSummary.run()
242 
243  print "...Validation Summary finished"
244 
245  def runVDriftSegmentCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
246 
247  print "Processing vDrift calibration"
248  #config.runselection = runselection
249  datasetstr = getDatasetStr(config.datasetpath)
250  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
251 
252  task_dir = config.base_dir + '/' + label
253  dtVDriftSegment = DTVDriftSegmentCalibration(run=run,
254  dir=task_dir,
255  config=config)
256  dtVDriftSegment.writeCfg()
257 
258  if runStep:
259  project_segment = dtVDriftSegment.run()
260 
261  print "Sent validation jobs with project",project_segment
262  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
263 
264  crabVDriftSegment = CrabWatch(project_segment)
265  crabVDriftSegment.setThreshold(config.jobsFinishedThreshold)
266  crabVDriftSegment.start()
267  crabVDriftSegment.join()
268 
269  if config.stageOutLocal:
270  output_dir = project_segment + "/res"
271  haddLocal(output_dir,result_file,'DTVDriftHistos')
272  elif config.stageOutCAF:
273  castor_dir = config.castorpath + "/" + config.userdircaf
274  haddInCastor(castor_dir,result_file,'DTVDriftHistos','rfio://castorcms/','?svcClass=cmscafuser')
275 
276  return project_segment
277 
278  return None
279 
280  def runVDriftSegmentWriter(self,run,root_file,config,runStep=True):
281 
282  print "Processing vDrift writer"
283  dtVDriftSegmentWriter = DTVDriftSegmentWriter(run=run,
284  dir=config.run_dir,
285  input_file=root_file,
286  output_dir=config.result_dir,
287  config=config)
288  dtVDriftSegmentWriter.writeCfg()
289 
290  if runStep:
291  dtVDriftSegmentWriter.run()
292 
293  print "Finished processing:"
294  for pset in dtVDriftSegmentWriter.configs: print "--->",pset
295 
296  def runVDriftMeanTimerCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
297 
298  print "Processing vDrift calibration"
299  #config.runselection = runselection
300  datasetstr = getDatasetStr(config.datasetpath)
301  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
302 
303  task_dir = config.base_dir + '/' + label
304  dtVDriftMeanTimer = DTVDriftMeanTimerCalibration(run=run,
305  dir=task_dir,
306  config=config)
307  dtVDriftMeanTimer.writeCfg()
308 
309  if runStep:
310  project_meantimer = dtVDriftMeanTimer.run()
311 
312  print "Sent validation jobs with project",project_meantimer
313  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
314 
315  crabVDriftMeanTimer = CrabWatch(project_meantimer)
316  crabVDriftMeanTimer.setThreshold(config.jobsFinishedThreshold)
317  crabVDriftMeanTimer.start()
318  crabVDriftMeanTimer.join()
319 
320  if config.stageOutLocal:
321  output_dir = project_meantimer + "/res"
322  haddLocal(output_dir,result_file,'DTTMaxHistos')
323  elif config.stageOutCAF:
324  castor_dir = config.castorpath + "/" + config.userdircaf
325  haddInCastor(castor_dir,result_file,'DTTMaxHistos','rfio://castorcms/','?svcClass=cmscafuser')
326 
327  return project_meantimer
328 
329  return None
330 
331  def runVDriftMeanTimerWriter(self,run,root_file,config,runStep=True):
332 
333  print "Processing vDrift writer"
334  dtVDriftMeanTimerWriter = DTVDriftMeanTimerWriter(run=run,
335  dir=config.run_dir,
336  input_file=root_file,
337  output_dir=config.result_dir,
338  config=config)
339  dtVDriftMeanTimerWriter.writeCfg()
340 
341  if runStep:
342  dtVDriftMeanTimerWriter.run()
343 
344  print "Finished processing:"
345  for pset in dtVDriftMeanTimerWriter.configs: print "--->",pset
346 
347  def runDQMClient(self,run,output_dir,config,runStep=True):
348 
349  print "Processing DQM Merge"
350 
351  if runStep:
352  dqm_files = []
353  if config.stageOutLocal:
354  dqm_files = listFilesLocal(output_dir,'DQM')
355  dqm_files = ['file:%s' % item for item in dqm_files]
356  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
357  dtDqmFinal.writeCfg()
358  dtDqmFinal.run()
359  elif config.stageOutCAF:
360  dqm_files = listFilesInCastor(output_dir,'DQM','')
361  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
362  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
363  dtDqmFinal.writeCfg()
364  dtDqmFinal.run()
365 
366  print "...DQM Merge finished"
367  else:
368  dqm_files = []
369  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
370  dtDqmFinal.writeCfg()
371 
372  def runDQMHarvesting(self,run,output_dir,config,runStep=True):
373 
374  print "Processing DQM harvesting"
375 
376  if runStep:
377  dqm_files = []
378  if config.stageOutLocal:
379  dqm_files = listFilesLocal(output_dir,'DQM')
380  dqm_files = ['file:%s' % item for item in dqm_files]
381  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
382  dtDqmFinal.writeCfg()
383  dtDqmFinal.run()
384  elif config.stageOutCAF:
385  dqm_files = listFilesInCastor(output_dir,'DQM','')
386  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
387  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
388  dtDqmFinal.writeCfg()
389  dtDqmFinal.run()
390 
391  print "...DQM harvesting finished"
392  else:
393  dqm_files = []
394  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
395  dtDqmFinal.writeCfg()
396 
397  def runDQMMerge(self,run,output_dir,config,runStep=True):
398 
399  print "Processing DQM merge"
400 
401  if runStep:
402  dqm_files = []
403  if config.stageOutLocal:
404  dqm_files = listFilesLocal(output_dir,'DQM')
405  dqm_files = ['file:%s' % item for item in dqm_files]
406  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
407  dtDQMMerge.writeCfg()
408  dtDQMMerge.run()
409  elif config.stageOutCAF:
410  dqm_files = listFilesInCastor(output_dir,'DQM','')
411  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
412  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
413  dtDQMMerge.writeCfg()
414  dtDQMMerge.run()
415 
416  print "...DQM merge finished"
417  else:
418  dqm_files = []
419  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
420  dtDQMMerge.writeCfg()
421 
422  ############################################################
423  # tTrig workflow
424  ############################################################
425  def runTtrigWorkflow(self,mode,run,config,execute=True):
426  trial = config.trial
427  runselection = config.runselection
428  ttrig_input_db = None
429  if hasattr(config,'inputTTrigDB') and config.inputTTrigDB: ttrig_input_db = os.path.abspath(config.inputTTrigDB)
430  result_dir = config.result_dir
431  if mode == 'timeboxes':
432  timeBoxes = os.path.abspath(result_dir + '/' + 'DTTimeBoxes_' + run + '.root')
433  ttrig_timeboxes_db = os.path.abspath(result_dir + '/' + 'ttrig_timeboxes_' + run + '.db')
434  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
435  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
436  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
437  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
438 
439  if not execute:
440  print "Writing configuration files.."
441  self.runTtrigProd(run,runselection,trial,config,False)
442  self.runTtrigWriter(run,config,False)
443  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config,False)
444  self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config,False)
445  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
446  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
447  self.runDQMClient(run,'',config,False)
448 
449  sys.exit(0)
450 
451  # Produce time-boxes
452  if not os.path.exists(timeBoxes): self.runTtrigProd(run,runselection,trial,config)
453  if not os.path.exists(timeBoxes): raise RuntimeError('Could not produce %s' % timeBoxes)
454 
455  # Write tTrig DB
456  if not os.path.exists(ttrig_timeboxes_db): self.runTtrigWriter(run,config)
457  if not os.path.exists(ttrig_timeboxes_db): raise RuntimeError('Could not produce %s' % ttrig_timeboxes_db)
458 
459  # Produce residuals
460  if not os.path.exists(residualsFirst):
461  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config)
462  if not os.path.exists(residualsFirst): raise RuntimeError('Could not produce %s' % residualsFirst)
463 
464  # Correction from residuals and write tTrig DB
465  if not os.path.exists(ttrig_residuals_db): self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config)
466  if not os.path.exists(ttrig_residuals_db): raise RuntimeError('Could not produce %s' % ttrig_residuals_db)
467 
468  # Validation
469  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
470 
471  """
472  # Summary of validation
473  if not os.path.exists(summaryResiduals): self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
474  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
475  """
476  # Produce DQM output
477  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
478  self.runDQMClient(run,dqm_output_dir,config)
479 
480  elif mode == 'residuals':
481  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
482  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
483  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
484  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
485 
486  if not execute:
487  print "Writing configuration files.."
488  if ttrig_input_db:
489  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config,False)
490  self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config,False)
491  else:
492  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config,False)
493  self.runTtrigResidualCorr(run,None,residualsFirst,config,False)
494 
495  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
496  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
497 
498  self.runDQMClient(run,'',config,False)
499 
500  sys.exit(0)
501 
502  # Produce residuals
503  if not os.path.exists(residualsFirst):
504  if ttrig_input_db:
505  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config)
506  else:
507  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config)
508  if not os.path.exists(residualsFirst): raise RuntimeError('Could not produce %s' % residualsFirst)
509 
510  # Correction from residuals and write tTrig DB
511  if not os.path.exists(ttrig_residuals_db):
512  if ttrig_input_db: self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config)
513  else: self.runTtrigResidualCorr(run,None,residualsFirst,config)
514  if not os.path.exists(ttrig_residuals_db): raise RuntimeError('Could not produce %s' % ttrig_residuals_db)
515 
516  # Validation
517  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
518 
519  """
520  # Summary of validation
521  if not os.path.exists(summaryResiduals):
522  self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
523  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
524  """
525  # Produce DQM output
526  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
527  self.runDQMClient(run,dqm_output_dir,config)
528 
529  elif mode == 'validation':
530  residualsValid = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
531  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
532 
533  if not execute:
534  print "Writing configuration files.."
535  if ttrig_input_db:
536  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config,False)
537  else:
538  self.runTtrigValid(run,runselection,trial,None,'Residuals',config,False)
539 
540  #self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config,False)
541  self.runDQMClient(run,'',config,False)
542 
543  sys.exit(0)
544 
545  # Validation
546  if ttrig_input_db:
547  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config)
548  else:
549  self.runTtrigValid(run,runselection,trial,None,'Residuals',config)
550 
551  """
552  # Summary of validation
553  if not os.path.exists(summaryResiduals):
554  self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config)
555  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
556  """
557 
558  # Produce DQM output
559  dqm_output_dir = self.dqmOutputDir('TTrigValidation','Residuals',config)
560  self.runDQMClient(run,dqm_output_dir,config)
561 
562  return 0
563 
564  ############################################################
565  # vDrift workflow
566  ############################################################
567  def runVDriftWorkflow(self,mode,run,config,execute=True):
568  trial = config.trial
569  runselection = config.runselection
570  result_dir = config.result_dir
571  if mode == 'segment':
572  vDriftHistos = os.path.abspath(result_dir + '/' + 'DTVDriftHistos_' + run + '.root')
573  vDrift_segment_db = os.path.abspath(result_dir + '/' + 'vDrift_segment_' + run + '.db')
574 
575  if not execute:
576  print "Writing configuration files.."
577  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config,False)
578  self.runVDriftSegmentWriter(run,vDriftHistos,config,False)
579 
580  sys.exit(0)
581 
582  # Produce vDrift histos
583  if not os.path.exists(vDriftHistos):
584  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config)
585  if not os.path.exists(vDriftHistos): raise RuntimeError('Could not produce %s' % vDriftHistos)
586 
587  # Write vDrift DB
588  if not os.path.exists(vDrift_segment_db): self.runVDriftSegmentWriter(run,vDriftHistos,config)
589  if not os.path.exists(vDrift_segment_db): raise RuntimeError('Could not produce %s' % vDrift_segment_db)
590 
591  elif mode == 'meantimer':
592  vDriftTMaxHistos = os.path.abspath(result_dir + '/' + 'DTTMaxHistos_' + run + '.root')
593  vDrift_meantimer_db = os.path.abspath(result_dir + '/' + 'vDrift_meantimer_' + run + '.db')
594 
595  if not execute:
596  print "Writing configuration files.."
597  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config,False)
598  self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config,False)
599 
600  sys.exit(0)
601 
602  # Produce t_max histos
603  if not os.path.exists(vDriftTMaxHistos):
604  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config)
605  if not os.path.exists(vDriftTMaxHistos): raise RuntimeError('Could not produce %s' % vDriftTMaxHistos)
606 
607  # Write vDrift DB
608  if not os.path.exists(vDrift_meantimer_db): self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config)
609  if not os.path.exists(vDrift_meantimer_db): raise RuntimeError('Could not produce %s' % vDrift_meantimer_db)
610 
611  return 0
612 
613  ############################################################
614  # noise workflow
615  ############################################################
616  def runNoiseWorkflow(self,mode,run,config,execute=True):
617  print "Processing noise calibration"
618 
619  trial = config.trial
620  runselection = config.runselection
621  result_dir = config.result_dir
622  result_file = os.path.abspath(result_dir + '/' + 'dtNoiseCalib_' + run + '.root')
623  noise_db = os.path.abspath(result_dir + '/' + 'noise_' + run + '.db')
624  noise_txt = os.path.abspath(result_dir + '/' + 'noise_' + run + '.txt')
625 
626  datasetstr = getDatasetStr(config.datasetpath)
627  #config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/NoiseCalibration/' + label + '/' + 'v' + str(trial)
628  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/NoiseCalibration/' + 'v' + str(trial)
629 
630  task_dir = config.base_dir + '/NoiseCalib'
631  dtNoiseCalibration = DTNoiseCalibration(run=run,
632  dir=task_dir,
633  config=config)
634  if not execute:
635  dtNoiseCalibration.writeCfg()
636  sys.exit(0)
637  else:
638  dtNoiseCalibration.writeCfg()
639  project_noise = dtNoiseCalibration.run()
640 
641  print "Sent calibration jobs with project",project_noise
642  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
643 
644  crabNoiseCalibration = CrabWatch(project_noise)
645  crabNoiseCalibration.setThreshold(config.jobsFinishedThreshold)
646  crabNoiseCalibration.start()
647  crabNoiseCalibration.join()
648 
649  if config.stageOutLocal:
650  crab_output_dir = project_noise + "/res"
651  retcode = copyFilesLocal(crab_output_dir,result_dir,'dtNoiseCalib')
652  retcode = copyFilesLocal(crab_output_dir,result_dir,'noise')
653  elif config.stageOutCAF:
654  castor_dir = config.castorpath + "/" + config.userdircaf
655  retcode = copyFilesFromCastor(castor_dir,result_dir,'dtNoiseCalib')
656  retcode = copyFilesFromCastor(castor_dir,result_dir,'noise')
657 
658  return 0
659 
660  ############################################################
661  # t0 workflow
662  ############################################################
663  def runT0Workflow(self,mode,run,config,execute=True):
664 
665  return 0
666 
667  ############################################################
668  # Validation workflow
669  ############################################################
670  def runValidationWorkflow(self,mode,run,config,execute=True):
671  print "Processing DQM validation"
672  trial = config.trial
673  runselection = config.runselection
674  result_dir = config.result_dir
675  datasetstr = getDatasetStr(config.datasetpath)
676  dirLabel = 'DQM'
677  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/DQMValidation/' + dirLabel + '/' + 'v' + str(trial)
678 
679  task_dir = config.base_dir + '/' + dirLabel
680  dtDQMValid = DTDQMValidation(run=run,
681  dir=task_dir,
682  config=config)
683  if not execute:
684  dtDQMValid.writeCfg()
685  self.runDQMMerge(run,'',config,False)
686  self.runDQMHarvesting(run,'',config,False)
687 
688  sys.exit(0)
689  else:
690  dtDQMValid.writeCfg()
691  project_valid = dtDQMValid.run()
692 
693  print "Sent validation jobs with project",project_valid
694  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
695 
696  crabValid = CrabWatch(project_valid)
697  crabValid.setThreshold(config.jobsFinishedThreshold)
698  crabValid.start()
699  crabValid.join()
700 
701  # Produce DQM output
702  dqm_output_dir = self.dqmOutputDir('DQMValidation',dirLabel,config)
703  self.runDQMMerge(run,dqm_output_dir,config)
704  # Run harvesting from merged DQM file
705  dqm_merge_dir = os.path.abspath(result_dir)
706  self.runDQMHarvesting(run,dqm_merge_dir,config)
707 
708  return 0
709 
710  ############################################################
711  # Analysis workflow
712  ############################################################
713  def runAnalysisWorkflow(self,mode,run,config,execute=True):
714  print "Processing analysis workflow"
715  trial = config.trial
716  runselection = config.runselection
717  result_dir = config.result_dir
718  if mode == 'residuals':
719  residualsFile = os.path.abspath(result_dir + '/' + 'DTResiduals_' + run + '.root')
720 
721  if not execute:
722  print "Writing configuration files.."
723  self.runAnalysisResiduals(run,runselection,trial,'Residuals',residualsFile,config,False)
724 
725  sys.exit(0)
726 
727  # Produce residuals
728  if not os.path.exists(residualsFile):
729  self.runAnalysisResiduals(run,runselection,trial,'Residuals',residualsFile,config)
730  if not os.path.exists(residualsFile): raise RuntimeError('Could not produce %s' % residualsFile)
731 
732  return 0
733 
734  ############################################################
735  # DB Validation workflow
736  ############################################################
737  def runDBValidationWorkflow(self,mode,run,inputFiles,config,execute=True):
738  print "Processing DB validation"
739 
740  dtDBValidation = None
741  if mode == 't0DB':
742  dtDBValidation = DTT0DBValidation(run=run,
743  dir=config.run_dir,
744  input_files=inputFiles,
745  output_dir=config.result_dir,
746  config=config)
747  #dtDBValidation.writeCfg()
748 
749  if execute:
750  dtDBValidation.run()
751 
752  print "Finished processing:"
753  for pset in dtDBValidation.configs: print "--->",pset
754 
755  return 0
DTCalibrationWorker.DTCalibrationWorker.runDBValidationWorkflow
def runDBValidationWorkflow
DB Validation workflow.
Definition: DTCalibrationWorker.py:737
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.getDatasetStr
def getDatasetStr
Definition: tools.py:15
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
DTDqm
Definition: DTDqm.py:1
tools.haddLocal
def haddLocal
Definition: tools.py:98
tools.copyFilesFromCastor
def copyFilesFromCastor
Definition: tools.py:55
DTCalibrationWorker.DTCalibrationWorker.runValidationWorkflow
def runValidationWorkflow
Validation workflow.
Definition: DTCalibrationWorker.py:670
tools.haddInCastor
def haddInCastor
Definition: tools.py:81
tools.copyFilesLocal
def copyFilesLocal
Definition: tools.py:68
Generated for CMSSW Reference Manual by doxygen 1.8.5