CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
DTCalibrationWorker.py
Go to the documentation of this file.
1 from CalibMuon.DTCalibration.Workflow.DTTTrigProd import DTTTrigProd
2 from CalibMuon.DTCalibration.Workflow.DTTTrigTimeBoxesWriter import DTTTrigTimeBoxesWriter
3 from CalibMuon.DTCalibration.Workflow.DTResidualCalibration import DTResidualCalibration
4 from CalibMuon.DTCalibration.Workflow.DTTTrigResidualCorr import DTTTrigResidualCorr
5 from CalibMuon.DTCalibration.Workflow.DTTTrigValid import DTTTrigValid
6 from CalibMuon.DTCalibration.Workflow.DTValidSummary import DTValidSummary
7 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentCalibration import DTVDriftSegmentCalibration
8 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentWriter import DTVDriftSegmentWriter
9 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerCalibration import DTVDriftMeanTimerCalibration
10 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerWriter import DTVDriftMeanTimerWriter
11 from CalibMuon.DTCalibration.Workflow.DTNoiseCalibration import DTNoiseCalibration
12 from CalibMuon.DTCalibration.Workflow.DTDQMValidation import DTDQMValidation
13 from CalibMuon.DTCalibration.Workflow.DTDQMMerge import DTDQMMerge
14 from CalibMuon.DTCalibration.Workflow.DTDQMHarvesting import DTDQMHarvesting
15 from CalibMuon.DTCalibration.Workflow.DTDqm import DTDqm
16 from CalibMuon.DTCalibration.Workflow.DTT0DBValidation import DTT0DBValidation
17 from CalibMuon.DTCalibration.Workflow.DTAnalysisResiduals import DTAnalysisResiduals
18 from CalibMuon.DTCalibration.Workflow.CrabWatch import CrabWatch
19 from CalibMuon.DTCalibration.Workflow.tools import listFilesInCastor,haddInCastor,listFilesLocal,haddLocal,copyFilesFromCastor,copyFilesLocal,parseInput,getDatasetStr
20 import sys,os,time,optparse
21 
22 class DTCalibrationWorker:
23  def __init__(self,run,config):
24  self.config = config
25  self.refRun = run
26 
27  def run(self,type,mode,execute):
28  config = None
29  if self.config: config = self.config
30  refRun = 1
31  if self.refRun: refRun = self.refRun
32 
33  if type == 'ttrig': self.runTtrigWorkflow(mode,refRun,config,execute)
34  elif type == 'vdrift': self.runVDriftWorkflow(mode,refRun,config,execute)
35  elif type == 'noise': self.runNoiseWorkflow(mode,refRun,config,execute)
36  elif type == 't0': self.runT0Workflow(mode,refRun,config,execute)
37  elif type == 'validation': self.runValidationWorkflow(mode,refRun,config,execute)
38  elif type == 'analysis': self.runAnalysisWorkflow(mode,refRun,config,execute)
39  elif type == 'dbvalidation':
40  inputFiles = []
41  if config.dbFiles: inputFiles = config.dbFiles
42  self.runDBValidationWorkflow(mode,refRun,inputFiles,config,execute)
43 
44  return 0
45 
46  def dqmOutputDir(self,type,dirLabel,config):
47  dqm_output_dir = ''
48  if config.stageOutLocal:
49  from crab_util import findLastWorkDir
50  cwd = os.getcwd()
51  crab_task_dir = config.base_dir + '/' + dirLabel
52  os.chdir(crab_task_dir)
53  crabdir = findLastWorkDir('crab_0_')
54  if not crabdir: raise RuntimeError,'Could not find CRAB dir in %s' % crab_task_dir
55  os.chdir(cwd)
56  dqm_output_dir = crabdir + "/res"
57  elif config.stageOutCAF:
58  datasetstr = getDatasetStr(config.datasetpath)
59  dqm_output_dir = config.castorpath + '/DTCalibration/' + datasetstr + '/Run' + str(run) + '/' + type + '/' + dirLabel + '/' + 'v' + str(trial)
60 
61  return dqm_output_dir
62 
63  def runTtrigProd(self,run,runselection,trial,config,runStep=True):
64 
65  print "Processing tTrig production"
66  #config.runselection = runselection
67  datasetstr = getDatasetStr(config.datasetpath)
68  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + 'TimeBoxes' + '/' + 'v' + str(trial)
69 
70  task_dir = config.base_dir + '/TimeBoxes'
71  dtTtrigProd = DTTTrigProd(run,task_dir,config)
72  dtTtrigProd.writeCfg()
73 
74  if runStep:
75  project_prod = dtTtrigProd.run()
76 
77  print "Sent calibration jobs with project",project_prod
78  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
79 
80  crabProd = CrabWatch(project_prod)
81  crabProd.setThreshold(config.jobsFinishedThreshold)
82  crabProd.start()
83  crabProd.join()
84 
85  result_file = config.result_dir + '/DTTimeBoxes_%s.root'%run
86  if config.stageOutLocal:
87  output_dir = project_prod + "/res"
88  haddLocal(output_dir,result_file)
89  elif config.stageOutCAF:
90  castor_dir = config.castorpath + "/" + config.userdircaf
91  haddInCastor(castor_dir,result_file,'DTTimeBoxes','rfio://castorcms/','?svcClass=cmscafuser')
92 
93  return project_prod
94 
95  return None
96 
97  def runTtrigWriter(self,run,config,runStep=True):
98 
99  print "Processing tTrig correction"
100  dtTtrigWriter = DTTTrigTimeBoxesWriter(run,config.run_dir,config.result_dir,config)
101  dtTtrigWriter.writeCfg()
102 
103  if runStep:
104  dtTtrigWriter.run()
105 
106  print "Finished processing:"
107  for pset in dtTtrigWriter.configs: print "--->",pset
108 
109  def runResidualCalib(self,run,runselection,trial,input_db,label,result_file,config,runStep=True):
110 
111  print "Processing tTrig calibration"
112  #config.runselection = runselection
113  datasetstr = getDatasetStr(config.datasetpath)
114  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + label + '/' + 'v' + str(trial)
115 
116  task_dir = config.base_dir + '/' + label
117  dtResidualCalib = DTResidualCalibration(run=run,
118  dir=task_dir,
119  input_db=input_db,
120  config=config)
121  dtResidualCalib.writeCfg()
122 
123  if runStep:
124  project_residualCalib = dtResidualCalib.run()
125 
126  print "Sent calibration jobs with project",project_residualCalib
127  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
128 
129  crabResidualCalib = CrabWatch(project_residualCalib)
130  crabResidualCalib.setThreshold(config.jobsFinishedThreshold)
131  crabResidualCalib.start()
132  crabResidualCalib.join()
133 
134  if config.stageOutLocal:
135  output_dir = project_residualCalib + "/res"
136  haddLocal(output_dir,result_file,'residuals')
137  elif config.stageOutCAF:
138  castor_dir = config.castorpath + "/" + config.userdircaf
139  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
140 
141  return project_residualCalib
142 
143  return None
144 
145  def runAnalysisResiduals(self,run,runselection,trial,label,result_file,config,runStep=True):
146 
147  print "Processing residuals analysis"
148  datasetstr = getDatasetStr(config.datasetpath)
149  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/AnalysisResiduals/' + label + '/' + 'v' + str(trial)
150 
151  task_dir = config.base_dir + '/' + label
152  dtAnalysisResiduals = DTAnalysisResiduals(run=run,
153  dir=task_dir,
154  config=config)
155  dtAnalysisResiduals.writeCfg()
156 
157  if runStep:
158  project_analysisResiduals = dtAnalysisResiduals.run()
159 
160  print "Sent jobs with project",project_analysisResiduals
161  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
162 
163  crabAnalysisResiduals = CrabWatch(project_analysisResiduals)
164  crabAnalysisResiduals.setThreshold(config.jobsFinishedThreshold)
165  crabAnalysisResiduals.start()
166  crabAnalysisResiduals.join()
167 
168  if config.stageOutLocal:
169  output_dir = project_analysisResiduals + "/res"
170  haddLocal(output_dir,result_file,'residuals')
171  elif config.stageOutCAF:
172  castor_dir = config.castorpath + "/" + config.userdircaf
173  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
174 
175  return project_analysisResiduals
176 
177  return None
178 
179  def runTtrigResidualCorr(self,run,input_db,root_file,config,runStep=True):
180 
181  print "Processing tTrig residual correction"
182  dtTtrigResidualCorr = DTTTrigResidualCorr(run=run,
183  dir=config.run_dir,
184  input_db=input_db,
185  residuals=root_file,
186  result_dir=config.result_dir,
187  config=config)
188  dtTtrigResidualCorr.writeCfg()
189 
190  if runStep:
191  dtTtrigResidualCorr.run()
192 
193  print "Finished processing:"
194  for pset in dtTtrigResidualCorr.configs: print "--->",pset
195 
196  def runTtrigValid(self,run,runselection,trial,input_db,label,config,runStep=True):
197 
198  print "Processing tTrig validation"
199  #config.runselection = runselection
200  datasetstr = getDatasetStr(config.datasetpath)
201  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigValidation/' + label + '/' + 'v' + str(trial)
202 
203  task_dir = config.base_dir + '/' + label
204  dtTtrigValid = DTTTrigValid(run=run,
205  dir=task_dir,
206  input_db=input_db,
207  config=config)
208  dtTtrigValid.writeCfg()
209 
210  if runStep:
211  project_valid = dtTtrigValid.run()
212 
213  print "Sent validation jobs with project",project_valid
214  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
215 
216  crabValid = CrabWatch(project_valid)
217  crabValid.setThreshold(config.jobsFinishedThreshold)
218  crabValid.start()
219  crabValid.join()
220 
221  """
222  if config.stageOutLocal:
223  output_dir = project_valid + "/res"
224  haddLocal(output_dir,result_file,'residuals')
225  elif config.stageOutCAF:
226  castor_dir = config.castorpath + "/" + config.userdircaf
227  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
228  """
229 
230  return project_valid
231 
232  return None
233 
234  def runTtrigValidSummary(self,run,input_file,output_file,config,runStep=True):
235 
236  print "Processing Validation Summary"
237  dtTtrigValidSummary = DTValidSummary(run,config.run_dir,input_file,output_file,config)
238  dtTtrigValidSummary.writeCfg()
239 
240  if runStep:
241  dtTtrigValidSummary.run()
242 
243  print "...Validation Summary finished"
244 
245  def runVDriftSegmentCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
246 
247  print "Processing vDrift calibration"
248  #config.runselection = runselection
249  datasetstr = getDatasetStr(config.datasetpath)
250  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
251 
252  task_dir = config.base_dir + '/' + label
253  dtVDriftSegment = DTVDriftSegmentCalibration(run=run,
254  dir=task_dir,
255  config=config)
256  dtVDriftSegment.writeCfg()
257 
258  if runStep:
259  project_segment = dtVDriftSegment.run()
260 
261  print "Sent validation jobs with project",project_segment
262  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
263 
264  crabVDriftSegment = CrabWatch(project_segment)
265  crabVDriftSegment.setThreshold(config.jobsFinishedThreshold)
266  crabVDriftSegment.start()
267  crabVDriftSegment.join()
268 
269  if config.stageOutLocal:
270  output_dir = project_segment + "/res"
271  haddLocal(output_dir,result_file,'DTVDriftHistos')
272  elif config.stageOutCAF:
273  castor_dir = config.castorpath + "/" + config.userdircaf
274  haddInCastor(castor_dir,result_file,'DTVDriftHistos','rfio://castorcms/','?svcClass=cmscafuser')
275 
276  return project_segment
277 
278  return None
279 
280  def runVDriftSegmentWriter(self,run,root_file,config,runStep=True):
281 
282  print "Processing vDrift writer"
283  dtVDriftSegmentWriter = DTVDriftSegmentWriter(run=run,
284  dir=config.run_dir,
285  input_file=root_file,
286  output_dir=config.result_dir,
287  config=config)
288  dtVDriftSegmentWriter.writeCfg()
289 
290  if runStep:
291  dtVDriftSegmentWriter.run()
292 
293  print "Finished processing:"
294  for pset in dtVDriftSegmentWriter.configs: print "--->",pset
295 
296  def runVDriftMeanTimerCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
297 
298  print "Processing vDrift calibration"
299  #config.runselection = runselection
300  datasetstr = getDatasetStr(config.datasetpath)
301  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
302 
303  task_dir = config.base_dir + '/' + label
304  dtVDriftMeanTimer = DTVDriftMeanTimerCalibration(run=run,
305  dir=task_dir,
306  config=config)
307  dtVDriftMeanTimer.writeCfg()
308 
309  if runStep:
310  project_meantimer = dtVDriftMeanTimer.run()
311 
312  print "Sent validation jobs with project",project_meantimer
313  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
314 
315  crabVDriftMeanTimer = CrabWatch(project_meantimer)
316  crabVDriftMeanTimer.setThreshold(config.jobsFinishedThreshold)
317  crabVDriftMeanTimer.start()
318  crabVDriftMeanTimer.join()
319 
320  if config.stageOutLocal:
321  output_dir = project_meantimer + "/res"
322  haddLocal(output_dir,result_file,'DTTMaxHistos')
323  elif config.stageOutCAF:
324  castor_dir = config.castorpath + "/" + config.userdircaf
325  haddInCastor(castor_dir,result_file,'DTTMaxHistos','rfio://castorcms/','?svcClass=cmscafuser')
326 
327  return project_meantimer
328 
329  return None
330 
331  def runVDriftMeanTimerWriter(self,run,root_file,config,runStep=True):
332 
333  print "Processing vDrift writer"
334  dtVDriftMeanTimerWriter = DTVDriftMeanTimerWriter(run=run,
335  dir=config.run_dir,
336  input_file=root_file,
337  output_dir=config.result_dir,
338  config=config)
339  dtVDriftMeanTimerWriter.writeCfg()
340 
341  if runStep:
342  dtVDriftMeanTimerWriter.run()
343 
344  print "Finished processing:"
345  for pset in dtVDriftMeanTimerWriter.configs: print "--->",pset
346 
347  def runDQMClient(self,run,output_dir,config,runStep=True):
348 
349  print "Processing DQM Merge"
350 
351  if runStep:
352  dqm_files = []
353  if config.stageOutLocal:
354  dqm_files = listFilesLocal(output_dir,'DQM')
355  dqm_files = ['file:%s' % item for item in dqm_files]
356  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
357  dtDqmFinal.writeCfg()
358  dtDqmFinal.run()
359  elif config.stageOutCAF:
360  dqm_files = listFilesInCastor(output_dir,'DQM','')
361  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
362  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
363  dtDqmFinal.writeCfg()
364  dtDqmFinal.run()
365 
366  print "...DQM Merge finished"
367  else:
368  dqm_files = []
369  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
370  dtDqmFinal.writeCfg()
371 
372  def runDQMHarvesting(self,run,output_dir,config,runStep=True):
373 
374  print "Processing DQM harvesting"
375 
376  if runStep:
377  dqm_files = []
378  if config.stageOutLocal:
379  dqm_files = listFilesLocal(output_dir,'DQM')
380  dqm_files = ['file:%s' % item for item in dqm_files]
381  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
382  dtDqmFinal.writeCfg()
383  dtDqmFinal.run()
384  elif config.stageOutCAF:
385  dqm_files = listFilesInCastor(output_dir,'DQM','')
386  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
387  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
388  dtDqmFinal.writeCfg()
389  dtDqmFinal.run()
390 
391  print "...DQM harvesting finished"
392  else:
393  dqm_files = []
394  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
395  dtDqmFinal.writeCfg()
396 
397  def runDQMMerge(self,run,output_dir,config,runStep=True):
398 
399  print "Processing DQM merge"
400 
401  if runStep:
402  dqm_files = []
403  if config.stageOutLocal:
404  dqm_files = listFilesLocal(output_dir,'DQM')
405  dqm_files = ['file:%s' % item for item in dqm_files]
406  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
407  dtDQMMerge.writeCfg()
408  dtDQMMerge.run()
409  elif config.stageOutCAF:
410  dqm_files = listFilesInCastor(output_dir,'DQM','')
411  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
412  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
413  dtDQMMerge.writeCfg()
414  dtDQMMerge.run()
415 
416  print "...DQM merge finished"
417  else:
418  dqm_files = []
419  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
420  dtDQMMerge.writeCfg()
421 
422  ############################################################
423  # tTrig workflow
424  ############################################################
425  def runTtrigWorkflow(self,mode,run,config,execute=True):
426  trial = config.trial
427  runselection = config.runselection
428  ttrig_input_db = None
429  if hasattr(config,'inputTTrigDB') and config.inputTTrigDB: ttrig_input_db = os.path.abspath(config.inputTTrigDB)
430  result_dir = config.result_dir
431  if mode == 'timeboxes':
432  timeBoxes = os.path.abspath(result_dir + '/' + 'DTTimeBoxes_' + run + '.root')
433  ttrig_timeboxes_db = os.path.abspath(result_dir + '/' + 'ttrig_timeboxes_' + run + '.db')
434  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
435  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
436  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
437  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
438 
439  if not execute:
440  print "Writing configuration files.."
441  self.runTtrigProd(run,runselection,trial,config,False)
442  self.runTtrigWriter(run,config,False)
443  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config,False)
444  self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config,False)
445  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
446  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
447  self.runDQMClient(run,'',config,False)
448 
449  sys.exit(0)
450 
451  # Produce time-boxes
452  if not os.path.exists(timeBoxes): self.runTtrigProd(run,runselection,trial,config)
453  if not os.path.exists(timeBoxes): raise RuntimeError,'Could not produce %s' % timeBoxes
454 
455  # Write tTrig DB
456  if not os.path.exists(ttrig_timeboxes_db): self.runTtrigWriter(run,config)
457  if not os.path.exists(ttrig_timeboxes_db): raise RuntimeError,'Could not produce %s' % ttrig_timeboxes_db
458 
459  # Produce residuals
460  if not os.path.exists(residualsFirst):
461  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config)
462  if not os.path.exists(residualsFirst): raise RuntimeError,'Could not produce %s' % residualsFirst
463 
464  # Correction from residuals and write tTrig DB
465  if not os.path.exists(ttrig_residuals_db): self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config)
466  if not os.path.exists(ttrig_residuals_db): raise RuntimeError,'Could not produce %s' % ttrig_residuals_db
467 
468  # Validation
469  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
470 
471  """
472  # Summary of validation
473  if not os.path.exists(summaryResiduals): self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
474  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
475  """
476  # Produce DQM output
477  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
478  self.runDQMClient(run,dqm_output_dir,config)
479 
480  elif mode == 'residuals':
481  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
482  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
483  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
484  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
485 
486  if not execute:
487  print "Writing configuration files.."
488  if ttrig_input_db:
489  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config,False)
490  self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config,False)
491  else:
492  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config,False)
493  self.runTtrigResidualCorr(run,None,residualsFirst,config,False)
494 
495  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
496  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
497  self.runDQMClient(run,'',config,False)
498 
499  sys.exit(0)
500 
501  # Produce residuals
502  if not os.path.exists(residualsFirst):
503  if ttrig_input_db:
504  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config)
505  else:
506  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config)
507  if not os.path.exists(residualsFirst): raise RuntimeError,'Could not produce %s' % residualsFirst
508 
509  # Correction from residuals and write tTrig DB
510  if not os.path.exists(ttrig_residuals_db):
511  if ttrig_input_db: self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config)
512  else: self.runTtrigResidualCorr(run,None,residualsFirst,config)
513  if not os.path.exists(ttrig_residuals_db): raise RuntimeError,'Could not produce %s' % ttrig_residuals_db
514 
515  # Validation
516  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
517 
518  """
519  # Summary of validation
520  if not os.path.exists(summaryResiduals):
521  self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
522  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
523  """
524  # Produce DQM output
525  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
526  self.runDQMClient(run,dqm_output_dir,config)
527 
528  elif mode == 'validation':
529  residualsValid = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
530  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
531 
532  if not execute:
533  print "Writing configuration files.."
534  if ttrig_input_db:
535  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config,False)
536  else:
537  self.runTtrigValid(run,runselection,trial,None,'Residuals',config,False)
538 
539  #self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config,False)
540  self.runDQMClient(run,'',config,False)
541 
542  sys.exit(0)
543 
544  # Validation
545  if ttrig_input_db:
546  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config)
547  else:
548  self.runTtrigValid(run,runselection,trial,None,'Residuals',config)
549 
550  """
551  # Summary of validation
552  if not os.path.exists(summaryResiduals):
553  self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config)
554  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
555  """
556 
557  # Produce DQM output
558  dqm_output_dir = self.dqmOutputDir('TTrigValidation','Residuals',config)
559  self.runDQMClient(run,dqm_output_dir,config)
560 
561  return 0
562 
563  ############################################################
564  # vDrift workflow
565  ############################################################
566  def runVDriftWorkflow(self,mode,run,config,execute=True):
567  trial = config.trial
568  runselection = config.runselection
569  result_dir = config.result_dir
570  if mode == 'segment':
571  vDriftHistos = os.path.abspath(result_dir + '/' + 'DTVDriftHistos_' + run + '.root')
572  vDrift_segment_db = os.path.abspath(result_dir + '/' + 'vDrift_segment_' + run + '.db')
573 
574  if not execute:
575  print "Writing configuration files.."
576  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config,False)
577  self.runVDriftSegmentWriter(run,vDriftHistos,config,False)
578 
579  sys.exit(0)
580 
581  # Produce vDrift histos
582  if not os.path.exists(vDriftHistos):
583  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config)
584  if not os.path.exists(vDriftHistos): raise RuntimeError,'Could not produce %s' % vDriftHistos
585 
586  # Write vDrift DB
587  if not os.path.exists(vDrift_segment_db): self.runVDriftSegmentWriter(run,vDriftHistos,config)
588  if not os.path.exists(vDrift_segment_db): raise RuntimeError,'Could not produce %s' % vDrift_segment_db
589 
590  elif mode == 'meantimer':
591  vDriftTMaxHistos = os.path.abspath(result_dir + '/' + 'DTTMaxHistos_' + run + '.root')
592  vDrift_meantimer_db = os.path.abspath(result_dir + '/' + 'vDrift_meantimer_' + run + '.db')
593 
594  if not execute:
595  print "Writing configuration files.."
596  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config,False)
597  self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config,False)
598 
599  sys.exit(0)
600 
601  # Produce t_max histos
602  if not os.path.exists(vDriftTMaxHistos):
603  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config)
604  if not os.path.exists(vDriftTMaxHistos): raise RuntimeError,'Could not produce %s' % vDriftTMaxHistos
605 
606  # Write vDrift DB
607  if not os.path.exists(vDrift_meantimer_db): self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config)
608  if not os.path.exists(vDrift_meantimer_db): raise RuntimeError,'Could not produce %s' % vDrift_meantimer_db
609 
610  return 0
611 
612  ############################################################
613  # noise workflow
614  ############################################################
615  def runNoiseWorkflow(self,mode,run,config,execute=True):
616  print "Processing noise calibration"
617 
618  trial = config.trial
619  runselection = config.runselection
620  result_dir = config.result_dir
621  result_file = os.path.abspath(result_dir + '/' + 'dtNoiseCalib_' + run + '.root')
622  noise_db = os.path.abspath(result_dir + '/' + 'noise_' + run + '.db')
623  noise_txt = os.path.abspath(result_dir + '/' + 'noise_' + run + '.txt')
624 
625  datasetstr = getDatasetStr(config.datasetpath)
626  #config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/NoiseCalibration/' + label + '/' + 'v' + str(trial)
627  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/NoiseCalibration/' + 'v' + str(trial)
628 
629  task_dir = config.base_dir + '/NoiseCalib'
630  dtNoiseCalibration = DTNoiseCalibration(run=run,
631  dir=task_dir,
632  config=config)
633  if not execute:
634  dtNoiseCalibration.writeCfg()
635  sys.exit(0)
636  else:
637  dtNoiseCalibration.writeCfg()
638  project_noise = dtNoiseCalibration.run()
639 
640  print "Sent calibration jobs with project",project_noise
641  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
642 
643  crabNoiseCalibration = CrabWatch(project_noise)
644  crabNoiseCalibration.setThreshold(config.jobsFinishedThreshold)
645  crabNoiseCalibration.start()
646  crabNoiseCalibration.join()
647 
648  if config.stageOutLocal:
649  crab_output_dir = project_noise + "/res"
650  retcode = copyFilesLocal(crab_output_dir,result_dir,'dtNoiseCalib')
651  retcode = copyFilesLocal(crab_output_dir,result_dir,'noise')
652  elif config.stageOutCAF:
653  castor_dir = config.castorpath + "/" + config.userdircaf
654  retcode = copyFilesFromCastor(castor_dir,result_dir,'dtNoiseCalib')
655  retcode = copyFilesFromCastor(castor_dir,result_dir,'noise')
656 
657  return 0
658 
659  ############################################################
660  # t0 workflow
661  ############################################################
662  def runT0Workflow(self,mode,run,config,execute=True):
663 
664  return 0
665 
666  ############################################################
667  # Validation workflow
668  ############################################################
669  def runValidationWorkflow(self,mode,run,config,execute=True):
670  print "Processing DQM validation"
671  trial = config.trial
672  runselection = config.runselection
673  result_dir = config.result_dir
674  datasetstr = getDatasetStr(config.datasetpath)
675  dirLabel = 'DQM'
676  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/DQMValidation/' + dirLabel + '/' + 'v' + str(trial)
677 
678  task_dir = config.base_dir + '/' + dirLabel
679  dtDQMValid = DTDQMValidation(run=run,
680  dir=task_dir,
681  config=config)
682  if not execute:
683  dtDQMValid.writeCfg()
684  self.runDQMMerge(run,'',config,False)
685  self.runDQMHarvesting(run,'',config,False)
686 
687  sys.exit(0)
688  else:
689  dtDQMValid.writeCfg()
690  project_valid = dtDQMValid.run()
691 
692  print "Sent validation jobs with project",project_valid
693  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
694 
695  crabValid = CrabWatch(project_valid)
696  crabValid.setThreshold(config.jobsFinishedThreshold)
697  crabValid.start()
698  crabValid.join()
699 
700  # Produce DQM output
701  dqm_output_dir = self.dqmOutputDir('DQMValidation',dirLabel,config)
702  self.runDQMMerge(run,dqm_output_dir,config)
703  # Run harvesting from merged DQM file
704  dqm_merge_dir = os.path.abspath(result_dir)
705  self.runDQMHarvesting(run,dqm_merge_dir,config)
706 
707  return 0
708 
709  ############################################################
710  # Analysis workflow
711  ############################################################
712  def runAnalysisWorkflow(self,mode,run,config,execute=True):
713  print "Processing analysis workflow"
714  trial = config.trial
715  runselection = config.runselection
716  result_dir = config.result_dir
717  if mode == 'residuals':
718  residualsFile = os.path.abspath(result_dir + '/' + 'DTResiduals_' + run + '.root')
719 
720  if not execute:
721  print "Writing configuration files.."
722  self.runAnalysisResiduals(run,runselection,trial,'Residuals',residualsFile,config,False)
723 
724  sys.exit(0)
725 
726  # Produce residuals
727  if not os.path.exists(residualsFile):
728  self.runAnalysisResiduals(run,runselection,trial,'Residuals',residualsFile,config)
729  if not os.path.exists(residualsFile): raise RuntimeError,'Could not produce %s' % residualsFile
730 
731  return 0
732 
733  ############################################################
734  # DB Validation workflow
735  ############################################################
736  def runDBValidationWorkflow(self,mode,run,inputFiles,config,execute=True):
737  print "Processing DB validation"
738 
739  dtDBValidation = None
740  if mode == 't0DB':
741  dtDBValidation = DTT0DBValidation(run=run,
742  dir=config.run_dir,
743  input_files=inputFiles,
744  output_dir=config.result_dir,
745  config=config)
746  #dtDBValidation.writeCfg()
747 
748  if execute:
749  dtDBValidation.run()
750 
751  print "Finished processing:"
752  for pset in dtDBValidation.configs: print "--->",pset
753 
754  return 0
DTCalibrationWorker.DTCalibrationWorker.runDBValidationWorkflow
def runDBValidationWorkflow
DB Validation workflow.
Definition: DTCalibrationWorker.py:736
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.getDatasetStr
def getDatasetStr
Definition: tools.py:15
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
DTDqm
Definition: DTDqm.py:1
tools.haddLocal
def haddLocal
Definition: tools.py:98
tools.copyFilesFromCastor
def copyFilesFromCastor
Definition: tools.py:55
DTCalibrationWorker.DTCalibrationWorker.runValidationWorkflow
def runValidationWorkflow
Validation workflow.
Definition: DTCalibrationWorker.py:669
tools.haddInCastor
def haddInCastor
Definition: tools.py:81
tools.copyFilesLocal
def copyFilesLocal
Definition: tools.py:68
Generated for CMSSW Reference Manual by doxygen 1.8.5