CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
DTCalibrationWorker.py
Go to the documentation of this file.
1 from CalibMuon.DTCalibration.Workflow.DTTTrigProd import DTTTrigProd
2 from CalibMuon.DTCalibration.Workflow.DTTTrigTimeBoxesWriter import DTTTrigTimeBoxesWriter
3 from CalibMuon.DTCalibration.Workflow.DTResidualCalibration import DTResidualCalibration
4 from CalibMuon.DTCalibration.Workflow.DTTTrigResidualCorr import DTTTrigResidualCorr
5 from CalibMuon.DTCalibration.Workflow.DTTTrigValid import DTTTrigValid
6 from CalibMuon.DTCalibration.Workflow.DTValidSummary import DTValidSummary
7 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentCalibration import DTVDriftSegmentCalibration
8 from CalibMuon.DTCalibration.Workflow.DTVDriftSegmentWriter import DTVDriftSegmentWriter
9 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerCalibration import DTVDriftMeanTimerCalibration
10 from CalibMuon.DTCalibration.Workflow.DTVDriftMeanTimerWriter import DTVDriftMeanTimerWriter
11 from CalibMuon.DTCalibration.Workflow.DTNoiseCalibration import DTNoiseCalibration
12 from CalibMuon.DTCalibration.Workflow.DTDQMValidation import DTDQMValidation
13 from CalibMuon.DTCalibration.Workflow.DTDQMMerge import DTDQMMerge
14 from CalibMuon.DTCalibration.Workflow.DTDQMHarvesting import DTDQMHarvesting
15 from CalibMuon.DTCalibration.Workflow.DTDqm import DTDqm
16 from CalibMuon.DTCalibration.Workflow.DTT0DBValidation import DTT0DBValidation
17 from CalibMuon.DTCalibration.Workflow.CrabWatch import CrabWatch
18 from CalibMuon.DTCalibration.Workflow.tools import listFilesInCastor,haddInCastor,listFilesLocal,haddLocal,copyFilesFromCastor,copyFilesLocal,parseInput,getDatasetStr
19 import sys,os,time,optparse
20 
21 class DTCalibrationWorker:
22  def __init__(self,run,config):
23  self.config = config
24  self.refRun = run
25 
26  def run(self,type,mode,execute):
27  config = None
28  if self.config: config = self.config
29  refRun = 1
30  if self.refRun: refRun = self.refRun
31 
32  if type == 'ttrig': self.runTtrigWorkflow(mode,refRun,config,execute)
33  elif type == 'vdrift': self.runVDriftWorkflow(mode,refRun,config,execute)
34  elif type == 'noise': self.runNoiseWorkflow(mode,refRun,config,execute)
35  elif type == 't0': self.runT0Workflow(mode,refRun,config,execute)
36  elif type == 'validation': self.runValidationWorkflow(mode,refRun,config,execute)
37  elif type == 'dbvalidation':
38  inputFiles = []
39  if config.dbFiles: inputFiles = config.dbFiles
40  self.runDBValidationWorkflow(mode,refRun,inputFiles,config,execute)
41 
42  return 0
43 
44  def dqmOutputDir(self,type,dirLabel,config):
45  dqm_output_dir = ''
46  if config.stageOutLocal:
47  from crab_util import findLastWorkDir
48  cwd = os.getcwd()
49  crab_task_dir = config.base_dir + '/' + dirLabel
50  os.chdir(crab_task_dir)
51  crabdir = findLastWorkDir('crab_0_')
52  if not crabdir: raise RuntimeError,'Could not find CRAB dir in %s' % crab_task_dir
53  os.chdir(cwd)
54  dqm_output_dir = crabdir + "/res"
55  elif config.stageOutCAF:
56  datasetstr = getDatasetStr(config.datasetpath)
57  dqm_output_dir = config.castorpath + '/DTCalibration/' + datasetstr + '/Run' + str(run) + '/' + type + '/' + dirLabel + '/' + 'v' + str(trial)
58 
59  return dqm_output_dir
60 
61  def runTtrigProd(self,run,runselection,trial,config,runStep=True):
62 
63  print "Processing tTrig production"
64  #config.runselection = runselection
65  datasetstr = getDatasetStr(config.datasetpath)
66  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + 'TimeBoxes' + '/' + 'v' + str(trial)
67 
68  task_dir = config.base_dir + '/TimeBoxes'
69  dtTtrigProd = DTTTrigProd(run,task_dir,config)
70  dtTtrigProd.writeCfg()
71 
72  if runStep:
73  project_prod = dtTtrigProd.run()
74 
75  print "Sent calibration jobs with project",project_prod
76  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
77 
78  crabProd = CrabWatch(project_prod)
79  crabProd.setThreshold(config.jobsFinishedThreshold)
80  crabProd.start()
81  crabProd.join()
82 
83  result_file = config.result_dir + '/DTTimeBoxes_%s.root'%run
84  if config.stageOutLocal:
85  output_dir = project_prod + "/res"
86  haddLocal(output_dir,result_file)
87  elif config.stageOutCAF:
88  castor_dir = config.castorpath + "/" + config.userdircaf
89  haddInCastor(castor_dir,result_file,'DTTimeBoxes','rfio://castorcms/','?svcClass=cmscafuser')
90 
91  return project_prod
92 
93  return None
94 
95  def runTtrigWriter(self,run,config,runStep=True):
96 
97  print "Processing tTrig correction"
98  dtTtrigWriter = DTTTrigTimeBoxesWriter(run,config.run_dir,config.result_dir,config)
99  dtTtrigWriter.writeCfg()
100 
101  if runStep:
102  dtTtrigWriter.run()
103 
104  print "Finished processing:"
105  for pset in dtTtrigWriter.configs: print "--->",pset
106 
107  def runResidualCalib(self,run,runselection,trial,input_db,label,result_file,config,runStep=True):
108 
109  print "Processing tTrig calibration"
110  #config.runselection = runselection
111  datasetstr = getDatasetStr(config.datasetpath)
112  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigCalibration/' + label + '/' + 'v' + str(trial)
113 
114  task_dir = config.base_dir + '/' + label
115  dtResidualCalib = DTResidualCalibration(run=run,
116  dir=task_dir,
117  input_db=input_db,
118  config=config)
119  dtResidualCalib.writeCfg()
120 
121  if runStep:
122  project_residualCalib = dtResidualCalib.run()
123 
124  print "Sent calibration jobs with project",project_residualCalib
125  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
126 
127  crabResidualCalib = CrabWatch(project_residualCalib)
128  crabResidualCalib.setThreshold(config.jobsFinishedThreshold)
129  crabResidualCalib.start()
130  crabResidualCalib.join()
131 
132  if config.stageOutLocal:
133  output_dir = project_residualCalib + "/res"
134  haddLocal(output_dir,result_file,'residuals')
135  elif config.stageOutCAF:
136  castor_dir = config.castorpath + "/" + config.userdircaf
137  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
138 
139  return project_residualCalib
140 
141  return None
142 
143  def runTtrigResidualCorr(self,run,input_db,root_file,config,runStep=True):
144 
145  print "Processing tTrig residual correction"
146  dtTtrigResidualCorr = DTTTrigResidualCorr(run=run,
147  dir=config.run_dir,
148  input_db=input_db,
149  residuals=root_file,
150  result_dir=config.result_dir,
151  config=config)
152  dtTtrigResidualCorr.writeCfg()
153 
154  if runStep:
155  dtTtrigResidualCorr.run()
156 
157  print "Finished processing:"
158  for pset in dtTtrigResidualCorr.configs: print "--->",pset
159 
160  def runTtrigValid(self,run,runselection,trial,input_db,label,config,runStep=True):
161 
162  print "Processing tTrig validation"
163  #config.runselection = runselection
164  datasetstr = getDatasetStr(config.datasetpath)
165  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/TTrigValidation/' + label + '/' + 'v' + str(trial)
166 
167  task_dir = config.base_dir + '/' + label
168  dtTtrigValid = DTTTrigValid(run=run,
169  dir=task_dir,
170  input_db=input_db,
171  config=config)
172  dtTtrigValid.writeCfg()
173 
174  if runStep:
175  project_valid = dtTtrigValid.run()
176 
177  print "Sent validation jobs with project",project_valid
178  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
179 
180  crabValid = CrabWatch(project_valid)
181  crabValid.setThreshold(config.jobsFinishedThreshold)
182  crabValid.start()
183  crabValid.join()
184 
185  """
186  if config.stageOutLocal:
187  output_dir = project_valid + "/res"
188  haddLocal(output_dir,result_file,'residuals')
189  elif config.stageOutCAF:
190  castor_dir = config.castorpath + "/" + config.userdircaf
191  haddInCastor(castor_dir,result_file,'residuals','rfio://castorcms/','?svcClass=cmscafuser')
192  """
193 
194  return project_valid
195 
196  return None
197 
198  def runTtrigValidSummary(self,run,input_file,output_file,config,runStep=True):
199 
200  print "Processing Validation Summary"
201  dtTtrigValidSummary = DTValidSummary(run,config.run_dir,input_file,output_file,config)
202  dtTtrigValidSummary.writeCfg()
203 
204  if runStep:
205  dtTtrigValidSummary.run()
206 
207  print "...Validation Summary finished"
208 
209  def runVDriftSegmentCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
210 
211  print "Processing vDrift calibration"
212  #config.runselection = runselection
213  datasetstr = getDatasetStr(config.datasetpath)
214  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
215 
216  task_dir = config.base_dir + '/' + label
217  dtVDriftSegment = DTVDriftSegmentCalibration(run=run,
218  dir=task_dir,
219  config=config)
220  dtVDriftSegment.writeCfg()
221 
222  if runStep:
223  project_segment = dtVDriftSegment.run()
224 
225  print "Sent validation jobs with project",project_segment
226  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
227 
228  crabVDriftSegment = CrabWatch(project_segment)
229  crabVDriftSegment.setThreshold(config.jobsFinishedThreshold)
230  crabVDriftSegment.start()
231  crabVDriftSegment.join()
232 
233  if config.stageOutLocal:
234  output_dir = project_segment + "/res"
235  haddLocal(output_dir,result_file,'DTVDriftHistos')
236  elif config.stageOutCAF:
237  castor_dir = config.castorpath + "/" + config.userdircaf
238  haddInCastor(castor_dir,result_file,'DTVDriftHistos','rfio://castorcms/','?svcClass=cmscafuser')
239 
240  return project_segment
241 
242  return None
243 
244  def runVDriftSegmentWriter(self,run,root_file,config,runStep=True):
245 
246  print "Processing vDrift writer"
247  dtVDriftSegmentWriter = DTVDriftSegmentWriter(run=run,
248  dir=config.run_dir,
249  input_file=root_file,
250  output_dir=config.result_dir,
251  config=config)
252  dtVDriftSegmentWriter.writeCfg()
253 
254  if runStep:
255  dtVDriftSegmentWriter.run()
256 
257  print "Finished processing:"
258  for pset in dtVDriftSegmentWriter.configs: print "--->",pset
259 
260  def runVDriftMeanTimerCalib(self,run,runselection,trial,label,result_file,config,runStep=True):
261 
262  print "Processing vDrift calibration"
263  #config.runselection = runselection
264  datasetstr = getDatasetStr(config.datasetpath)
265  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/VDriftCalibration/' + label + '/' + 'v' + str(trial)
266 
267  task_dir = config.base_dir + '/' + label
268  dtVDriftMeanTimer = DTVDriftMeanTimerCalibration(run=run,
269  dir=task_dir,
270  config=config)
271  dtVDriftMeanTimer.writeCfg()
272 
273  if runStep:
274  project_meantimer = dtVDriftMeanTimer.run()
275 
276  print "Sent validation jobs with project",project_meantimer
277  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
278 
279  crabVDriftMeanTimer = CrabWatch(project_meantimer)
280  crabVDriftMeanTimer.setThreshold(config.jobsFinishedThreshold)
281  crabVDriftMeanTimer.start()
282  crabVDriftMeanTimer.join()
283 
284  if config.stageOutLocal:
285  output_dir = project_meantimer + "/res"
286  haddLocal(output_dir,result_file,'DTTMaxHistos')
287  elif config.stageOutCAF:
288  castor_dir = config.castorpath + "/" + config.userdircaf
289  haddInCastor(castor_dir,result_file,'DTTMaxHistos','rfio://castorcms/','?svcClass=cmscafuser')
290 
291  return project_meantimer
292 
293  return None
294 
295  def runVDriftMeanTimerWriter(self,run,root_file,config,runStep=True):
296 
297  print "Processing vDrift writer"
298  dtVDriftMeanTimerWriter = DTVDriftMeanTimerWriter(run=run,
299  dir=config.run_dir,
300  input_file=root_file,
301  output_dir=config.result_dir,
302  config=config)
303  dtVDriftMeanTimerWriter.writeCfg()
304 
305  if runStep:
306  dtVDriftMeanTimerWriter.run()
307 
308  print "Finished processing:"
309  for pset in dtVDriftMeanTimerWriter.configs: print "--->",pset
310 
311  def runDQMClient(self,run,output_dir,config,runStep=True):
312 
313  print "Processing DQM Merge"
314 
315  if runStep:
316  dqm_files = []
317  if config.stageOutLocal:
318  dqm_files = listFilesLocal(output_dir,'DQM')
319  dqm_files = ['file:%s' % item for item in dqm_files]
320  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
321  dtDqmFinal.writeCfg()
322  dtDqmFinal.run()
323  elif config.stageOutCAF:
324  dqm_files = listFilesInCastor(output_dir,'DQM','')
325  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
326  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
327  dtDqmFinal.writeCfg()
328  dtDqmFinal.run()
329 
330  print "...DQM Merge finished"
331  else:
332  dqm_files = []
333  dtDqmFinal = DTDqm(run,config.run_dir,dqm_files,config.result_dir,config)
334  dtDqmFinal.writeCfg()
335 
336  def runDQMHarvesting(self,run,output_dir,config,runStep=True):
337 
338  print "Processing DQM harvesting"
339 
340  if runStep:
341  dqm_files = []
342  if config.stageOutLocal:
343  dqm_files = listFilesLocal(output_dir,'DQM')
344  dqm_files = ['file:%s' % item for item in dqm_files]
345  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
346  dtDqmFinal.writeCfg()
347  dtDqmFinal.run()
348  elif config.stageOutCAF:
349  dqm_files = listFilesInCastor(output_dir,'DQM','')
350  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
351  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
352  dtDqmFinal.writeCfg()
353  dtDqmFinal.run()
354 
355  print "...DQM harvesting finished"
356  else:
357  dqm_files = []
358  dtDqmFinal = DTDQMHarvesting(run,config.run_dir,dqm_files,config.result_dir,config)
359  dtDqmFinal.writeCfg()
360 
361  def runDQMMerge(self,run,output_dir,config,runStep=True):
362 
363  print "Processing DQM merge"
364 
365  if runStep:
366  dqm_files = []
367  if config.stageOutLocal:
368  dqm_files = listFilesLocal(output_dir,'DQM')
369  dqm_files = ['file:%s' % item for item in dqm_files]
370  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
371  dtDQMMerge.writeCfg()
372  dtDQMMerge.run()
373  elif config.stageOutCAF:
374  dqm_files = listFilesInCastor(output_dir,'DQM','')
375  dqm_files = [file.replace('/castor/cern.ch/cms','') for file in dqm_files]
376  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
377  dtDQMMerge.writeCfg()
378  dtDQMMerge.run()
379 
380  print "...DQM merge finished"
381  else:
382  dqm_files = []
383  dtDQMMerge = DTDQMMerge(run,config.run_dir,dqm_files,config.result_dir,config)
384  dtDQMMerge.writeCfg()
385 
386  ############################################################
387  # tTrig workflow
388  ############################################################
389  def runTtrigWorkflow(self,mode,run,config,execute=True):
390  trial = config.trial
391  runselection = config.runselection
392  ttrig_input_db = None
393  if hasattr(config,'inputTTrigDB') and config.inputTTrigDB: ttrig_input_db = os.path.abspath(config.inputTTrigDB)
394  result_dir = config.result_dir
395  if mode == 'timeboxes':
396  timeBoxes = os.path.abspath(result_dir + '/' + 'DTTimeBoxes_' + run + '.root')
397  ttrig_timeboxes_db = os.path.abspath(result_dir + '/' + 'ttrig_timeboxes_' + run + '.db')
398  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
399  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
400  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
401  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
402 
403  if not execute:
404  print "Writing configuration files.."
405  self.runTtrigProd(run,runselection,trial,config,False)
406  self.runTtrigWriter(run,config,False)
407  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config,False)
408  self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config,False)
409  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
410  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
411  self.runDQMClient(run,'',config,False)
412 
413  sys.exit(0)
414 
415  # Produce time-boxes
416  if not os.path.exists(timeBoxes): self.runTtrigProd(run,runselection,trial,config)
417  if not os.path.exists(timeBoxes): raise RuntimeError,'Could not produce %s' % timeBoxes
418 
419  # Write tTrig DB
420  if not os.path.exists(ttrig_timeboxes_db): self.runTtrigWriter(run,config)
421  if not os.path.exists(ttrig_timeboxes_db): raise RuntimeError,'Could not produce %s' % ttrig_timeboxes_db
422 
423  # Produce residuals
424  if not os.path.exists(residualsFirst):
425  self.runResidualCalib(run,runselection,trial,ttrig_timeboxes_db,'Residuals',residualsFirst,config)
426  if not os.path.exists(residualsFirst): raise RuntimeError,'Could not produce %s' % residualsFirst
427 
428  # Correction from residuals and write tTrig DB
429  if not os.path.exists(ttrig_residuals_db): self.runTtrigResidualCorr(run,ttrig_timeboxes_db,residualsFirst,config)
430  if not os.path.exists(ttrig_residuals_db): raise RuntimeError,'Could not produce %s' % ttrig_residuals_db
431 
432  # Validation
433  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
434 
435  """
436  # Summary of validation
437  if not os.path.exists(summaryResiduals): self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
438  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
439  """
440  # Produce DQM output
441  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
442  self.runDQMClient(run,dqm_output_dir,config)
443 
444  elif mode == 'residuals':
445  residualsFirst = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
446  ttrig_residuals_db = os.path.abspath(result_dir + '/' + 'ttrig_residuals_' + run + '.db')
447  residualsResidCorr = os.path.abspath(result_dir + '/' + 'DTResidualValidation_ResidCorr_' + run + '.root')
448  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
449 
450  if not execute:
451  print "Writing configuration files.."
452  if ttrig_input_db:
453  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config,False)
454  self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config,False)
455  else:
456  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config,False)
457  self.runTtrigResidualCorr(run,None,residualsFirst,config,False)
458 
459  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config,False)
460  #self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config,False)
461  self.runDQMClient(run,'',config,False)
462 
463  sys.exit(0)
464 
465  # Produce residuals
466  if not os.path.exists(residualsFirst):
467  if ttrig_input_db:
468  self.runResidualCalib(run,runselection,trial,ttrig_input_db,'Residuals',residualsFirst,config)
469  else:
470  self.runResidualCalib(run,runselection,trial,None,'Residuals',residualsFirst,config)
471  if not os.path.exists(residualsFirst): raise RuntimeError,'Could not produce %s' % residualsFirst
472 
473  # Correction from residuals and write tTrig DB
474  if not os.path.exists(ttrig_residuals_db):
475  if ttrig_input_db: self.runTtrigResidualCorr(run,ttrig_input_db,residualsFirst,config)
476  else: self.runTtrigResidualCorr(run,None,residualsFirst,config)
477  if not os.path.exists(ttrig_residuals_db): raise RuntimeError,'Could not produce %s' % ttrig_residuals_db
478 
479  # Validation
480  self.runTtrigValid(run,runselection,trial,ttrig_residuals_db,'ResidualsResidCorr',config)
481 
482  """
483  # Summary of validation
484  if not os.path.exists(summaryResiduals):
485  self.runTtrigValidSummary(run,residualsResidCorr,summaryResiduals,config)
486  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
487  """
488  # Produce DQM output
489  dqm_output_dir = self.dqmOutputDir('TTrigValidation','ResidualsResidCorr',config)
490  self.runDQMClient(run,dqm_output_dir,config)
491 
492  elif mode == 'validation':
493  residualsValid = os.path.abspath(result_dir + '/' + 'DTResidualValidation_' + run + '.root')
494  summaryResiduals = os.path.abspath(result_dir + '/' + 'SummaryResiduals_' + run + '.root')
495 
496  if not execute:
497  print "Writing configuration files.."
498  if ttrig_input_db:
499  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config,False)
500  else:
501  self.runTtrigValid(run,runselection,trial,None,'Residuals',config,False)
502 
503  #self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config,False)
504  self.runDQMClient(run,'',config,False)
505 
506  sys.exit(0)
507 
508  # Validation
509  if ttrig_input_db:
510  self.runTtrigValid(run,runselection,trial,ttrig_input_db,'Residuals',config)
511  else:
512  self.runTtrigValid(run,runselection,trial,None,'Residuals',config)
513 
514  """
515  # Summary of validation
516  if not os.path.exists(summaryResiduals):
517  self.runTtrigValidSummary(run,residualsValid,summaryResiduals,config)
518  if not os.path.exists(summaryResiduals): raise RuntimeError,'Could not produce %s' % summaryResiduals
519  """
520 
521  # Produce DQM output
522  dqm_output_dir = self.dqmOutputDir('TTrigValidation','Residuals',config)
523  self.runDQMClient(run,dqm_output_dir,config)
524 
525  return 0
526 
527  ############################################################
528  # vDrift workflow
529  ############################################################
530  def runVDriftWorkflow(self,mode,run,config,execute=True):
531  trial = config.trial
532  runselection = config.runselection
533  result_dir = config.result_dir
534  if mode == 'segment':
535  vDriftHistos = os.path.abspath(result_dir + '/' + 'DTVDriftHistos_' + run + '.root')
536  vDrift_segment_db = os.path.abspath(result_dir + '/' + 'vDrift_segment_' + run + '.db')
537 
538  if not execute:
539  print "Writing configuration files.."
540  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config,False)
541  self.runVDriftSegmentWriter(run,vDriftHistos,config,False)
542 
543  sys.exit(0)
544 
545  # Produce vDrift histos
546  if not os.path.exists(vDriftHistos):
547  self.runVDriftSegmentCalib(run,runselection,trial,'VDriftHistos',vDriftHistos,config)
548  if not os.path.exists(vDriftHistos): raise RuntimeError,'Could not produce %s' % vDriftHistos
549 
550  # Write vDrift DB
551  if not os.path.exists(vDrift_segment_db): self.runVDriftSegmentWriter(run,vDriftHistos,config)
552  if not os.path.exists(vDrift_segment_db): raise RuntimeError,'Could not produce %s' % vDrift_segment_db
553 
554  elif mode == 'meantimer':
555  vDriftTMaxHistos = os.path.abspath(result_dir + '/' + 'DTTMaxHistos_' + run + '.root')
556  vDrift_meantimer_db = os.path.abspath(result_dir + '/' + 'vDrift_meantimer_' + run + '.db')
557 
558  if not execute:
559  print "Writing configuration files.."
560  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config,False)
561  self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config,False)
562 
563  sys.exit(0)
564 
565  # Produce t_max histos
566  if not os.path.exists(vDriftTMaxHistos):
567  self.runVDriftMeanTimerCalib(run,runselection,trial,'VDriftTMaxHistos',vDriftTMaxHistos,config)
568  if not os.path.exists(vDriftTMaxHistos): raise RuntimeError,'Could not produce %s' % vDriftTMaxHistos
569 
570  # Write vDrift DB
571  if not os.path.exists(vDrift_meantimer_db): self.runVDriftMeanTimerWriter(run,vDriftTMaxHistos,config)
572  if not os.path.exists(vDrift_meantimer_db): raise RuntimeError,'Could not produce %s' % vDrift_meantimer_db
573 
574  return 0
575 
576  ############################################################
577  # noise workflow
578  ############################################################
579  def runNoiseWorkflow(self,mode,run,config,execute=True):
580  print "Processing noise calibration"
581 
582  trial = config.trial
583  runselection = config.runselection
584  result_dir = config.result_dir
585  result_file = os.path.abspath(result_dir + '/' + 'dtNoiseCalib_' + run + '.root')
586  noise_db = os.path.abspath(result_dir + '/' + 'noise_' + run + '.db')
587  noise_txt = os.path.abspath(result_dir + '/' + 'noise_' + run + '.txt')
588 
589  datasetstr = getDatasetStr(config.datasetpath)
590  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/NoiseCalibration/' + label + '/' + 'v' + str(trial)
591 
592  task_dir = config.base_dir + '/NoiseCalib'
593  dtNoiseCalibration = DTNoiseCalibration(run=run,
594  dir=task_dir,
595  config=config)
596  if not execute:
597  dtNoiseCalibration.writeCfg()
598  sys.exit(0)
599  else:
600  dtNoiseCalibration.writeCfg()
601  project_noise = dtNoiseCalibration.run()
602 
603  print "Sent calibration jobs with project",project_noise
604  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
605 
606  crabNoiseCalibration = CrabWatch(project_noise)
607  crabNoiseCalibration.setThreshold(config.jobsFinishedThreshold)
608  crabNoiseCalibration.start()
609  crabNoiseCalibration.join()
610 
611  if config.stageOutLocal:
612  crab_output_dir = project_noise + "/res"
613  retcode = copyFilesLocal(crab_output_dir,result_dir,'dtNoiseCalib')
614  retcode = copyFilesLocal(crab_output_dir,result_dir,'noise')
615  elif config.stageOutCAF:
616  castor_dir = config.castorpath + "/" + config.userdircaf
617  retcode = copyFilesFromCastor(castor_dir,result_dir,'dtNoiseCalib')
618  retcode = copyFilesFromCastor(castor_dir,result_dir,'noise')
619 
620  return 0
621 
622  ############################################################
623  # t0 workflow
624  ############################################################
625  def runT0Workflow(self,mode,run,config,execute=True):
626 
627  return 0
628 
629  ############################################################
630  # Validation workflow
631  ############################################################
632  def runValidationWorkflow(self,mode,run,config,execute=True):
633  print "Processing DQM validation"
634  trial = config.trial
635  runselection = config.runselection
636  result_dir = config.result_dir
637  datasetstr = getDatasetStr(config.datasetpath)
638  dirLabel = 'DQM'
639  config.userdircaf = 'DTCalibration/' + datasetstr + '/Run' + str(run) + '/DQMValidation/' + dirLabel + '/' + 'v' + str(trial)
640 
641  task_dir = config.base_dir + '/' + dirLabel
642  dtDQMValid = DTDQMValidation(run=run,
643  dir=task_dir,
644  config=config)
645  if not execute:
646  dtDQMValid.writeCfg()
647  self.runDQMMerge(run,'',config,False)
648  self.runDQMHarvesting(run,'',config,False)
649 
650  sys.exit(0)
651  else:
652  dtDQMValid.writeCfg()
653  project_valid = dtDQMValid.run()
654 
655  print "Sent validation jobs with project",project_valid
656  print "%.0f%% of jobs are required to finish" % config.jobsFinishedThreshold
657 
658  crabValid = CrabWatch(project_valid)
659  crabValid.setThreshold(config.jobsFinishedThreshold)
660  crabValid.start()
661  crabValid.join()
662 
663  # Produce DQM output
664  dqm_output_dir = self.dqmOutputDir('DQMValidation',dirLabel,config)
665  self.runDQMMerge(run,dqm_output_dir,config)
666  # Run harvesting from merged DQM file
667  dqm_merge_dir = os.path.abspath(result_dir)
668  self.runDQMHarvesting(run,dqm_merge_dir,config)
669 
670  return 0
671 
672  ############################################################
673  # DB Validation workflow
674  ############################################################
675  def runDBValidationWorkflow(self,mode,run,inputFiles,config,execute=True):
676  print "Processing DB validation"
677 
678  dtDBValidation = None
679  if mode == 't0DB':
680  dtDBValidation = DTT0DBValidation(run=run,
681  dir=config.run_dir,
682  input_files=inputFiles,
683  output_dir=config.result_dir,
684  config=config)
685  #dtDBValidation.writeCfg()
686 
687  if execute:
688  dtDBValidation.run()
689 
690  print "Finished processing:"
691  for pset in dtDBValidation.configs: print "--->",pset
692 
693  return 0
DTCalibrationWorker.DTCalibrationWorker.runDBValidationWorkflow
def runDBValidationWorkflow
DB Validation workflow.
Definition: DTCalibrationWorker.py:675
tools.listFilesInCastor
def listFilesInCastor
Definition: tools.py:30
tools.getDatasetStr
def getDatasetStr
Definition: tools.py:15
tools.listFilesLocal
def listFilesLocal
Definition: tools.py:42
DTDqm
Definition: DTDqm.py:1
tools.haddLocal
def haddLocal
Definition: tools.py:98
tools.copyFilesFromCastor
def copyFilesFromCastor
Definition: tools.py:55
DTCalibrationWorker.DTCalibrationWorker.runValidationWorkflow
def runValidationWorkflow
Validation workflow.
Definition: DTCalibrationWorker.py:632
tools.haddInCastor
def haddInCastor
Definition: tools.py:81
tools.copyFilesLocal
def copyFilesLocal
Definition: tools.py:68
Generated for CMSSW Reference Manual by doxygen 1.8.5