CMS 3D CMS Logo
DTTtrigWorkflow.py
Go to the documentation of this file.
1 from __future__ import absolute_import
2 import os
3 import logging
4 import glob
5 
6 from . import tools
7 import FWCore.ParameterSet.Config as cms
8 from .DTWorkflow import DTWorkflow
9 
10 log = logging.getLogger(__name__)
11 
12 class DTttrigWorkflow( DTWorkflow ):
13  """ This class creates and performce / submits ttrig workflow jobs"""
14  def __init__(self, options):
15  # call parent constructor
16  super( DTttrigWorkflow, self ).__init__( options )
17 
18  self.outpath_command_tag = "TTrigCalibration"
19  # Dict to map workflow modes to output file name
20  output_file_dict ={ "timeboxes" : "DTTimeBoxes.root",
21  "residuals" : 'residuals.root',
22  "validation" : "DQM.root"
23  }
24  # Dict to map workflow modes to output folders in main path
25  self.outpath_workflow_mode_dict = {
26  "timeboxes" : "TimeBoxes",
27  "residuals" : "Residuals",
28  "validation" : "TtrigValidation"
29  }
30  # Dict to map workflow modes to database file name
31  self.output_db_dict = { "timeboxes" : {
32  "write": "ttrig_timeboxes_uncorrected_"
33  + self.tag
34  + ".db",
35  "correction": "ttrig_timeboxes_"
36  + self.tag
37  + ".db"
38  },
39  "residuals" : "ttrig_residuals_" + self.tag + ".db"}
40  self.output_file = output_file_dict[self.options.workflow_mode]
41  self.output_files = [self.output_file]
42 
43  def prepare_workflow(self):
44  """ Generalized function to prepare workflow dependent on workflow mode"""
45  function_name = "prepare_" + self.options.workflow_mode + "_" + self.options.command
46  try:
47  fill_function = getattr(self, function_name)
48  except AttributeError:
49  errmsg = "Class `{}` does not implement `{}`"
50  raise NotImplementedError( errmsg.format(self.__class__.__name__,
51  function_name))
52  log.debug("Preparing workflow with function %s" % function_name)
53  # call chosen function
54  fill_function()
55 
56  def get_output_db(self, workflow_mode, command):
57  output_db_file = self.output_db_dict[workflow_mode]
58  if isinstance(output_db_file, dict):
59  return output_db_file[command]
60  return output_db_file
61  ####################################################################
62  # Prepare functions for timeboxes #
63  ####################################################################
64  def prepare_timeboxes_submit(self):
65  self.pset_name = 'dtTTrigCalibration_cfg.py'
66  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cfg'
67  if self.options.datasettype == "Cosmics":
68  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cosmics_cfg'
69  log.debug('Using pset template ' + self.pset_template)
70  self.process = tools.loadCmsProcess(self.pset_template)
71  self.process.GlobalTag.globaltag = self.options.globaltag
72  self.process.dtTTrigCalibration.rootFileName = self.output_file
73  self.process.dtTTrigCalibration.digiLabel = self.digilabel
74 
75  if self.options.inputDBTag:
76  self.add_local_custom_db()
77  self.prepare_common_submit()
78  self.write_pset_file()
79 
80  def prepare_timeboxes_check(self):
81  self.load_options_command("submit")
82 
83  def prepare_timeboxes_write(self):
84  self.output_db_file = self.output_db_dict[self.options.workflow_mode]
85  if isinstance(self.output_db_dict[self.options.workflow_mode], dict):
86  self.output_db_file = self.output_db_file[self.options.command]
87  self.prepare_common_write()
88  merged_file = os.path.join(self.result_path, self.output_file)
89  ttrig_uncorrected_db = os.path.join(self.result_path,
90  self.get_output_db("timeboxes", "write"))
91  self.pset_name = 'dtTTrigWriter_cfg.py'
92  self.pset_template = "CalibMuon.DTCalibration.dtTTrigWriter_cfg"
93  self.process = tools.loadCmsProcess(self.pset_template)
94  self.process.dtTTrigWriter.rootFileName = "file:///" + merged_file
95  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_uncorrected_db
96  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
97  self.write_pset_file()
98 
99  def prepare_timeboxes_correction(self):
100  self.load_options_command("submit")
101  self.pset_name = 'dtTTrigCorrection_cfg.py'
102  self.pset_template = "CalibMuon.DTCalibration.dtTTrigCorrection_cfg"
103  ttrig_timeboxes_db = "ttrig_timeboxes_"+ self.tag + ".db"
104  ttrig_timeboxes_db = os.path.join(self.result_path,
105  self.get_output_db("timeboxes", "correction"))
106  self.process = tools.loadCmsProcess(self.pset_template)
107  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
108  self.process.source.firstRun = self.options.run
109  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_timeboxes_db
110 
111  if not self.options.inputCalibDB:
112  self.options.inputCalibDB = os.path.join(self.result_path,
113  self.get_output_db( "timeboxes", "write" ) )
114  if self.options.inputVDriftDB:
115  log.warning("Options inputVDriftDB has no effect for timeboxes correction")
116  if self.options.inputT0DB:
117  log.warning("Options inputT0DB has no effect for timeboxes correction")
118  self.add_local_calib_db(local=True)
119  self.write_pset_file()
120 
121  def prepare_timeboxes_dump(self):
122  self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
123  self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
124  if self.options.input_dumpDB:
125  try:
126  test = self.result_path
127  self.load_options_command("correction")
128  except:
129  pass
130  dbpath = os.path.abspath(self.options.input_dumpDB)
131  else:
132  dbpath = os.path.abspath( os.path.join(self.result_path,
133  self.get_output_db("timeboxes",
134  "correction")))
135  self.prepare_common_dump(dbpath)
136  self.write_pset_file()
137 
138  def prepare_timeboxes_all(self):
139  # individual prepare functions for all tasks will be called in
140  # main implementation of all
141  self.all_commands=["submit", "check", "write", "correction", "dump"]
142 
143  ####################################################################
144  # prepare functions for residuals #
145  ####################################################################
146  def prepare_residuals_submit(self):
147  self.pset_name = 'dtResidualCalibration_cfg.py'
148  self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cfg'
149  if self.options.datasettype == "Cosmics":
150  self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cosmics_cfg'
151  self.process = tools.loadCmsProcess(self.pset_template)
152  #~ self.process.GlobalTag.globaltag = cms.string(self.options.globaltag)
153  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
154  self.process.dtResidualCalibration.rootFileName = self.output_file
155  self.prepare_common_submit()
156  if self.options.fromMuons:
157  self.process.dtResidualCalibration.Muons = cms.InputTag("muons")
158  if self.options.histoRange:
159  self.process.dtResidualCalibration.histogramRange = cms.double(float(self.options.histoRange))
160  if self.options.inputCalibDB:
161  self.add_local_calib_db()
162  self.write_pset_file()
163 
164  def prepare_residuals_check(self):
165  self.load_options_command("submit")
166 
167  def prepare_residuals_correction(self):
168  self.pset_name = "dtTTrigResidualCorrection_cfg.py"
169  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigResidualCorrection_cfg'
170  self.process = tools.loadCmsProcess(self.pset_template)
171  self.load_options_command("submit")
172  self.process.source.firstRun = cms.untracked.uint32(self.options.run)
173  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
174 
175  tag = self.prepare_common_write()
176  if self.options.inputT0DB:
177  log.warning("Option inputT0DB not supported for residual corrections")
178 
179  if self.options.inputDBTag:
180  self.add_local_custom_db(local=True)
181  if self.options.inputVDriftDB:
182  self.add_local_vdrift_db(local=True)
183  if self.options.inputCalibDB:
184  self.add_local_calib_db(local=True)
185  # Change DB label if running on Cosmics
186  if self.options.datasettype == "Cosmics":
187  self.process.dtTTrigResidualCorrection.dbLabel = 'cosmics'
188  self.process.dtTTrigResidualCorrection.correctionAlgoConfig.dbLabel = 'cosmics'
189  ttrig_ResidCorr_db = os.path.abspath( os.path.join(self.result_path,
190  self.get_output_db("residuals", "write")))
191  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_ResidCorr_db
192  rootfile_path = os.path.abspath( os.path.join(self.result_path, self.output_file))
193  merged_file = os.path.join(self.result_path, self.output_file)
194  self.process.dtTTrigResidualCorrection.correctionAlgoConfig.residualsRootFile = merged_file
195  self.write_pset_file()
196 
197  def prepare_residuals_dump(self):
198  self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
199  self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
200  if self.options.input_dumpDB:
201  dbpath = os.path.abspath(self.options.input_dumpDB)
202  else:
203  try:
204  test = self.result_path
205  self.load_options_command("write")
206  except:
207  pass
208  dbpath = os.path.abspath( os.path.join(self.result_path,
209  self.get_output_db("residuals",
210  "write")))
211  self.prepare_common_dump(dbpath)
212  self.write_pset_file()
213 
214  def prepare_residuals_all(self):
215  # individual prepare functions for all tasks will be called in
216  # main implementation of all
217  self.all_commands=["submit", "check", "correction", "dump"]
218 
219  ####################################################################
220  # prepare functions for validation #
221  ####################################################################
222 
223  def prepare_validation_submit(self):
224  self.pset_name = 'dtCalibValidation_cfg.py'
225  self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cfg'
226  if self.options.datasettype == "Cosmics":
227  self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cosmics_cfg'
228  if self.options.fromMuons:
229  self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidationFromMuons_cfg'
230  self.process = tools.loadCmsProcess(self.pset_template)
231  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
232  self.prepare_common_submit()
233  if self.options.inputCalibDB:
234  self.add_local_calib_db()
235  self.write_pset_file()
236 
237  def prepare_validation_check(self):
238  self.load_options_command("submit")
239 
240  def prepare_validation_write(self):
241  self.pset_name = 'dtDQMClient_cfg.py'
242  self.pset_template = 'CalibMuon.DTCalibration.dtDQMClient_cfg'
243  self.process = tools.loadCmsProcess(self.pset_template)
244  self.prepare_common_write(do_hadd = False)
245  dqm_files = glob.glob(os.path.join( self.local_path,
246  "unmerged_results",
247  "DQM_*.root"))
248  dqm_files[:] = ["file://"+txt for txt in dqm_files]
249  self.process.source.fileNames = dqm_files
250  self.process.dqmSaver.dirName = os.path.abspath(self.result_path)
251  self.process.dqmSaver.workflow = str(self.options.datasetpath)
252  if self.process.DQMStore.collateHistograms == True:
253  self.process.dqmSaver.forceRunNumber = self.options.run
254  self.write_pset_file()
255 
256  def summary(self):
257  pass
258 
259  def prepare_validation_all(self):
260  # individual prepare functions for all tasks will be called in
261  # main implementation of all
262  self.all_commands=["submit", "check","write"]
263 
264  ####################################################################
265  # CLI creation #
266  ####################################################################
267  @classmethod
268  def add_parser_options(cls, subparser_container):
269  ttrig_parser = subparser_container.add_parser( "ttrig",
270  #parents=[mutual_parent_parser, common_parent_parser],
271  help = "" ) # What does ttrig
272 
273 
274  ################################################################
275  # Sub parser options for workflow modes #
276  ################################################################
277  ttrig_subparsers = ttrig_parser.add_subparsers( dest="workflow_mode",
278  help="Possible workflow modes",)
279  ## Add all workflow modes for ttrig
280  ttrig_timeboxes_subparser = ttrig_subparsers.add_parser( "timeboxes",
281  help = "" )
282  ttrig_residuals_subparser = ttrig_subparsers.add_parser( "residuals",
283  help = "" )
284  ttrig_validation_subparser = ttrig_subparsers.add_parser( "validation",
285  help = "" )
286  ################################################################
287  # Sub parser options for workflow mode timeboxes #
288  ################################################################
289  ttrig_timeboxes_subparsers = ttrig_timeboxes_subparser.add_subparsers( dest="command",
290  help="Possible commands for timeboxes")
291  ttrig_timeboxes_submit_parser = ttrig_timeboxes_subparsers.add_parser(
292  "submit",
293  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
294  super(DTttrigWorkflow,cls).get_submission_options_parser(),
295  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
296  help = "Submit job to the GRID via crab3")
297 
298  ttrig_timeboxes_check_parser = ttrig_timeboxes_subparsers.add_parser(
299  "check",
300  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
301  super(DTttrigWorkflow,cls).get_check_options_parser(),],
302  help = "Check status of submitted jobs")
303 
304  ttrig_timeboxes_write_parser = ttrig_timeboxes_subparsers.add_parser(
305  "write",
306  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
307  super(DTttrigWorkflow,cls).get_write_options_parser()
308  ],
309  help = "Write result from root output to text file")
310 
311  ttrig_timeboxes_correction_parser = ttrig_timeboxes_subparsers.add_parser(
312  "correction",
313  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
314  super(DTttrigWorkflow,cls).get_local_input_db_options_parser()
315  ],
316  help = "Perform correction on uncorrected ttrig database")
317 
318  ttrig_timeboxes_dump_parser = ttrig_timeboxes_subparsers.add_parser(
319  "dump",
320  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
321  super(DTttrigWorkflow,cls).get_dump_options_parser()],
322  help = "Dump database to text file")
323 
324  ttrig_timeboxes_all_parser = ttrig_timeboxes_subparsers.add_parser(
325  "all",
326  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
327  super(DTttrigWorkflow,cls).get_submission_options_parser(),
328  super(DTttrigWorkflow,cls).get_check_options_parser(),
329  super(DTttrigWorkflow,cls).get_input_db_options_parser(),
330  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
331  super(DTttrigWorkflow,cls).get_write_options_parser(),
332  super(DTttrigWorkflow,cls).get_dump_options_parser()
333  ],
334  help = "Perform all steps: submit, check, write, correction,"\
335  "dump in this order")
336 
337  ################################################################
338  # Sub parser options for workflow mode residuals #
339  ################################################################
340  ttrig_residuals_subparsers = ttrig_residuals_subparser.add_subparsers( dest="command",
341  help="Possible commands for residuals")
342  ttrig_residuals_submit_parser = ttrig_residuals_subparsers.add_parser(
343  "submit",
344  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
345  super(DTttrigWorkflow,cls).get_submission_options_parser(),
346  super(DTttrigWorkflow,cls).get_check_options_parser(),
347  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
348  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
349  help = "Submit job to the GRID via crab3")
350 
351  ttrig_residuals_check_parser = ttrig_residuals_subparsers.add_parser(
352  "check",
353  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
354  super(DTttrigWorkflow,cls).get_check_options_parser(),],
355  help = "Check status of submitted jobs")
356 
357  ttrig_residuals_correct_parser = ttrig_residuals_subparsers.add_parser(
358  "correction",
359  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
360  super(DTttrigWorkflow,cls).get_write_options_parser(),
361  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
362  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
363  help = "Perform residual corrections")
364  ttrig_residuals_correct_parser.add_argument("--globaltag",
365  help="Alternative globalTag. Otherwise the gt for sunmission is used")
366 
367  ttrig_residuals_dump_parser = ttrig_residuals_subparsers.add_parser(
368  "dump",
369  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
370  super(DTttrigWorkflow,cls).get_dump_options_parser()],
371  help = "Dump database to text file")
372 
373  ttrig_residuals_all_parser = ttrig_residuals_subparsers.add_parser(
374  "all",
375  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
376  super(DTttrigWorkflow,cls).get_submission_options_parser(),
377  super(DTttrigWorkflow,cls).get_check_options_parser(),
378  super(DTttrigWorkflow,cls).get_write_options_parser(),
379  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
380  super(DTttrigWorkflow,cls).get_dump_options_parser(),
381  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
382  help = "Perform all steps: submit, check, correct, dump")
383 
384  ################################################################
385  # Sub parser options for workflow mode validation #
386  ################################################################
387  ttrig_validation_subparsers = ttrig_validation_subparser.add_subparsers( dest="command",
388  help="Possible commands for residuals")
389 
390  ttrig_validation_submit_parser = ttrig_validation_subparsers.add_parser(
391  "submit",
392  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
393  super(DTttrigWorkflow,cls).get_submission_options_parser(),
394  super(DTttrigWorkflow,cls).get_check_options_parser(),
395  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
396  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
397  help = "Submit job to the GRID via crab3")
398 
399 
400  ttrig_validation_check_parser = ttrig_validation_subparsers.add_parser(
401  "check",
402  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
403  super(DTttrigWorkflow,cls).get_check_options_parser(),],
404  help = "Check status of submitted jobs")
405 
406  ttrig_validation_write_parser = ttrig_validation_subparsers.add_parser(
407  "write",
408  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
409  super(DTttrigWorkflow,cls).get_write_options_parser()],
410  help = "Create summary for validation")
411 
412  ttrig_validation_all_parser = ttrig_validation_subparsers.add_parser(
413  "all",
414  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
415  super(DTttrigWorkflow,cls).get_submission_options_parser(),
416  super(DTttrigWorkflow,cls).get_check_options_parser(),
417  super(DTttrigWorkflow,cls).get_write_options_parser(),
418  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
419  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
420  help = "Perform all steps: submit, check, summary")
DTWorkflow.DTWorkflow.add_local_vdrift_db
def add_local_vdrift_db(self, local=False)
Definition: DTWorkflow.py:148
DTWorkflow.DTWorkflow.add_local_calib_db
def add_local_calib_db(self, local=False)
Definition: DTWorkflow.py:163
DTWorkflow.DTWorkflow.load_options_command
def load_options_command(self, command)
Definition: DTWorkflow.py:404
CLIHelper.CLIHelper.get_common_options_parser
def get_common_options_parser(cls)
Definition: CLIHelper.py:5
CLIHelper.CLIHelper.get_dump_options_parser
def get_dump_options_parser(cls)
Definition: CLIHelper.py:135
DTWorkflow.DTWorkflow.prepare_common_write
def prepare_common_write(self, do_hadd=True)
Definition: DTWorkflow.py:212
CLIHelper.CLIHelper.get_input_db_options_parser
def get_input_db_options_parser(cls)
Definition: CLIHelper.py:50
DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit
def prepare_residuals_submit(self)
prepare functions for residuals #
Definition: DTTtrigWorkflow.py:146
DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_submit
def prepare_timeboxes_submit(self)
Prepare functions for timeboxes #.
Definition: DTTtrigWorkflow.py:64
CLIHelper.CLIHelper.get_local_input_db_options_parser
def get_local_input_db_options_parser(cls)
Definition: CLIHelper.py:65
tools.loadCmsProcess
def loadCmsProcess(psetPath)
Definition: tools.py:56
DTWorkflow.DTWorkflow.prepare_common_submit
def prepare_common_submit(self)
Definition: DTWorkflow.py:194
DTWorkflow.DTWorkflow.prepare_common_dump
def prepare_common_dump(self, db_path)
Definition: DTWorkflow.py:229
DTTtrigWorkflow.DTttrigWorkflow.get_output_db
def get_output_db(self, workflow_mode, command)
Definition: DTTtrigWorkflow.py:56
DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit
def prepare_validation_submit(self)
prepare functions for validation #
Definition: DTTtrigWorkflow.py:223
DTWorkflow.DTWorkflow.write_pset_file
def write_pset_file(self)
Definition: DTWorkflow.py:377
CLIHelper.CLIHelper.get_submission_options_parser
def get_submission_options_parser(cls)
Definition: CLIHelper.py:79
str
#define str(s)
Definition: TestProcessor.cc:48
DTTtrigWorkflow.DTttrigWorkflow.add_parser_options
def add_parser_options(cls, subparser_container)
CLI creation #.
Definition: DTTtrigWorkflow.py:268
CLIHelper.CLIHelper.get_write_options_parser
def get_write_options_parser(cls)
Definition: CLIHelper.py:125
DTWorkflow.DTWorkflow.add_local_custom_db
def add_local_custom_db(self)
Definition: DTWorkflow.py:183
CLIHelper.CLIHelper.get_check_options_parser
def get_check_options_parser(cls)
Definition: CLIHelper.py:113
Generated for CMSSW Reference Manual by doxygen 1.8.11