CMS 3D CMS Logo
DTTtrigWorkflow.py
Go to the documentation of this file.
1 import os
2 import logging
3 import glob
4 
5 import tools
6 import FWCore.ParameterSet.Config as cms
7 from DTWorkflow import DTWorkflow
8 
9 log = logging.getLogger(__name__)
10 
11 class DTttrigWorkflow( DTWorkflow ):
12  """ This class creates and performce / submits ttrig workflow jobs"""
13  def __init__(self, options):
14  # call parent constructor
15  super( DTttrigWorkflow, self ).__init__( options )
16 
17  self.outpath_command_tag = "TTrigCalibration"
18  # Dict to map workflow modes to output file name
19  output_file_dict ={ "timeboxes" : "DTTimeBoxes.root",
20  "residuals" : 'residuals.root',
21  "validation" : "DQM.root"
22  }
23  # Dict to map workflow modes to output folders in main path
24  self.outpath_workflow_mode_dict = {
25  "timeboxes" : "TimeBoxes",
26  "residuals" : "Residuals",
27  "validation" : "TtrigValidation"
28  }
29  # Dict to map workflow modes to database file name
30  self.output_db_dict = { "timeboxes" : {
31  "write": "ttrig_timeboxes_uncorrected_"
32  + self.tag
33  + ".db",
34  "correction": "ttrig_timeboxes_"
35  + self.tag
36  + ".db"
37  },
38  "residuals" : "ttrig_residuals_" + self.tag + ".db"}
39  self.output_file = output_file_dict[self.options.workflow_mode]
40  self.output_files = [self.output_file]
41 
42  def prepare_workflow(self):
43  """ Generalized function to prepare workflow dependent on workflow mode"""
44  function_name = "prepare_" + self.options.workflow_mode + "_" + self.options.command
45  try:
46  fill_function = getattr(self, function_name)
47  except AttributeError:
48  errmsg = "Class `{}` does not implement `{}`"
49  raise NotImplementedError( errmsg.format(self.__class__.__name__,
50  function_name))
51  log.debug("Preparing workflow with function %s" % function_name)
52  # call chosen function
53  fill_function()
54 
55  def get_output_db(self, workflow_mode, command):
56  output_db_file = self.output_db_dict[workflow_mode]
57  if type(output_db_file) == dict:
58  return output_db_file[command]
59  return output_db_file
60  ####################################################################
61  # Prepare functions for timeboxes #
62  ####################################################################
63  def prepare_timeboxes_submit(self):
64  self.pset_name = 'dtTTrigCalibration_cfg.py'
65  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cfg'
66  if self.options.datasettype == "Cosmics":
67  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigCalibration_cosmics_cfg'
68  log.debug('Using pset template ' + self.pset_template)
69  self.process = tools.loadCmsProcess(self.pset_template)
70  self.process.GlobalTag.globaltag = self.options.globaltag
71  self.process.dtTTrigCalibration.rootFileName = self.output_file
72  self.process.dtTTrigCalibration.digiLabel = self.digilabel
73 
74  if self.options.inputDBTag:
75  self.add_local_custom_db()
76  self.prepare_common_submit()
77  self.write_pset_file()
78 
79  def prepare_timeboxes_check(self):
80  self.load_options_command("submit")
81 
82  def prepare_timeboxes_write(self):
83  self.output_db_file = self.output_db_dict[self.options.workflow_mode]
84  if type(self.output_db_dict[self.options.workflow_mode]) == dict:
85  self.output_db_file = self.output_db_file[self.options.command]
86  self.prepare_common_write()
87  merged_file = os.path.join(self.result_path, self.output_file)
88  ttrig_uncorrected_db = os.path.join(self.result_path,
89  self.get_output_db("timeboxes", "write"))
90  self.pset_name = 'dtTTrigWriter_cfg.py'
91  self.pset_template = "CalibMuon.DTCalibration.dtTTrigWriter_cfg"
92  self.process = tools.loadCmsProcess(self.pset_template)
93  self.process.dtTTrigWriter.rootFileName = "file:///" + merged_file
94  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_uncorrected_db
95  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
96  self.write_pset_file()
97 
98  def prepare_timeboxes_correction(self):
99  self.load_options_command("submit")
100  self.pset_name = 'dtTTrigCorrection_cfg.py'
101  self.pset_template = "CalibMuon.DTCalibration.dtTTrigCorrection_cfg"
102  ttrig_timeboxes_db = "ttrig_timeboxes_"+ self.tag + ".db"
103  ttrig_timeboxes_db = os.path.join(self.result_path,
104  self.get_output_db("timeboxes", "correction"))
105  self.process = tools.loadCmsProcess(self.pset_template)
106  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
107  self.process.source.firstRun = self.options.run
108  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_timeboxes_db
109 
110  if not self.options.inputCalibDB:
111  self.options.inputCalibDB = os.path.join(self.result_path,
112  self.get_output_db( "timeboxes", "write" ) )
113  if self.options.inputVDriftDB:
114  log.warning("Options inputVDriftDB has no effect for timeboxes correction")
115  if self.options.inputT0DB:
116  log.warning("Options inputT0DB has no effect for timeboxes correction")
117  self.add_local_calib_db(local=True)
118  self.write_pset_file()
119 
120  def prepare_timeboxes_dump(self):
121  self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
122  self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
123  if self.options.input_dumpDB:
124  try:
125  test = self.result_path
126  self.load_options_command("correction")
127  except:
128  pass
129  dbpath = os.path.abspath(self.options.input_dumpDB)
130  else:
131  dbpath = os.path.abspath( os.path.join(self.result_path,
132  self.get_output_db("timeboxes",
133  "correction")))
134  self.prepare_common_dump(dbpath)
135  self.write_pset_file()
136 
137  def prepare_timeboxes_all(self):
138  # individual prepare functions for all tasks will be called in
139  # main implementation of all
140  self.all_commands=["submit", "check", "write", "correction", "dump"]
141 
142  ####################################################################
143  # prepare functions for residuals #
144  ####################################################################
145  def prepare_residuals_submit(self):
146  self.pset_name = 'dtResidualCalibration_cfg.py'
147  self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cfg'
148  if self.options.datasettype == "Cosmics":
149  self.pset_template = 'CalibMuon.DTCalibration.dtResidualCalibration_cosmics_cfg'
150  self.process = tools.loadCmsProcess(self.pset_template)
151  #~ self.process.GlobalTag.globaltag = cms.string(self.options.globaltag)
152  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
153  self.process.dtResidualCalibration.rootFileName = self.output_file
154  self.prepare_common_submit()
155  if self.options.histoRange:
156  self.process.dtResidualCalibration.histogramRange = cms.double(float(self.options.histoRange))
157  if self.options.inputCalibDB:
158  self.add_local_calib_db()
159  self.write_pset_file()
160 
161  def prepare_residuals_check(self):
162  self.load_options_command("submit")
163 
164  def prepare_residuals_correction(self):
165  self.pset_name = "dtTTrigResidualCorrection_cfg.py"
166  self.pset_template = 'CalibMuon.DTCalibration.dtTTrigResidualCorrection_cfg'
167  self.process = tools.loadCmsProcess(self.pset_template)
168  self.load_options_command("submit")
169  self.process.source.firstRun = cms.untracked.uint32(self.options.run)
170  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
171 
172  tag = self.prepare_common_write()
173  if self.options.inputT0DB:
174  log.warning("Option inputT0DB not supported for residual corrections")
175 
176  if self.options.inputDBTag:
177  self.add_local_custom_db(local=True)
178  if self.options.inputVDriftDB:
179  self.add_local_vdrift_db(local=True)
180  if self.options.inputCalibDB:
181  self.add_local_calib_db(local=True)
182  # Change DB label if running on Cosmics
183  if self.options.datasettype == "Cosmics":
184  self.process.dtTTrigResidualCorrection.dbLabel = 'cosmics'
185  self.process.dtTTrigResidualCorrection.correctionAlgoConfig.dbLabel = 'cosmics'
186  ttrig_ResidCorr_db = os.path.abspath( os.path.join(self.result_path,
187  self.get_output_db("residuals", "write")))
188  self.process.PoolDBOutputService.connect = 'sqlite_file:%s' % ttrig_ResidCorr_db
189  rootfile_path = os.path.abspath( os.path.join(self.result_path, self.output_file))
190  merged_file = os.path.join(self.result_path, self.output_file)
191  self.process.dtTTrigResidualCorrection.correctionAlgoConfig.residualsRootFile = merged_file
192  self.write_pset_file()
193 
194  def prepare_residuals_dump(self):
195  self.pset_name = 'dumpDBToFile_ResidCorr_cfg.py'
196  self.pset_template = 'CalibMuon.DTCalibration.dumpDBToFile_ttrig_cfg'
197  if self.options.input_dumpDB:
198  dbpath = os.path.abspath(self.options.input_dumpDB)
199  else:
200  try:
201  test = self.result_path
202  self.load_options_command("write")
203  except:
204  pass
205  dbpath = os.path.abspath( os.path.join(self.result_path,
206  self.get_output_db("residuals",
207  "write")))
208  self.prepare_common_dump(dbpath)
209  self.write_pset_file()
210 
211  def prepare_residuals_all(self):
212  # individual prepare functions for all tasks will be called in
213  # main implementation of all
214  self.all_commands=["submit", "check", "correction", "dump"]
215 
216  ####################################################################
217  # prepare functions for validation #
218  ####################################################################
219 
220  def prepare_validation_submit(self):
221  self.pset_name = 'dtCalibValidation_cfg.py'
222  self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cfg'
223  if self.options.datasettype == "Cosmics":
224  self.pset_template = 'CalibMuon.DTCalibration.dtCalibValidation_cosmics_cfg'
225  self.process = tools.loadCmsProcess(self.pset_template)
226  self.process.GlobalTag.globaltag = cms.string(str(self.options.globaltag))
227  self.prepare_common_submit()
228  self.add_local_calib_db()
229  self.write_pset_file()
230 
231  def prepare_validation_check(self):
232  self.load_options_command("submit")
233 
234  def prepare_validation_write(self):
235  self.pset_name = 'dtDQMClient_cfg.py'
236  self.pset_template = 'CalibMuon.DTCalibration.dtDQMClient_cfg'
237  self.process = tools.loadCmsProcess(self.pset_template)
238  self.prepare_common_write(do_hadd = False)
239  dqm_files = glob.glob(os.path.join( self.local_path,
240  "unmerged_results",
241  "DQM_*.root"))
242  dqm_files[:] = ["file://"+txt for txt in dqm_files]
243  self.process.source.fileNames = dqm_files
244  self.process.dqmSaver.dirName = os.path.abspath(self.result_path)
245  self.process.dqmSaver.workflow = str(self.options.datasetpath)
246  if self.process.DQMStore.collateHistograms == True:
247  self.process.dqmSaver.forceRunNumber = self.options.run
248  self.write_pset_file()
249 
250  def summary(self):
251  pass
252 
253  def prepare_validation_all(self):
254  # individual prepare functions for all tasks will be called in
255  # main implementation of all
256  self.all_commands=["submit", "check","write"]
257 
258  ####################################################################
259  # CLI creation #
260  ####################################################################
261  @classmethod
262  def add_parser_options(cls, subparser_container):
263  ttrig_parser = subparser_container.add_parser( "ttrig",
264  #parents=[mutual_parent_parser, common_parent_parser],
265  help = "" ) # What does ttrig
266 
267 
268  ################################################################
269  # Sub parser options for workflow modes #
270  ################################################################
271  ttrig_subparsers = ttrig_parser.add_subparsers( dest="workflow_mode",
272  help="Possible workflow modes",)
273  ## Add all workflow modes for ttrig
274  ttrig_timeboxes_subparser = ttrig_subparsers.add_parser( "timeboxes",
275  help = "" )
276  ttrig_residuals_subparser = ttrig_subparsers.add_parser( "residuals",
277  help = "" )
278  ttrig_validation_subparser = ttrig_subparsers.add_parser( "validation",
279  help = "" )
280  ################################################################
281  # Sub parser options for workflow mode timeboxes #
282  ################################################################
283  ttrig_timeboxes_subparsers = ttrig_timeboxes_subparser.add_subparsers( dest="command",
284  help="Possible commands for timeboxes")
285  ttrig_timeboxes_submit_parser = ttrig_timeboxes_subparsers.add_parser(
286  "submit",
287  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
288  super(DTttrigWorkflow,cls).get_submission_options_parser(),
289  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
290  help = "Submit job to the GRID via crab3")
291 
292  ttrig_timeboxes_check_parser = ttrig_timeboxes_subparsers.add_parser(
293  "check",
294  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
295  super(DTttrigWorkflow,cls).get_check_options_parser(),],
296  help = "Check status of submitted jobs")
297 
298  ttrig_timeboxes_write_parser = ttrig_timeboxes_subparsers.add_parser(
299  "write",
300  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
301  super(DTttrigWorkflow,cls).get_write_options_parser()
302  ],
303  help = "Write result from root output to text file")
304 
305  ttrig_timeboxes_correction_parser = ttrig_timeboxes_subparsers.add_parser(
306  "correction",
307  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
308  super(DTttrigWorkflow,cls).get_local_input_db_options_parser()
309  ],
310  help = "Perform correction on uncorrected ttrig database")
311 
312  ttrig_timeboxes_dump_parser = ttrig_timeboxes_subparsers.add_parser(
313  "dump",
314  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
315  super(DTttrigWorkflow,cls).get_dump_options_parser()],
316  help = "Dump database to text file")
317 
318  ttrig_timeboxes_all_parser = ttrig_timeboxes_subparsers.add_parser(
319  "all",
320  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
321  super(DTttrigWorkflow,cls).get_submission_options_parser(),
322  super(DTttrigWorkflow,cls).get_check_options_parser(),
323  super(DTttrigWorkflow,cls).get_input_db_options_parser(),
324  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
325  super(DTttrigWorkflow,cls).get_write_options_parser(),
326  super(DTttrigWorkflow,cls).get_dump_options_parser()
327  ],
328  help = "Perform all steps: submit, check, write, correction,"\
329  "dump in this order")
330 
331  ################################################################
332  # Sub parser options for workflow mode residuals #
333  ################################################################
334  ttrig_residuals_subparsers = ttrig_residuals_subparser.add_subparsers( dest="command",
335  help="Possible commands for residuals")
336  ttrig_residuals_submit_parser = ttrig_residuals_subparsers.add_parser(
337  "submit",
338  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
339  super(DTttrigWorkflow,cls).get_submission_options_parser(),
340  super(DTttrigWorkflow,cls).get_check_options_parser(),
341  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
342  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
343  help = "Submit job to the GRID via crab3")
344 
345  ttrig_residuals_check_parser = ttrig_residuals_subparsers.add_parser(
346  "check",
347  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
348  super(DTttrigWorkflow,cls).get_check_options_parser(),],
349  help = "Check status of submitted jobs")
350 
351  ttrig_residuals_correct_parser = ttrig_residuals_subparsers.add_parser(
352  "correction",
353  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
354  super(DTttrigWorkflow,cls).get_write_options_parser(),
355  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
356  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
357  help = "Perform residual corrections")
358  ttrig_residuals_correct_parser.add_argument("--globaltag",
359  help="Alternative globalTag. Otherwise the gt for sunmission is used")
360 
361  ttrig_residuals_dump_parser = ttrig_residuals_subparsers.add_parser(
362  "dump",
363  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
364  super(DTttrigWorkflow,cls).get_dump_options_parser()],
365  help = "Dump database to text file")
366 
367  ttrig_residuals_all_parser = ttrig_residuals_subparsers.add_parser(
368  "all",
369  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
370  super(DTttrigWorkflow,cls).get_submission_options_parser(),
371  super(DTttrigWorkflow,cls).get_check_options_parser(),
372  super(DTttrigWorkflow,cls).get_write_options_parser(),
373  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
374  super(DTttrigWorkflow,cls).get_dump_options_parser(),
375  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
376  help = "Perform all steps: submit, check, correct, dump")
377 
378  ################################################################
379  # Sub parser options for workflow mode validation #
380  ################################################################
381  ttrig_validation_subparsers = ttrig_validation_subparser.add_subparsers( dest="command",
382  help="Possible commands for residuals")
383 
384  ttrig_validation_submit_parser = ttrig_validation_subparsers.add_parser(
385  "submit",
386  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
387  super(DTttrigWorkflow,cls).get_submission_options_parser(),
388  super(DTttrigWorkflow,cls).get_check_options_parser(),
389  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
390  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
391  help = "Submit job to the GRID via crab3")
392 
393 
394  ttrig_validation_check_parser = ttrig_validation_subparsers.add_parser(
395  "check",
396  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
397  super(DTttrigWorkflow,cls).get_check_options_parser(),],
398  help = "Check status of submitted jobs")
399 
400  ttrig_validation_write_parser = ttrig_validation_subparsers.add_parser(
401  "write",
402  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
403  super(DTttrigWorkflow,cls).get_write_options_parser()],
404  help = "Create summary for validation")
405 
406  ttrig_validation_all_parser = ttrig_validation_subparsers.add_parser(
407  "all",
408  parents=[super(DTttrigWorkflow,cls).get_common_options_parser(),
409  super(DTttrigWorkflow,cls).get_submission_options_parser(),
410  super(DTttrigWorkflow,cls).get_check_options_parser(),
411  super(DTttrigWorkflow,cls).get_write_options_parser(),
412  super(DTttrigWorkflow,cls).get_local_input_db_options_parser(),
413  super(DTttrigWorkflow,cls).get_input_db_options_parser()],
414  help = "Perform all steps: submit, check, summary")
DTWorkflow.DTWorkflow.add_local_vdrift_db
def add_local_vdrift_db(self, local=False)
Definition: DTWorkflow.py:146
DTWorkflow.DTWorkflow.add_local_calib_db
def add_local_calib_db(self, local=False)
Definition: DTWorkflow.py:161
DTWorkflow.DTWorkflow.load_options_command
def load_options_command(self, command)
Definition: DTWorkflow.py:402
CLIHelper.CLIHelper.get_common_options_parser
def get_common_options_parser(cls)
Definition: CLIHelper.py:5
CLIHelper.CLIHelper.get_dump_options_parser
def get_dump_options_parser(cls)
Definition: CLIHelper.py:133
DTWorkflow.DTWorkflow.prepare_common_write
def prepare_common_write(self, do_hadd=True)
Definition: DTWorkflow.py:210
CLIHelper.CLIHelper.get_input_db_options_parser
def get_input_db_options_parser(cls)
Definition: CLIHelper.py:50
DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit
def prepare_residuals_submit(self)
prepare functions for residuals #
Definition: DTTtrigWorkflow.py:145
DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_submit
def prepare_timeboxes_submit(self)
Prepare functions for timeboxes #.
Definition: DTTtrigWorkflow.py:63
CLIHelper.CLIHelper.get_local_input_db_options_parser
def get_local_input_db_options_parser(cls)
Definition: CLIHelper.py:65
tools.loadCmsProcess
def loadCmsProcess(psetPath)
Definition: tools.py:55
DTWorkflow.DTWorkflow.prepare_common_submit
def prepare_common_submit(self)
Definition: DTWorkflow.py:192
DTWorkflow.DTWorkflow.prepare_common_dump
def prepare_common_dump(self, db_path)
Definition: DTWorkflow.py:227
DTTtrigWorkflow.DTttrigWorkflow.get_output_db
def get_output_db(self, workflow_mode, command)
Definition: DTTtrigWorkflow.py:55
DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit
def prepare_validation_submit(self)
prepare functions for validation #
Definition: DTTtrigWorkflow.py:220
DTWorkflow.DTWorkflow.write_pset_file
def write_pset_file(self)
Definition: DTWorkflow.py:375
CLIHelper.CLIHelper.get_submission_options_parser
def get_submission_options_parser(cls)
Definition: CLIHelper.py:79
DTTtrigWorkflow.DTttrigWorkflow.add_parser_options
def add_parser_options(cls, subparser_container)
CLI creation #.
Definition: DTTtrigWorkflow.py:262
CLIHelper.CLIHelper.get_write_options_parser
def get_write_options_parser(cls)
Definition: CLIHelper.py:123
DTWorkflow.DTWorkflow.add_local_custom_db
def add_local_custom_db(self)
Definition: DTWorkflow.py:181
CLIHelper.CLIHelper.get_check_options_parser
def get_check_options_parser(cls)
Definition: CLIHelper.py:111
Generated for CMSSW Reference Manual by doxygen 1.8.11