CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Static Public Member Functions | Public Attributes | Private Attributes
DTWorkflow.DTWorkflow Class Reference
Inheritance diagram for DTWorkflow.DTWorkflow:
DTT0WireWorkflow.DTT0WireWorkflow

Public Member Functions

def __init__
 
def add_local_calib_db
 
def add_local_custom_db
 
def add_local_t0_db
 
def add_local_vdrift_db
 
def add_preselection
 
def add_raw_option
 
def all
 
def check
 
def check_missing_options
 
def correction
 
def dump
 
def dump_options
 
def get_config_name
 
def get_output_files
 
def load_options
 
def load_options_command
 
def local_path
 
def outpath_workflow_mode_tag
 
def prepare_common_dump
 
def prepare_common_submit
 
def prepare_common_write
 
def prepare_workflow
 
def pset_path
 
def pset_template_base_bath
 
def remote_out_path
 
def result_path
 
def run
 
def runCMSSWtask
 
def submit
 
def tag
 
def user
 
def write
 
def write_pset_file
 

Static Public Member Functions

def addPoolDBESSource
 

Public Attributes

 digilabel
 
 files_reveived
 
 input_files
 
 options
 
 outpath_command_tag
 
 output_files
 
 process
 
 pset_name
 
 required_options_dict
 
 required_options_prepare_dict
 
 run_all_command
 

Private Attributes

 _user
 

Detailed Description

This is the base class for all DTWorkflows and contains some
    common tasks

Definition at line 18 of file DTWorkflow.py.

Constructor & Destructor Documentation

def DTWorkflow.DTWorkflow.__init__ (   self,
  options 
)

Definition at line 21 of file DTWorkflow.py.

21 
22  def __init__(self, options):
23  self.options = options
24  super( DTWorkflow, self ).__init__()
25  self.digilabel = "muonDTDigis"
26  # dict to hold required variables. Can not be marked in argparse to allow
27  # loading of options from config
28  self.required_options_dict = {}
29  self.required_options_prepare_dict = {}
30  self.fill_required_options_dict()
31  self.fill_required_options_prepare_dict()
32  # These variables are determined in the derived classes
33  self.pset_name = ""
34  self.outpath_command_tag = ""
35  self.output_files = []
36  self.input_files = []
37 
38  self.run_all_command = False
39  self.files_reveived = False
40  self._user = ""
41  # change to working directory
42  os.chdir(self.options.working_dir)

Member Function Documentation

def DTWorkflow.DTWorkflow.add_local_calib_db (   self,
  local = False 
) 
Add a local calib database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 163 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, DTWorkflow.DTWorkflow.process, and str.

Referenced by DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit().

163 
164  def add_local_calib_db(self, local=False):
165  """ Add a local calib database as input. Use the option local is used
166  if the pset is processed locally and not with crab.
167  """
168  label = ''
169  if self.options.datasettype == "Cosmics":
170  label = 'cosmics'
171  if local:
172  connect = os.path.abspath(self.options.inputCalibDB)
173  else:
174  connect = os.path.basename(self.options.inputCalibDB)
175  self.addPoolDBESSource( process = self.process,
176  moduleName = 'calibDB',
177  record = 'DTTtrigRcd',
178  tag = 'ttrig',
179  connect = str("sqlite_file:%s" % connect),
180  label = label
181  )
182  self.input_files.append( os.path.abspath(self.options.inputCalibDB) )
str
#define str(s)
Definition: TestProcessor.cc:54
def DTWorkflow.DTWorkflow.add_local_custom_db (   self)

Definition at line 183 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, validateAlignments.ValidationJobMultiIOV.options, DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction().

183 
184  def add_local_custom_db(self):
185  for option in ('inputDBRcd', 'connectStrDBTag'):
186  if hasattr(self.options, option) and not getattr(self.options, option):
187  raise ValueError("Option %s needed for custom input db" % option)
188  self.addPoolDBESSource( process = self.process,
189  record = self.options.inputDBRcd,
190  tag = self.options.inputDBTag,
191  connect = self.options.connectStrDBTag,
192  moduleName = 'customDB%s' % self.options.inputDBRcd
193  )
def DTWorkflow.DTWorkflow.add_local_t0_db (   self,
  local = False 
) 
Add a local t0 database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 133 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

133 
134  def add_local_t0_db(self, local=False):
135  """ Add a local t0 database as input. Use the option local is used
136  if the pset is processed locally and not with crab.
137  """
138  if local:
139  connect = os.path.abspath(self.options.inputT0DB)
140  else:
141  connect = os.path.basename(self.options.inputT0DB)
142  self.addPoolDBESSource( process = self.process,
143  moduleName = 't0DB',
144  record = 'DTT0Rcd',
145  tag = 't0',
146  connect = 'sqlite_file:%s' % connect)
147  self.input_files.append(os.path.abspath(self.options.inputT0DB))
def DTWorkflow.DTWorkflow.add_local_vdrift_db (   self,
  local = False 
) 
Add a local vdrift database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 148 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), and DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write().

148 
149  def add_local_vdrift_db(self, local=False):
150  """ Add a local vdrift database as input. Use the option local is used
151  if the pset is processed locally and not with crab.
152  """
153  if local:
154  connect = os.path.abspath(self.options.inputVDriftDB)
155  else:
156  connect = os.path.basename(self.options.inputVDriftDB)
157  self.addPoolDBESSource( process = self.process,
158  moduleName = 'vDriftDB',
159  record = 'DTMtimeRcd',
160  tag = 'vDrift',
161  connect = 'sqlite_file:%s' % connect)
162  self.input_files.append( os.path.abspath(self.options.inputVDriftDB) )
def DTWorkflow.DTWorkflow.add_preselection (   self) 
Add preselection to the process object stored in workflow_object

Definition at line 120 of file DTWorkflow.py.

References tools.prependPaths(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

120 
121  def add_preselection(self):
122  """ Add preselection to the process object stored in workflow_object"""
123  if not hasattr(self, "process"):
124  raise NameError("Process is not initalized in workflow object")
125  pathsequence = self.options.preselection.split(':')[0]
126  seqname = self.options.preselection.split(':')[1]
127  self.process.load(pathsequence)
128  tools.prependPaths(self.process, seqname)
tools.prependPaths
def prependPaths
Definition: tools.py:66
def DTWorkflow.DTWorkflow.add_raw_option (   self)

Definition at line 129 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.digilabel, tools.prependPaths(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

129 
130  def add_raw_option(self):
131  getattr(self.process, self.digilabel).inputLabel = 'rawDataCollector'
132  tools.prependPaths(self.process,self.digilabel)
tools.prependPaths
def prependPaths
Definition: tools.py:66
def DTWorkflow.DTWorkflow.addPoolDBESSource (   process,
  moduleName,
  record,
  tag,
  connect = 'sqlite_file:',
  label = '' 
)
static

Definition at line 248 of file DTWorkflow.py.

References str.

Referenced by DTWorkflow.DTWorkflow.add_local_calib_db(), DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), and DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit().

248 
249  label='',):
250 
251  from CondCore.CondDB.CondDB_cfi import CondDB
252 
253  calibDB = cms.ESSource("PoolDBESSource",
254  CondDB,
255  timetype = cms.string('runnumber'),
256  toGet = cms.VPSet(cms.PSet(
257  record = cms.string(record),
258  tag = cms.string(tag),
259  label = cms.untracked.string(label)
260  )),
261  )
262  calibDB.connect = cms.string( str(connect) )
263  #if authPath: calibDB.DBParameters.authenticationPath = authPath
264  if 'oracle:' in connect:
265  calibDB.DBParameters.authenticationPath = '/afs/cern.ch/cms/DB/conddb'
266  setattr(process,moduleName,calibDB)
267  setattr(process,"es_prefer_" + moduleName,cms.ESPrefer('PoolDBESSource',
268  moduleName)
269  )
CondDB_cfi
str
#define str(s)
Definition: TestProcessor.cc:54
def DTWorkflow.DTWorkflow.all (   self) 
generalized function to perform several workflow mode commands in chain.
    All commands mus be specified in self.all_commands list in workflow mode specific
    prepare function in child workflow objects.

Definition at line 94 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.all_commands, DTVdriftWorkflow.DTvdriftWorkflow.all_commands, DTTtrigWorkflow.DTttrigWorkflow.all_commands, splitter.FileObj.run, uploader.FileObj.run, uploadPayloads.FileObj.run, RPCRunIOV::RunIOV_Item.run, beamspot::BeamSpotContainer.run, TB06RecoH2.run, RPCDQMObject.run, TB06Reco.run, MultiEventFilter::Event.run, DTCalibrationWorker.DTCalibrationWorker.run(), L1Analysis::L1AnalysisEventDataFormat.run, TrackInfoProducerAlgorithm.run(), SurveyAlignmentAlgorithm.run(), EfficiencyPlotter.run, lumi::Lumi2DB::LumiSource.run, generateEDF.LumiInfo.run, WZInterestingEventSelector::event.run, MuonRecoTest.run, crabWrapper.CrabWrapper.run(), MatacqProducer::MatacqEventId.run, DTResolutionAnalysisTest.run, options.ConnectionHLTMenu.run, Point.run, HIPAlignmentAlgorithm.run(), DTWorkflow.DTWorkflow.run(), BeamSpotRcdReader::theBSfromDB.run, DTDataIntegrityTest.run, BeamSpotOnlineRecordsReader::theBSOfromDB.run, DTBlockedROChannelsTest.run, CSCOverlapsAlignmentAlgorithm.run(), DTChamberEfficiencyTest.run, DTResolutionTest.run, ConstantStepOdeSolver.run(), DTEfficiencyTest.run, ME::Header.run, ApeSettingAlgorithm.run(), MuonAlignmentFromReference.run(), DQMNet::CoreObject.run, personalPlayback.Playback.run, MillePedeAlignmentAlgorithm.run(), DTLocalTriggerBaseTest.run, MatrixUtil.InputInfo.run, EcalDeadCellDeltaRFilter.run, QIE8Simulator.run(), HitEff.run, cond::RunInfo_t.run, AlignmentAlgorithmBase.run(), SiPixelErrorEstimation.run, personalPlayback.FrameworkJob.run, SiPixelTrackingRecHitsValid.run, dqm_interfaces.DirWalkerFile.run, JsonOutputProducer::JsonEvent.run, SummaryOutputProducer.run, SummaryOutputProducer::Event.run, and DTWorkflow.DTWorkflow.run_all_command.

94 
95  def all(self):
96  """ generalized function to perform several workflow mode commands in chain.
97  All commands mus be specified in self.all_commands list in workflow mode specific
98  prepare function in child workflow objects.
99  """
100  self.run_all_command = True
101  for command in self.all_commands:
102  self.options.command = command
103  self.run()
def DTWorkflow.DTWorkflow.check (   self) 
Function to check status of submitted tasks

Definition at line 107 of file DTWorkflow.py.

References CrabHelper.CrabHelper.check_crabtask().

107 
108  def check(self):
109  """ Function to check status of submitted tasks """
110  self.check_crabtask()
def DTWorkflow.DTWorkflow.check_missing_options (   self,
  requirements_dict 
)

Definition at line 43 of file DTWorkflow.py.

References join(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.run().

43 
44  def check_missing_options(self, requirements_dict):
45  missing_options = []
46  # check if all required options exist
47  if self.options.command in requirements_dict:
48  for option in requirements_dict[self.options.command]:
49  if not (hasattr(self.options, option)
50  and ( (getattr(self.options,option))
51  or isinstance(getattr(self.options,option), bool) )):
52  missing_options.append(option)
53  if len(missing_options) > 0:
54  err = "The following CLI options are missing"
55  err += " for command %s: " % self.options.command
56  err += " ".join(missing_options)
57  raise ValueError(err)
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def DTWorkflow.DTWorkflow.correction (   self)

Definition at line 117 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

117 
118  def correction(self):
119  self.runCMSSWtask()
def DTWorkflow.DTWorkflow.dump (   self)

Definition at line 114 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

114 
115  def dump(self):
116  self.runCMSSWtask()
def DTWorkflow.DTWorkflow.dump_options (   self)

Definition at line 391 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.get_config_name(), DTWorkflow.DTWorkflow.local_path(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.run().

391 
392  def dump_options(self):
393  with open(os.path.join(self.local_path, self.get_config_name()),"w") as out_file:
394  json.dump(vars(self.options), out_file, indent=4)
def DTWorkflow.DTWorkflow.get_config_name (   self,
  command = "" 
) 
Create the name for the output json file which will be dumped

Definition at line 385 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.dump_options(), and DTWorkflow.DTWorkflow.load_options_command().

385 
386  def get_config_name(self, command= ""):
387  """ Create the name for the output json file which will be dumped"""
388  if not command:
389  command = self.options.command
390  return "config_" + command + ".json"
def DTWorkflow.DTWorkflow.get_output_files (   self,
  crabtask,
  output_path 
)

Definition at line 270 of file DTWorkflow.py.

Referenced by CrabHelper.CrabHelper.check_crabtask(), and DTWorkflow.DTWorkflow.prepare_common_write().

270 
271  def get_output_files(self, crabtask, output_path):
272  self.crab.callCrabCommand( ["getoutput",
273  "--outputpath",
274  output_path,
275  crabtask.crabFolder ] )
def DTWorkflow.DTWorkflow.load_options (   self,
  config_file_path 
)

Definition at line 395 of file DTWorkflow.py.

References DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.load_options_command(), and DTWorkflow.DTWorkflow.run().

395 
396  def load_options(self, config_file_path):
397  if not os.path.exists(config_file_path):
398  raise IOError("File %s not found" % config_file_path)
399  with open(config_file_path, "r") as input_file:
400  config_json = json.load(input_file)
401  for key, val in config_json.items():
402  if not hasattr(self.options, key) or not getattr(self.options, key):
403  setattr(self.options, key, val)
def DTWorkflow.DTWorkflow.load_options_command (   self,
  command 
) 
Load options for previous command in workflow

Definition at line 404 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.get_config_name(), DTWorkflow.DTWorkflow.load_options(), and DTWorkflow.DTWorkflow.local_path().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_check(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_check(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_check(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_check(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_check().

404 
405  def load_options_command(self, command ):
406  """Load options for previous command in workflow """
407  if not self.options.config_path:
408  if not self.options.run:
409  raise RuntimeError("Option run is required if no config path specified")
410  if not os.path.exists(self.local_path):
411  raise IOError("Local path %s does not exist" % self.local_path)
412  self.options.config_path = os.path.join(self.local_path,
413  self.get_config_name(command))
414  self.load_options( self.options.config_path )
415 
def DTWorkflow.DTWorkflow.local_path (   self) 
Output path on local machine

Definition at line 334 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_command_tag, DTVdriftWorkflow.DTvdriftWorkflow.outpath_command_tag, DTTtrigWorkflow.DTttrigWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_command_tag, and DTWorkflow.DTWorkflow.outpath_workflow_mode_tag().

Referenced by CrabHelper.CrabHelper.check_crabtask(), CrabHelper.CrabHelper.crab_config_filepath(), DTWorkflow.DTWorkflow.dump_options(), DTWorkflow.DTWorkflow.load_options_command(), DTWorkflow.DTWorkflow.prepare_common_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write(), DTWorkflow.DTWorkflow.pset_path(), DTWorkflow.DTWorkflow.result_path(), DTWorkflow.DTWorkflow.run(), CrabHelper.CrabHelper.write_crabConfig(), and DTWorkflow.DTWorkflow.write_pset_file().

334 
335  def local_path(self):
336  """ Output path on local machine """
337  if self.options.run and self.options.label:
338  prefix = "Run%d-%s_v%d" % ( self.options.run,
339  self.options.label,
340  self.options.trial)
341  else:
342  prefix = ""
343  if self.outpath_workflow_mode_tag:
344  path = os.path.join( self.options.working_dir,
345  prefix,
346  self.outpath_workflow_mode_tag)
347  else:
348  path = os.path.join( self.options.working_dir,
349  prefix,
350  self.outpath_command_tag )
351  return path
def DTWorkflow.DTWorkflow.outpath_workflow_mode_tag (   self)

Definition at line 314 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_workflow_mode_dict, DTVdriftWorkflow.DTvdriftWorkflow.outpath_workflow_mode_dict, and DTTtrigWorkflow.DTttrigWorkflow.outpath_workflow_mode_dict.

Referenced by DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.remote_out_path().

314 
315  def outpath_workflow_mode_tag(self):
316  if not self.options.workflow_mode in self.outpath_workflow_mode_dict:
317  raise NotImplementedError("%s missing in outpath_workflow_mode_dict" % self.options.workflow_mode)
318  return self.outpath_workflow_mode_dict[self.options.workflow_mode]
def DTWorkflow.DTWorkflow.prepare_common_dump (   self,
  db_path 
)

Definition at line 229 of file DTWorkflow.py.

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump().

229 
230  def prepare_common_dump(self, db_path):
231  self.process = tools.loadCmsProcess(self.pset_template)
232  self.process.calibDB.connect = 'sqlite_file:%s' % db_path
233  try:
234  path = self.result_path
235  except:
236  path = os.getcwd()
237  print("path", path)
238  out_path = os.path.abspath(os.path.join(path,
239  os.path.splitext(db_path)[0] + ".txt"))
240 
241  self.process.dumpToFile.outputFileName = out_path
tools.loadCmsProcess
def loadCmsProcess
Definition: tools.py:56
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def DTWorkflow.DTWorkflow.prepare_common_submit (   self) 
Common operations used in most prepare_[workflow_mode]_submit functions

Definition at line 194 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), DTWorkflow.DTWorkflow.add_preselection(), DTWorkflow.DTWorkflow.add_raw_option(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit().

194 
195  def prepare_common_submit(self):
196  """ Common operations used in most prepare_[workflow_mode]_submit functions"""
197  if not self.options.run:
198  raise ValueError("Option run is required for submission!")
199  if hasattr(self.options, "inputT0DB") and self.options.inputT0DB:
200  self.add_local_t0_db()
201 
202  if hasattr(self.options, "inputVDriftDB") and self.options.inputVDriftDB:
203  self.add_local_vdrift_db()
204 
205  if hasattr(self.options, "inputDBTag") and self.options.inputDBTag:
206  self.add_local_custom_db()
207 
208  if self.options.run_on_RAW:
209  self.add_raw_option()
210  if self.options.preselection:
211  self.add_preselection()
def DTWorkflow.DTWorkflow.prepare_common_write (   self,
  do_hadd = True 
) 
Common operations used in most prepare_[workflow_mode]_erite functions

Definition at line 212 of file DTWorkflow.py.

References CrabHelper.CrabHelper.crab_config_filepath(), DTWorkflow.DTWorkflow.files_reveived, DTWorkflow.DTWorkflow.get_output_files(), tools.haddLocal(), DTWorkflow.DTWorkflow.load_options_command(), DTWorkflow.DTWorkflow.local_path(), mergeLHE.BaseLHEMerger.output_file, DTT0WireWorkflow.DTT0WireWorkflow.output_file, DTVdriftWorkflow.DTvdriftWorkflow.output_file, DTTtrigWorkflow.DTttrigWorkflow.output_file, HltDiff.output_file, and DTWorkflow.DTWorkflow.result_path().

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

212 
213  def prepare_common_write(self, do_hadd=True):
214  """ Common operations used in most prepare_[workflow_mode]_erite functions"""
215  self.load_options_command("submit")
216  output_path = os.path.join( self.local_path, "unmerged_results" )
217  merged_file = os.path.join(self.result_path, self.output_file)
218  crabtask = self.crabFunctions.CrabTask(crab_config = self.crab_config_filepath,
219  initUpdate = False)
220  if not (self.options.skip_stageout or self.files_reveived or self.options.no_exec):
221  self.get_output_files(crabtask, output_path)
222  log.info("Received files from storage element")
223  log.info("Using hadd to merge output files")
224  if not self.options.no_exec and do_hadd:
225  returncode = tools.haddLocal(output_path, merged_file)
226  if returncode != 0:
227  raise RuntimeError("Failed to merge files with hadd")
228  return crabtask.crabConfig.Data.outputDatasetTag
tools.haddLocal
def haddLocal
Definition: tools.py:41
def DTWorkflow.DTWorkflow.prepare_workflow (   self) 
Abstract implementation of prepare workflow function

Definition at line 88 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.run().

88 
89  def prepare_workflow(self):
90  """ Abstract implementation of prepare workflow function"""
91  errmsg = "Class `{}` does not implement `{}`"
92  raise NotImplementedError( errmsg.format(self.__class__.__name__,
93  "prepare_workflow"))
def DTWorkflow.DTWorkflow.pset_path (   self) 
full path to the pset file

Definition at line 370 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path(), DTWorkflow.DTWorkflow.pset_name, DTT0WireWorkflow.DTT0WireWorkflow.pset_name, DTVdriftWorkflow.DTvdriftWorkflow.pset_name, and DTTtrigWorkflow.DTttrigWorkflow.pset_name.

Referenced by DTWorkflow.DTWorkflow.runCMSSWtask(), and DTWorkflow.DTWorkflow.write_pset_file().

370 
371  def pset_path(self):
372  """ full path to the pset file """
373  basepath = os.path.join( self.local_path, "psets")
374  if not os.path.exists( basepath ):
375  os.makedirs( basepath )
376  return os.path.join( basepath, self.pset_name )
def DTWorkflow.DTWorkflow.pset_template_base_bath (   self) 
Base path to folder containing pset files for cmsRun

Definition at line 360 of file DTWorkflow.py.

360 
361  def pset_template_base_bath(self):
362  """ Base path to folder containing pset files for cmsRun"""
363  return os.path.expandvars(os.path.join("$CMSSW_BASE",
364  "src",
365  "CalibMuon",
366  "test",
367  )
368  )
def DTWorkflow.DTWorkflow.remote_out_path (   self) 
Output path on remote excluding user base path
Returns a dict if crab is used due to crab path setting policy

Definition at line 292 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_command_tag, DTVdriftWorkflow.DTvdriftWorkflow.outpath_command_tag, DTTtrigWorkflow.DTttrigWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_workflow_mode_tag(), str, FWLite.ElectronMVAID.tag, tnp::TagProbePair.tag, Entry< T >.tag, MeasurementEstimator::OpaquePayload.tag, cond::persistency::IOVEditorData.tag, Inspector.Inspector.tag, BeamSpotFakeConditions.tag, TrackListMerger::TkEDGetTokenss.tag, FWLite.WorkingPoints.tag, MuIsolatorResultProducer< BT >::DepositConf.tag, DQMNet::CoreObject.tag, personalPlayback.FrameworkJob.tag, DTWorkflow.DTWorkflow.tag(), EcalTPGParamReaderFromDB.user, dataset.BaseDataset.user, production_tasks.Task.user, popcon::RpcDataV.user, popcon::RpcDataT.user, popcon::RPCObPVSSmapData.user, popcon::RpcObGasData.user, popcon::RpcDataI.user, popcon::RpcDataS.user, popcon::RpcDataUXC.user, popcon::RpcDataGasMix.user, popcon::RpcDataFebmap.user, EcalDBConnection.user, production_tasks.ParseOptions.user, MatrixInjector.MatrixInjector.user, production_tasks.FindOnCastor.user, production_tasks.CheckForMask.user, production_tasks.CheckForWrite.user, and DTWorkflow.DTWorkflow.user().

292 
293  def remote_out_path(self):
294  """ Output path on remote excluding user base path
295  Returns a dict if crab is used due to crab path setting policy"""
296  if self.options.command =="submit":
297  return {
298  "outLFNDirBase" : os.path.join( "/store",
299  "user",
300  self.user,
301  'DTCalibration/',
302  self.outpath_command_tag,
303  self.outpath_workflow_mode_tag),
304  "outputDatasetTag" : self.tag
305  }
306  else:
307  return os.path.join( 'DTCalibration/',
308  datasetstr,
309  'Run' + str(self.options.run),
310  self.outpath_command_tag,
311  self.outpath_workflow_mode_tag,
312  'v' + str(self.options.trial),
)
str
#define str(s)
Definition: TestProcessor.cc:54
def DTWorkflow.DTWorkflow.result_path (   self)

Definition at line 353 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

353 
354  def result_path(self):
355  result_path = os.path.abspath(os.path.join(self.local_path,"results"))
356  if not os.path.exists(result_path):
357  os.makedirs(result_path)
358  return result_path
def DTWorkflow.DTWorkflow.run (   self) 
Generalized function to run workflow command

Definition at line 58 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.check_missing_options(), DTWorkflow.DTWorkflow.dump_options(), DTWorkflow.DTWorkflow.load_options(), DTWorkflow.DTWorkflow.local_path(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, validateAlignments.ValidationJobMultiIOV.options, DTT0WireWorkflow.DTT0WireWorkflow.prepare_workflow(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_workflow(), DTTtrigWorkflow.DTttrigWorkflow.prepare_workflow(), DTWorkflow.DTWorkflow.prepare_workflow(), DTWorkflow.DTWorkflow.required_options_dict, and DTWorkflow.DTWorkflow.required_options_prepare_dict.

Referenced by DTWorkflow.DTWorkflow.all(), Types.EventID.cppID(), Types.LuminosityBlockID.cppID(), and o2olib.O2OTool.execute().

58 
59  def run(self):
60  """ Generalized function to run workflow command"""
61  msg = "Preparing %s workflow" % self.options.workflow
62  if hasattr(self.options, "command"):
63  msg += " for command %s" % self.options.command
64  log.info(msg)
65  if self.options.config_path:
66  self.load_options( self.options.config_path )
67  #check if all options to prepare the command are used
68  self.check_missing_options(self.required_options_prepare_dict)
69  self.prepare_workflow()
70  # create output folder if they do not exist yet
71  if not os.path.exists( self.local_path ):
72  os.makedirs(self.local_path)
73  # dump used options
74  self.dump_options()
75  #check if all options to run the command are used
76  self.check_missing_options(self.required_options_dict)
77  try:
78  run_function = getattr(self, self.options.command)
79  except AttributeError:
80  errmsg = "Class `{}` does not implement `{}` for workflow %s" % self.options.workflow
81  if hasattr(self.options, "workflow_mode"):
82  errmsg += "and workflow mode %s" % self.options.workflow_mode
83  raise NotImplementedError( errmsg.format(self.__class__.__name__,
84  self.options.command))
85  log.debug("Running command %s" % self.options.command)
86  # call chosen function
87  run_function()
def DTWorkflow.DTWorkflow.runCMSSWtask (   self,
  pset_path = "" 
) 
Run a cmsRun job locally. The member variable self.pset_path is used
    if pset_path argument is not given

Definition at line 276 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.pset_name, DTT0WireWorkflow.DTT0WireWorkflow.pset_name, DTVdriftWorkflow.DTvdriftWorkflow.pset_name, DTTtrigWorkflow.DTttrigWorkflow.pset_name, and DTWorkflow.DTWorkflow.pset_path().

Referenced by DTWorkflow.DTWorkflow.correction(), DTWorkflow.DTWorkflow.dump(), DTT0WireWorkflow.DTT0WireWorkflow.submit(), and DTWorkflow.DTWorkflow.write().

276 
277  def runCMSSWtask(self, pset_path=""):
278  """ Run a cmsRun job locally. The member variable self.pset_path is used
279  if pset_path argument is not given"""
280  if self.options.no_exec:
281  return 0
282  process = subprocess.Popen( "cmsRun %s" % self.pset_path,
283  stdout=subprocess.PIPE,
284  stderr=subprocess.STDOUT,
285  shell = True)
286  stdout = process.communicate()[0]
287  log.info(stdout)
288  if process.returncode != 0:
289  raise RuntimeError("Failed to use cmsRun for pset %s" % self.pset_name)
290  return process.returncode
def DTWorkflow.DTWorkflow.submit (   self)

Definition at line 104 of file DTWorkflow.py.

References CrabHelper.CrabHelper.submit_crab_task().

104 
105  def submit(self):
106  self.submit_crab_task()
def DTWorkflow.DTWorkflow.tag (   self)

Definition at line 320 of file DTWorkflow.py.

References str.

Referenced by DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), and DTWorkflow.DTWorkflow.remote_out_path().

320 
321  def tag(self):
322  return 'Run' + str(self.options.run) + '_v' + str(self.options.trial)
str
#define str(s)
Definition: TestProcessor.cc:54
def DTWorkflow.DTWorkflow.user (   self)

Definition at line 324 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow._user, DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.remote_out_path().

324 
325  def user(self):
326  if self._user:
327  return self._user
328  if hasattr(self.options, "user") and self.options.user:
329  self._user = self.options.user
330  else:
331  self._user = self.crab.checkusername()
332  return self._user
def DTWorkflow.DTWorkflow.write (   self)

Definition at line 111 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

Referenced by pkg.AbstractPkg.generate(), and querying.connection.write_and_commit().

111 
112  def write(self):
113  self.runCMSSWtask()
def DTWorkflow.DTWorkflow.write_pset_file (   self)

Definition at line 377 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.pset_path().

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

377 
378  def write_pset_file(self):
379  if not hasattr(self, "process"):
380  raise NameError("Process is not initalized in workflow object")
381  if not os.path.exists(self.local_path):
382  os.makedirs(self.local_path)
383  with open( self.pset_path,'w') as pfile:
384  pfile.write(self.process.dumpPython())

Member Data Documentation

DTWorkflow.DTWorkflow._user
private

Definition at line 39 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.user().

DTWorkflow.DTWorkflow.digilabel

Definition at line 24 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.add_raw_option().

DTWorkflow.DTWorkflow.files_reveived

Definition at line 38 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.prepare_common_write().

DTWorkflow.DTWorkflow.input_files

Definition at line 35 of file DTWorkflow.py.

Referenced by mergeLHE.MG5LOLHEMerger.merge(), mergeLHE.ExternalCppLHEMerger.merge(), and mergeLHE.DefaultLHEMerger.merge_init_blocks().

DTWorkflow.DTWorkflow.options

Definition at line 22 of file DTWorkflow.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), submitPVValidationJobs.BetterConfigParser.__updateDict(), DTWorkflow.DTWorkflow.add_local_custom_db(), confdb.HLTProcess.buildOptions(), DTWorkflow.DTWorkflow.check_missing_options(), betterConfigParser.BetterConfigParser.checkInput(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_taskname(), DTWorkflow.DTWorkflow.dump_options(), CrabHelper.CrabHelper.fill_options_from_crab_config(), confdb.HLTProcess.fixPrescales(), DTWorkflow.DTWorkflow.load_options(), DTWorkflow.DTWorkflow.prepare_common_submit(), edmIntegrityCheck.IntegrityCheck.query(), cmsswPreprocessor.CmsswPreprocessor.run(), DTWorkflow.DTWorkflow.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.CleanJobFiles.run(), and DTWorkflow.DTWorkflow.user().

DTWorkflow.DTWorkflow.outpath_command_tag

Definition at line 33 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.remote_out_path().

DTWorkflow.DTWorkflow.output_files

Definition at line 34 of file DTWorkflow.py.

DTWorkflow.DTWorkflow.process

Definition at line 230 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.add_local_calib_db(), DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), DTWorkflow.DTWorkflow.add_preselection(), DTWorkflow.DTWorkflow.add_raw_option(), ConfigBuilder.ConfigBuilder.addCustomise(), ConfigBuilder.ConfigBuilder.addOutput(), ConfigBuilder.ConfigBuilder.completeInputCommand(), adaptToRunAtMiniAOD.adaptToRunAtMiniAOD.convertModuleToMiniAODInput(), ConfigBuilder.ConfigBuilder.prepare(), ConfigBuilder.ConfigBuilder.prepare_DQM(), ConfigBuilder.ConfigBuilder.prepare_GEN(), ConfigBuilder.ConfigBuilder.prepare_HLT(), ConfigBuilder.ConfigBuilder.prepare_PATFILTER(), ConfigBuilder.ConfigBuilder.prepare_VALIDATION(), runTauIdMVA.TauIDEmbedder.processDeepProducer(), ConfigBuilder.ConfigBuilder.renameHLTprocessInSequence(), ConfigBuilder.ConfigBuilder.renameInputTagsInSequence(), runTauIdMVA.TauIDEmbedder.runTauID(), ConfigBuilder.ConfigBuilder.scheduleSequence(), and runTauIdMVA.TauIDEmbedder.tauIDMVAinputs().

DTWorkflow.DTWorkflow.pset_name

Definition at line 32 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.pset_path(), and DTWorkflow.DTWorkflow.runCMSSWtask().

DTWorkflow.DTWorkflow.required_options_dict

Definition at line 27 of file DTWorkflow.py.

Referenced by CLIHelper.CLIHelper.fill_required_options_dict(), and DTWorkflow.DTWorkflow.run().

DTWorkflow.DTWorkflow.required_options_prepare_dict

Definition at line 28 of file DTWorkflow.py.

Referenced by CLIHelper.CLIHelper.fill_required_options_prepare_dict(), and DTWorkflow.DTWorkflow.run().

DTWorkflow.DTWorkflow.run_all_command

Definition at line 37 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.all().

Generated for CMSSW Reference Manual by doxygen 1.8.5