DTWorkflow.DTWorkflow Class Reference

Inheritance diagram for DTWorkflow.DTWorkflow:

Public Member Functions
def	__init__ (self, options)
def	add_local_calib_db (self, local=False)
def	add_local_custom_db (self)
def	add_local_t0_db (self, local=False)
def	add_local_vdrift_db (self, local=False)
def	add_preselection (self)
def	add_raw_option (self)
def	all (self)
def	check (self)
def	check_missing_options (self, requirements_dict)
def	correction (self)
def	dump (self)
def	dump_options (self)
def	get_config_name (self, command="")
def	get_output_files (self, crabtask, output_path)
def	load_options (self, config_file_path)
def	load_options_command (self, command)
def	local_path (self)
def	outpath_workflow_mode_tag (self)
def	prepare_common_dump (self, db_path)
def	prepare_common_submit (self)
def	prepare_common_write (self, do_hadd=True)
def	prepare_workflow (self)
def	pset_path (self)
def	pset_template_base_bath (self)
def	remote_out_path (self)
def	result_path (self)
def	run (self)
def	runCMSSWtask (self, pset_path="")
def	submit (self)
def	tag (self)
def	user (self)
def	write (self)
def	write_pset_file (self)
Public Member Functions inherited from CLIHelper.CLIHelper
def	add_parser_options (cls, parser)
def	fill_required_options_dict (self)
def	fill_required_options_prepare_dict (self)
def	get_check_options_parser (cls)
def	get_common_options_parser (cls)
def	get_dump_options_parser (cls)
def	get_input_db_options_parser (cls)
def	get_local_input_db_options_parser (cls)
def	get_submission_options_parser (cls)
def	get_write_options_parser (cls)
Public Member Functions inherited from CrabHelper.CrabHelper
def	__init__ (self)
def	cert_info (self)
def	check_crabtask (self)
def	crab (self)
def	crab_config_filename (self)
def	crab_config_filepath (self)
def	crab_taskname (self)
def	create_crab_config (self)
def	fill_options_from_crab_config (self)
def	submit_crab_task (self)
def	voms_proxy_create (self, passphrase=None)
def	voms_proxy_time_left (self)
def	write_crabConfig (self)

Static Public Member Functions
def	addPoolDBESSource (process, moduleName, record, tag, connect='sqlite_file:', label='')

Public Attributes
	digilabel
	files_reveived
	input_files
	options
	outpath_command_tag
	output_files
	process
	pset_name
	required_options_dict
	required_options_prepare_dict
	run_all_command
Public Attributes inherited from CrabHelper.CrabHelper
	crab_config
	crabFunctions

Private Attributes
	_user

Detailed Description

This is the base class for all DTWorkflows and contains some
    common tasks 

Definition at line 18 of file DTWorkflow.py.

Constructor & Destructor Documentation

def DTWorkflow.DTWorkflow.__init__	(		self,
			options
	)

Definition at line 21 of file DTWorkflow.py.

    def __init__(self, options):
        self.options = options
        super( DTWorkflow, self ).__init__()
        self.digilabel = "muonDTDigis"
        # dict to hold required variables. Can not be marked in argparse to allow
        # loading of options from config
        self.required_options_dict = {}
        self.required_options_prepare_dict = {}
        self.fill_required_options_dict()
        self.fill_required_options_prepare_dict()
        # These variables are determined in the derived classes
        self.pset_name = ""
        self.outpath_command_tag = ""
        self.output_files = []
        self.input_files = []

        self.run_all_command = False
        self.files_reveived = False
        self._user = ""
        # change to working directory
        os.chdir(self.options.working_dir)

Member Function Documentation

def DTWorkflow.DTWorkflow.add_local_calib_db	(		self,
			local = False
	)

Add a local calib database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 163 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, DTWorkflow.DTWorkflow.process, and str.

Referenced by DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit().

    def add_local_calib_db(self, local=False):
        """ Add a local calib database as input. Use the option local is used
            if the pset is processed locally and not with crab.
         """
        label = ''
        if self.options.datasettype == "Cosmics":
            label = 'cosmics'
        if local:
            connect = os.path.abspath(self.options.inputCalibDB)
        else:
            connect = os.path.basename(self.options.inputCalibDB)
        self.addPoolDBESSource( process = self.process,
                                moduleName = 'calibDB',
                                record = 'DTTtrigRcd',
                                tag = 'ttrig',
                                connect = str("sqlite_file:%s" % connect),
                                label = label
                                )
        self.input_files.append( os.path.abspath(self.options.inputCalibDB) )

def DTWorkflow.DTWorkflow.add_local_custom_db

(

self

)

Definition at line 183 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, validateAlignments.ValidationJobMultiIOV.options, DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction().

    def add_local_custom_db(self):
        for option in ('inputDBRcd', 'connectStrDBTag'):
            if hasattr(self.options, option) and not getattr(self.options, option):
                raise ValueError("Option %s needed for custom input db" % option)
        self.addPoolDBESSource( process = self.process,
                                    record = self.options.inputDBRcd,
                                    tag = self.options.inputDBTag,
                                    connect = self.options.connectStrDBTag,
                                    moduleName = 'customDB%s' % self.options.inputDBRcd
                                   )

def DTWorkflow.DTWorkflow.add_local_t0_db	(		self,
			local = False
	)

Add a local t0 database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 133 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

    def add_local_t0_db(self, local=False):
        """ Add a local t0 database as input. Use the option local is used
            if the pset is processed locally and not with crab.
        """
        if local:
            connect = os.path.abspath(self.options.inputT0DB)
        else:
            connect = os.path.basename(self.options.inputT0DB)
        self.addPoolDBESSource( process = self.process,
                                moduleName = 't0DB',
                                record = 'DTT0Rcd',
                                tag = 't0',
                                connect =  'sqlite_file:%s' % connect)
        self.input_files.append(os.path.abspath(self.options.inputT0DB))

def DTWorkflow.DTWorkflow.add_local_vdrift_db	(		self,
			local = False
	)

Add a local vdrift database as input. Use the option local is used
    if the pset is processed locally and not with crab.

Definition at line 148 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.addPoolDBESSource(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), and DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write().

    def add_local_vdrift_db(self, local=False):
        """ Add a local vdrift database as input. Use the option local is used
            if the pset is processed locally and not with crab.
         """
        if local:
            connect = os.path.abspath(self.options.inputVDriftDB)
        else:
            connect = os.path.basename(self.options.inputVDriftDB)
        self.addPoolDBESSource( process = self.process,
                                moduleName = 'vDriftDB',
                                record = 'DTMtimeRcd',
                                tag = 'vDrift',
                                connect = 'sqlite_file:%s' % connect)
        self.input_files.append( os.path.abspath(self.options.inputVDriftDB) )

def DTWorkflow.DTWorkflow.add_preselection

(

self

)

Add preselection to the process object stored in workflow_object

Definition at line 120 of file DTWorkflow.py.

References tools.prependPaths(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

    def add_preselection(self):
        """ Add preselection to the process object stored in workflow_object"""
        if not hasattr(self, "process"):
            raise NameError("Process is not initalized in workflow object")
        pathsequence = self.options.preselection.split(':')[0]
        seqname = self.options.preselection.split(':')[1]
        self.process.load(pathsequence)
        tools.prependPaths(self.process, seqname)

def DTWorkflow.DTWorkflow.add_raw_option

(

self

)

Definition at line 129 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.digilabel, tools.prependPaths(), DTT0WireWorkflow.DTT0WireWorkflow.process, DTVdriftWorkflow.DTvdriftWorkflow.process, DTTtrigWorkflow.DTttrigWorkflow.process, and DTWorkflow.DTWorkflow.process.

Referenced by DTWorkflow.DTWorkflow.prepare_common_submit().

    def add_raw_option(self):
        getattr(self.process, self.digilabel).inputLabel = 'rawDataCollector'
        tools.prependPaths(self.process,self.digilabel)

def DTWorkflow.DTWorkflow.addPoolDBESSource (   process,   moduleName,   record,   tag,   connect = 'sqlite_file:',   label = ''  )

static

Definition at line 248 of file DTWorkflow.py.

References str.

Referenced by DTWorkflow.DTWorkflow.add_local_calib_db(), DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), and DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit().

                           label='',):

        from CondCore.CondDB.CondDB_cfi import CondDB

        calibDB = cms.ESSource("PoolDBESSource",
                               CondDB,
                               timetype = cms.string('runnumber'),
                               toGet = cms.VPSet(cms.PSet(
                                   record = cms.string(record),
                                   tag = cms.string(tag),
                                   label = cms.untracked.string(label)
                                    )),
                               )
        calibDB.connect = cms.string( str(connect) )
        #if authPath: calibDB.DBParameters.authenticationPath = authPath
        if 'oracle:' in connect:
            calibDB.DBParameters.authenticationPath = '/afs/cern.ch/cms/DB/conddb'
        setattr(process,moduleName,calibDB)
        setattr(process,"es_prefer_" + moduleName,cms.ESPrefer('PoolDBESSource',
                                                                moduleName)
                                                                )

def DTWorkflow.DTWorkflow.all

(

self

)

generalized function to perform several workflow mode commands in chain.
    All commands mus be specified in self.all_commands list in workflow mode specific
    prepare function in child workflow objects.

Definition at line 94 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.all_commands, DTVdriftWorkflow.DTvdriftWorkflow.all_commands, DTTtrigWorkflow.DTttrigWorkflow.all_commands, splitter.FileObj.run, uploader.FileObj.run, uploadPayloads.FileObj.run, RPCRunIOV::RunIOV_Item.run, beamspot::BeamSpotContainer.run, TB06RecoH2.run, TB06Reco.run, RPCDQMObject.run, MultiEventFilter::Event.run, DTCalibrationWorker.DTCalibrationWorker.run(), L1Analysis::L1AnalysisEventDataFormat.run, TrackInfoProducerAlgorithm.run(), SurveyAlignmentAlgorithm.run(), lumi::Lumi2DB::LumiSource.run, MuonMillepedeAlgorithm.run(), generateEDF.LumiInfo.run, EfficiencyPlotter.run, WZInterestingEventSelector::event.run, MuonDTLocalMillepedeAlgorithm.run(), crabWrapper.CrabWrapper.run(), HIPAlignmentAlgorithm.run(), Point.run, MuonRecoTest.run, options.ConnectionHLTMenu.run, DTWorkflow.DTWorkflow.run(), DTResolutionAnalysisTest.run, BeamSpotRcdReader::theBSfromDB.run, MatacqProducer::MatacqEventId.run, CSCOverlapsAlignmentAlgorithm.run(), DTBlockedROChannelsTest.run, DTDataIntegrityTest.run, DTChamberEfficiencyTest.run, DTResolutionTest.run, ApeSettingAlgorithm.run(), ME::Header.run, ConstantStepOdeSolver.run(), DTEfficiencyTest.run, MillePedeAlignmentAlgorithm.run(), MuonAlignmentFromReference.run(), personalPlayback.Playback.run, DQMNet::CoreObject.run, MatrixUtil.InputInfo.run, DTLocalTriggerBaseTest.run, HitEff.run, EcalDeadCellDeltaRFilter.run, QIE8Simulator.run(), AlignmentAlgorithmBase.run(), cond::RunInfo_t.run, SiPixelErrorEstimation.run, personalPlayback.FrameworkJob.run, SiPixelTrackingRecHitsValid.run, dqm_interfaces.DirWalkerFile.run, JsonOutputProducer::JsonEvent.run, SummaryOutputProducer.run, SummaryOutputProducer::Event.run, and DTWorkflow.DTWorkflow.run_all_command.

    def all(self):
        """ generalized function to perform several workflow mode commands in chain.
            All commands mus be specified in self.all_commands list in workflow mode specific
            prepare function in child workflow objects.
        """
        self.run_all_command = True
        for command in self.all_commands:
            self.options.command = command
            self.run()

def DTWorkflow.DTWorkflow.check

(

self

)

Function to check status of submitted tasks 

Definition at line 107 of file DTWorkflow.py.

References CrabHelper.CrabHelper.check_crabtask().

    def check(self):
        """ Function to check status of submitted tasks """
        self.check_crabtask()

def DTWorkflow.DTWorkflow.check_missing_options	(		self,
			requirements_dict
	)

Definition at line 43 of file DTWorkflow.py.

References join(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.run().

    def check_missing_options(self, requirements_dict):
        missing_options = []
        # check if all required options exist
        if self.options.command in requirements_dict:
            for option in requirements_dict[self.options.command]:
                if not (hasattr(self.options, option)
                    and ( (getattr(self.options,option))
                          or isinstance(getattr(self.options,option), bool) )):
                    missing_options.append(option)
        if len(missing_options) > 0:
            err = "The following CLI options are missing"
            err += " for command %s: " % self.options.command
            err += " ".join(missing_options)
            raise ValueError(err)

def DTWorkflow.DTWorkflow.correction

(

self

)

Definition at line 117 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

    def correction(self):
        self.runCMSSWtask()

def DTWorkflow.DTWorkflow.dump

(

self

)

Definition at line 114 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

    def dump(self):
        self.runCMSSWtask()

def DTWorkflow.DTWorkflow.dump_options

(

self

)

Definition at line 391 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.get_config_name(), DTWorkflow.DTWorkflow.local_path(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.run().

    def dump_options(self):
        with open(os.path.join(self.local_path, self.get_config_name()),"w") as out_file:
            json.dump(vars(self.options), out_file, indent=4)

def DTWorkflow.DTWorkflow.get_config_name	(		self,
			command = ""
	)

Create the name for the output json file which will be dumped

Definition at line 385 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.dump_options(), and DTWorkflow.DTWorkflow.load_options_command().

    def get_config_name(self, command= ""):
        """ Create the name for the output json file which will be dumped"""
        if not command:
            command = self.options.command
        return "config_" + command + ".json"

def DTWorkflow.DTWorkflow.get_output_files	(		self,
			crabtask,
			output_path
	)

Definition at line 270 of file DTWorkflow.py.

Referenced by CrabHelper.CrabHelper.check_crabtask(), and DTWorkflow.DTWorkflow.prepare_common_write().

    def get_output_files(self, crabtask, output_path):
        self.crab.callCrabCommand( ["getoutput",
                                    "--outputpath",
                                    output_path,
                                    crabtask.crabFolder ] )

def DTWorkflow.DTWorkflow.load_options	(		self,
			config_file_path
	)

Definition at line 395 of file DTWorkflow.py.

References DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.load_options_command(), and DTWorkflow.DTWorkflow.run().

    def load_options(self, config_file_path):
        if not os.path.exists(config_file_path):
            raise IOError("File %s not found" % config_file_path)
        with open(config_file_path, "r") as input_file:
            config_json = json.load(input_file)
            for key, val in config_json.items():
                if not hasattr(self.options, key) or not getattr(self.options, key):
                    setattr(self.options, key, val)

def DTWorkflow.DTWorkflow.load_options_command	(		self,
			command
	)

Load options for previous command in workflow 

Definition at line 404 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.get_config_name(), DTWorkflow.DTWorkflow.load_options(), and DTWorkflow.DTWorkflow.local_path().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_check(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_check(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_check(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_check(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_check().

    def load_options_command(self, command ):
        """Load options for previous command in workflow """
        if not self.options.config_path:
            if not self.options.run:
                raise RuntimeError("Option run is required if no config path specified")
            if not os.path.exists(self.local_path):
                raise IOError("Local path %s does not exist" % self.local_path)
            self.options.config_path = os.path.join(self.local_path,
                                                    self.get_config_name(command))
        self.load_options( self.options.config_path )

def DTWorkflow.DTWorkflow.local_path

(

self

)

Output path on local machine 

Definition at line 334 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_command_tag, DTVdriftWorkflow.DTvdriftWorkflow.outpath_command_tag, DTTtrigWorkflow.DTttrigWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_command_tag, and DTWorkflow.DTWorkflow.outpath_workflow_mode_tag().

Referenced by CrabHelper.CrabHelper.check_crabtask(), CrabHelper.CrabHelper.crab_config_filepath(), DTWorkflow.DTWorkflow.dump_options(), DTWorkflow.DTWorkflow.load_options_command(), DTWorkflow.DTWorkflow.prepare_common_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write(), DTWorkflow.DTWorkflow.pset_path(), DTWorkflow.DTWorkflow.result_path(), DTWorkflow.DTWorkflow.run(), CrabHelper.CrabHelper.write_crabConfig(), and DTWorkflow.DTWorkflow.write_pset_file().

    def local_path(self):
        """ Output path on local machine """
        if self.options.run and self.options.label:
            prefix = "Run%d-%s_v%d" % ( self.options.run,
                                        self.options.label,
                                        self.options.trial)
        else:
            prefix = ""
        if self.outpath_workflow_mode_tag:
            path = os.path.join( self.options.working_dir,
                                 prefix,
                                 self.outpath_workflow_mode_tag)
        else:
            path =  os.path.join( self.options.working_dir,
                                  prefix,
                                  self.outpath_command_tag )
        return path

def DTWorkflow.DTWorkflow.outpath_workflow_mode_tag

(

self

)

Definition at line 314 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_workflow_mode_dict, DTVdriftWorkflow.DTvdriftWorkflow.outpath_workflow_mode_dict, and DTTtrigWorkflow.DTttrigWorkflow.outpath_workflow_mode_dict.

Referenced by DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.remote_out_path().

    def outpath_workflow_mode_tag(self):
        if not self.options.workflow_mode in self.outpath_workflow_mode_dict:
            raise NotImplementedError("%s missing in outpath_workflow_mode_dict" % self.options.workflow_mode)
        return self.outpath_workflow_mode_dict[self.options.workflow_mode]

def DTWorkflow.DTWorkflow.prepare_common_dump	(		self,
			db_path
	)

Definition at line 229 of file DTWorkflow.py.

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump().

    def prepare_common_dump(self, db_path):
        self.process = tools.loadCmsProcess(self.pset_template)
        self.process.calibDB.connect = 'sqlite_file:%s' % db_path
        try:
            path = self.result_path
        except:
            path = os.getcwd()
        print("path", path)
        out_path = os.path.abspath(os.path.join(path,
                                                os.path.splitext(db_path)[0] + ".txt"))

        self.process.dumpToFile.outputFileName = out_path

def DTWorkflow.DTWorkflow.prepare_common_submit

(

self

)

Common operations used in most prepare_[workflow_mode]_submit functions

Definition at line 194 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), DTWorkflow.DTWorkflow.add_preselection(), DTWorkflow.DTWorkflow.add_raw_option(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit().

    def prepare_common_submit(self):
        """ Common operations used in most prepare_[workflow_mode]_submit functions"""
        if not self.options.run:
            raise ValueError("Option run is required for submission!")
        if hasattr(self.options, "inputT0DB") and self.options.inputT0DB:
            self.add_local_t0_db()

        if hasattr(self.options, "inputVDriftDB") and self.options.inputVDriftDB:
            self.add_local_vdrift_db()

        if hasattr(self.options, "inputDBTag") and self.options.inputDBTag:
            self.add_local_custom_db()

        if self.options.run_on_RAW:
            self.add_raw_option()
        if self.options.preselection:
            self.add_preselection()

def DTWorkflow.DTWorkflow.prepare_common_write	(		self,
			do_hadd = True
	)

Common operations used in most prepare_[workflow_mode]_erite functions

Definition at line 212 of file DTWorkflow.py.

References CrabHelper.CrabHelper.crab_config_filepath(), DTWorkflow.DTWorkflow.files_reveived, DTWorkflow.DTWorkflow.get_output_files(), tools.haddLocal(), DTWorkflow.DTWorkflow.load_options_command(), DTWorkflow.DTWorkflow.local_path(), DTT0WireWorkflow.DTT0WireWorkflow.output_file, DTVdriftWorkflow.DTvdriftWorkflow.output_file, DTTtrigWorkflow.DTttrigWorkflow.output_file, HltDiff.output_file, and DTWorkflow.DTWorkflow.result_path().

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

    def prepare_common_write(self, do_hadd=True):
        """ Common operations used in most prepare_[workflow_mode]_erite functions"""
        self.load_options_command("submit")
        output_path = os.path.join( self.local_path, "unmerged_results" )
        merged_file = os.path.join(self.result_path, self.output_file)
        crabtask = self.crabFunctions.CrabTask(crab_config = self.crab_config_filepath,
                                               initUpdate = False)
        if not (self.options.skip_stageout or self.files_reveived or self.options.no_exec):
            self.get_output_files(crabtask, output_path)
            log.info("Received files from storage element")
            log.info("Using hadd to merge output files")
        if not self.options.no_exec and do_hadd:
            returncode = tools.haddLocal(output_path, merged_file)
            if returncode != 0:
                raise RuntimeError("Failed to merge files with hadd")
        return crabtask.crabConfig.Data.outputDatasetTag

def DTWorkflow.DTWorkflow.prepare_workflow

(

self

)

Abstract implementation of prepare workflow function

Definition at line 88 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.run().

    def prepare_workflow(self):
        """ Abstract implementation of prepare workflow function"""
        errmsg = "Class `{}` does not implement `{}`"
        raise NotImplementedError( errmsg.format(self.__class__.__name__,
                                                     "prepare_workflow"))

def DTWorkflow.DTWorkflow.pset_path

(

self

)

full path to the pset file 

Definition at line 370 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path(), DTWorkflow.DTWorkflow.pset_name, DTT0WireWorkflow.DTT0WireWorkflow.pset_name, DTVdriftWorkflow.DTvdriftWorkflow.pset_name, and DTTtrigWorkflow.DTttrigWorkflow.pset_name.

Referenced by DTWorkflow.DTWorkflow.runCMSSWtask(), and DTWorkflow.DTWorkflow.write_pset_file().

    def pset_path(self):
        """ full path to the pset file """
        basepath = os.path.join( self.local_path, "psets")
        if not os.path.exists( basepath ):
            os.makedirs( basepath )
        return os.path.join( basepath, self.pset_name )

def DTWorkflow.DTWorkflow.pset_template_base_bath

(

self

)

Base path to folder containing pset files for cmsRun

Definition at line 360 of file DTWorkflow.py.

    def pset_template_base_bath(self):
        """ Base path to folder containing pset files for cmsRun"""
        return os.path.expandvars(os.path.join("$CMSSW_BASE",
                                               "src",
                                               "CalibMuon",
                                               "test",
                                               )
                                 )

def DTWorkflow.DTWorkflow.remote_out_path

(

self

)

Output path on remote excluding user base path
Returns a dict if crab is used due to crab path setting policy

Definition at line 292 of file DTWorkflow.py.

References DTT0WireWorkflow.DTT0WireWorkflow.outpath_command_tag, DTVdriftWorkflow.DTvdriftWorkflow.outpath_command_tag, DTTtrigWorkflow.DTttrigWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_command_tag, DTWorkflow.DTWorkflow.outpath_workflow_mode_tag(), str, FWLite.ElectronMVAID.tag, tnp::TagProbePair.tag, cond::persistency::IOVProxyData.tag, cond::persistency::IOVEditorData.tag, MeasurementEstimator::OpaquePayload.tag, Inspector.Inspector.tag, BeamSpotFakeConditions.tag, Entry< T >.tag, TrackListMerger::TkEDGetTokenss.tag, cond::Tag_t.tag, FWLite.WorkingPoints.tag, MuIsolatorResultProducer< BT >::DepositConf.tag, DQMNet::CoreObject.tag, personalPlayback.FrameworkJob.tag, DTWorkflow.DTWorkflow.tag(), EcalTPGParamReaderFromDB.user, dataset.BaseDataset.user, production_tasks.Task.user, popcon::RpcDataT.user, popcon::RPCObPVSSmapData.user, popcon::RpcDataV.user, popcon::RpcObGasData.user, popcon::RpcDataI.user, popcon::RpcDataS.user, popcon::RpcDataUXC.user, popcon::RpcDataGasMix.user, popcon::RpcDataFebmap.user, MatrixInjector.MatrixInjector.user, cmsPerfSuite.PerfSuite.user, production_tasks.ParseOptions.user, EcalDBConnection.user, production_tasks.FindOnCastor.user, production_tasks.CheckForMask.user, production_tasks.CheckForWrite.user, and DTWorkflow.DTWorkflow.user().

    def remote_out_path(self):
        """ Output path on remote excluding user base path
        Returns a dict if crab is used due to crab path setting policy"""
        if self.options.command =="submit":
            return {
                "outLFNDirBase" : os.path.join( "/store",
                                                "user",
                                                self.user,
                                                'DTCalibration/',
                                                self.outpath_command_tag,
                                                self.outpath_workflow_mode_tag),
                "outputDatasetTag" : self.tag
                    }
        else:
            return os.path.join( 'DTCalibration/',
                                 datasetstr,
                                 'Run' + str(self.options.run),
                                 self.outpath_command_tag,
                                 self.outpath_workflow_mode_tag,
                                 'v' + str(self.options.trial),
                                )

def DTWorkflow.DTWorkflow.result_path

(

self

)

Definition at line 353 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path().

Referenced by DTWorkflow.DTWorkflow.prepare_common_write(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

    def result_path(self):
        result_path = os.path.abspath(os.path.join(self.local_path,"results"))
        if not os.path.exists(result_path):
            os.makedirs(result_path)
        return result_path

def DTWorkflow.DTWorkflow.run

(

self

)

Generalized function to run workflow command

Definition at line 58 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.check_missing_options(), DTWorkflow.DTWorkflow.dump_options(), DTWorkflow.DTWorkflow.load_options(), DTWorkflow.DTWorkflow.local_path(), DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, validateAlignments.ValidationJobMultiIOV.options, DTT0WireWorkflow.DTT0WireWorkflow.prepare_workflow(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_workflow(), DTTtrigWorkflow.DTttrigWorkflow.prepare_workflow(), DTWorkflow.DTWorkflow.prepare_workflow(), DTWorkflow.DTWorkflow.required_options_dict, and DTWorkflow.DTWorkflow.required_options_prepare_dict.

Referenced by DTWorkflow.DTWorkflow.all(), Types.EventID.cppID(), Types.LuminosityBlockID.cppID(), and o2olib.O2OTool.execute().

    def run(self):
        """ Generalized function to run workflow command"""
        msg = "Preparing %s workflow" % self.options.workflow
        if hasattr(self.options, "command"):
            msg += " for command %s" % self.options.command
        log.info(msg)
        if self.options.config_path:
            self.load_options( self.options.config_path )
        #check if all options to prepare the command are used
        self.check_missing_options(self.required_options_prepare_dict)
        self.prepare_workflow()
        # create output folder if they do not exist yet
        if not os.path.exists( self.local_path ):
            os.makedirs(self.local_path)
        # dump used options
        self.dump_options()
        #check if all options to run the command are used
        self.check_missing_options(self.required_options_dict)
        try:
            run_function = getattr(self, self.options.command)
        except AttributeError:
            errmsg = "Class `{}` does not implement `{}` for workflow %s" % self.options.workflow
            if hasattr(self.options, "workflow_mode"):
                errmsg += "and workflow mode %s" % self.options.workflow_mode
            raise NotImplementedError( errmsg.format(self.__class__.__name__,
                                                     self.options.command))
        log.debug("Running command %s" % self.options.command)
        # call chosen function
        run_function()

def DTWorkflow.DTWorkflow.runCMSSWtask	(		self,
			pset_path = ""
	)

Run a cmsRun job locally. The member variable self.pset_path is used
    if pset_path argument is not given

Definition at line 276 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.pset_name, DTT0WireWorkflow.DTT0WireWorkflow.pset_name, DTVdriftWorkflow.DTvdriftWorkflow.pset_name, DTTtrigWorkflow.DTttrigWorkflow.pset_name, and DTWorkflow.DTWorkflow.pset_path().

Referenced by DTWorkflow.DTWorkflow.correction(), DTWorkflow.DTWorkflow.dump(), DTT0WireWorkflow.DTT0WireWorkflow.submit(), and DTWorkflow.DTWorkflow.write().

    def runCMSSWtask(self, pset_path=""):
        """ Run a cmsRun job locally. The member variable self.pset_path is used
            if pset_path argument is not given"""
        if self.options.no_exec:
            return 0
        process = subprocess.Popen( "cmsRun %s" % self.pset_path,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            shell = True)
        stdout = process.communicate()[0]
        log.info(stdout)
        if process.returncode != 0:
            raise RuntimeError("Failed to use cmsRun for pset %s" % self.pset_name)
        return process.returncode

def DTWorkflow.DTWorkflow.submit

(

self

)

Definition at line 104 of file DTWorkflow.py.

References CrabHelper.CrabHelper.submit_crab_task().

    def submit(self):
        self.submit_crab_task()

def DTWorkflow.DTWorkflow.tag

(

self

)

Definition at line 320 of file DTWorkflow.py.

References str.

Referenced by DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), and DTWorkflow.DTWorkflow.remote_out_path().

    def tag(self):
        return 'Run' + str(self.options.run) + '_v' + str(self.options.trial)

def DTWorkflow.DTWorkflow.user

(

self

)

Definition at line 324 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow._user, DTCalibrationWorker.DTCalibrationWorker.options, DTWorkflow.DTWorkflow.options, and validateAlignments.ValidationJobMultiIOV.options.

Referenced by DTWorkflow.DTWorkflow.remote_out_path().

    def user(self):
        if self._user:
            return self._user
        if hasattr(self.options, "user") and self.options.user:
            self._user = self.options.user
        else:
            self._user = self.crab.checkusername()
        return self._user

def DTWorkflow.DTWorkflow.write

(

self

)

Definition at line 111 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.runCMSSWtask().

Referenced by pkg.AbstractPkg.generate(), and querying.connection.write_and_commit().

    def write(self):
        self.runCMSSWtask()

def DTWorkflow.DTWorkflow.write_pset_file

(

self

)

Definition at line 377 of file DTWorkflow.py.

References DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.pset_path().

Referenced by DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit(), and DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write().

    def write_pset_file(self):
        if not hasattr(self, "process"):
            raise NameError("Process is not initalized in workflow object")
        if not os.path.exists(self.local_path):
            os.makedirs(self.local_path)
        with open( self.pset_path,'w') as pfile:
            pfile.write(self.process.dumpPython())

Member Data Documentation

DTWorkflow.DTWorkflow._user

private

Definition at line 39 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.user().

DTWorkflow.DTWorkflow.digilabel

Definition at line 24 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.add_raw_option().

DTWorkflow.DTWorkflow.files_reveived

Definition at line 38 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.prepare_common_write().

DTWorkflow.DTWorkflow.input_files

Definition at line 35 of file DTWorkflow.py.

DTWorkflow.DTWorkflow.options

Definition at line 22 of file DTWorkflow.py.

Referenced by DOTExport.DotProducer.__call__(), betterConfigParser.BetterConfigParser.__updateDict(), DTWorkflow.DTWorkflow.add_local_custom_db(), confdb.HLTProcess.buildOptions(), DTWorkflow.DTWorkflow.check_missing_options(), betterConfigParser.BetterConfigParser.checkInput(), DOTExport.DotProducer.connectTags(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_taskname(), DTWorkflow.DTWorkflow.dump_options(), DOTExport.DotExport.export(), CrabHelper.CrabHelper.fill_options_from_crab_config(), confdb.HLTProcess.fixPrescales(), confdb.HLTProcess.getRawConfigurationFromDB(), confdb.HLTProcess.getSetupConfigurationFromDB(), DOTExport.DotProducer.getTopLevel(), DTWorkflow.DTWorkflow.load_options(), DOTExport.DotProducer.makePath(), DOTExport.DotProducer.nodeLabel(), DOTExport.DotProducer.nodeURL(), DTWorkflow.DTWorkflow.prepare_common_submit(), DOTExport.DotProducer.produceLegend(), DOTExport.DotProducer.produceNodes(), DOTExport.DotProducer.producePaths(), DOTExport.DotProducer.produceServices(), DOTExport.DotProducer.produceSource(), edmIntegrityCheck.IntegrityCheck.query(), cmsswPreprocessor.CmsswPreprocessor.run(), DTWorkflow.DTWorkflow.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.CleanJobFiles.run(), DOTExport.DotProducer.seqRecurseChildren(), FileExportPlugin.FileExportPlugin.setOption(), DTWorkflow.DTWorkflow.user(), and DOTExport.DotExport.write_output().

DTWorkflow.DTWorkflow.outpath_command_tag

Definition at line 33 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.local_path(), and DTWorkflow.DTWorkflow.remote_out_path().

DTWorkflow.DTWorkflow.output_files

Definition at line 34 of file DTWorkflow.py.

DTWorkflow.DTWorkflow.process

Definition at line 230 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.add_local_calib_db(), DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), DTWorkflow.DTWorkflow.add_preselection(), DTWorkflow.DTWorkflow.add_raw_option(), ConfigBuilder.ConfigBuilder.addCustomise(), ConfigBuilder.ConfigBuilder.addOutput(), ConfigBuilder.ConfigBuilder.completeInputCommand(), Vispa.Plugins.ConfigEditor.ConfigDataAccessor.ConfigDataAccessor.dumpPython(), Vispa.Plugins.ConfigEditor.ConfigDataAccessor.ConfigDataAccessor.open(), Vispa.Plugins.ConfigEditor.ConfigDataAccessor.ConfigDataAccessor.outputEventContent(), ConfigBuilder.ConfigBuilder.prepare(), ConfigBuilder.ConfigBuilder.prepare_DQM(), ConfigBuilder.ConfigBuilder.prepare_GEN(), ConfigBuilder.ConfigBuilder.prepare_HLT(), ConfigBuilder.ConfigBuilder.prepare_PATFILTER(), ConfigBuilder.ConfigBuilder.prepare_VALIDATION(), runTauIdMVA.TauIDEmbedder.processDeepProducer(), ConfigBuilder.ConfigBuilder.renameHLTprocessInSequence(), ConfigBuilder.ConfigBuilder.renameInputTagsInSequence(), ConfigBuilder.ConfigBuilder.scheduleSequence(), Vispa.Plugins.ConfigEditor.ConfigDataAccessor.ConfigDataAccessor.setProcess(), and Vispa.Plugins.ConfigEditor.ConfigDataAccessor.ConfigDataAccessor.setProperty().

DTWorkflow.DTWorkflow.pset_name

Definition at line 32 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.pset_path(), and DTWorkflow.DTWorkflow.runCMSSWtask().

DTWorkflow.DTWorkflow.required_options_dict

Definition at line 27 of file DTWorkflow.py.

Referenced by CLIHelper.CLIHelper.fill_required_options_dict(), and DTWorkflow.DTWorkflow.run().

DTWorkflow.DTWorkflow.required_options_prepare_dict

Definition at line 28 of file DTWorkflow.py.

Referenced by CLIHelper.CLIHelper.fill_required_options_prepare_dict(), and DTWorkflow.DTWorkflow.run().

DTWorkflow.DTWorkflow.run_all_command

Definition at line 37 of file DTWorkflow.py.

Referenced by DTWorkflow.DTWorkflow.all().

---

Generated for CMSSW Reference Manual by  1.8.11