validateAlignments.ValidationJobMultiIOV Class Reference

Inheritance diagram for validateAlignments.ValidationJobMultiIOV:

Public Member Functions
def	__init__ (self, validation, config, options, outPath, *args, **kwargs)
def	__iter__ (self)
def	__next__ (self)
def	createJob (self)
def	getValidation (self)
def	needsproxy (self)
def	runJob (self)
Public Member Functions inherited from validateAlignments.ValidationBase
def	__init__ (self, validation, config, options)

Static Public Member Functions
def	runCondorJobs (outdir)

Public Attributes
	config
	end
	optionMultiIOV
	options
	outPath
	start
	validation
	validations
	valSection
	valType
Public Attributes inherited from validateAlignments.ValidationBase
	alignments
	commandLineOptions
	config
	preexisting
	valName
	valSection
	valType

Private Member Functions
def	__performMultiIOV (self, validation, alignments, config, options, outPath)

Detailed Description

Definition at line 294 of file validateAlignments.py.

Constructor & Destructor Documentation

__init__()

def validateAlignments.ValidationJobMultiIOV.__init__	(		self,
			validation,
			config,
			options,
			outPath,
		*	args,
		**	kwargs
	)

Definition at line 296 of file validateAlignments.py.

    def __init__( self, validation, config, options, outPath, *args, **kwargs):
        self.start = 0
        self.end = args
        super(ValidationJobMultiIOV, self).__init__( validation, config, options )
        self.optionMultiIOV = self.config.getboolean( self.valSection, "multiIOV" )
        if self.optionMultiIOV == True:
            self.validation = validation
            self.config = config
            self.options = options
            self.outPath = outPath
            self.validations = self.__performMultiIOV(self.validation, self.alignments, self.config,
                                                  self.options, self.outPath)
 
 

Member Function Documentation

__iter__()

def validateAlignments.ValidationJobMultiIOV.__iter__

(

self

)

Definition at line 473 of file validateAlignments.py.

    def __iter__(self):
        yield self
 

__next__()

def validateAlignments.ValidationJobMultiIOV.__next__

(

self

)

Definition at line 476 of file validateAlignments.py.

    def __next__(self):
        if self.start >= len(self.end):
            raise StopIteration
        else:
            self.start += 1
            return self.end[self.start-1]
 
 

References validateAlignments.ValidationJob.end, validateAlignments.ValidationJobMultiIOV.end, svgfig.LineAxis.end, validateAlignments.ValidationJob.start, validateAlignments.ValidationJobMultiIOV.start, and svgfig.LineAxis.start.

__performMultiIOV()

def validateAlignments.ValidationJobMultiIOV.__performMultiIOV (   self,   validation,   alignments,   config,   options,   outPath  )

private

Definition at line 310 of file validateAlignments.py.

    def __performMultiIOV(self, validation, alignments, config, options, outPath):
        validations = []
        if self.valType == "compare":
            alignmentsList = alignments.split( "," )
            firstAlignList = alignmentsList[0].split()
            firstAlignName = firstAlignList[0].strip()
            secondAlignList = alignmentsList[1].split()
            secondAlignName = secondAlignList[0].strip()
            compareAlignments = "%s"%firstAlignName + "_vs_%s"%secondAlignName
            sectionMultiIOV = "multiIOV:compare"
            if not self.config.has_section(sectionMultiIOV):
                raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
            iovList = self.config.get( sectionMultiIOV, "iovs" )
            iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
            for iov in iovList:
                    tmpConfig = BetterConfigParser()
                    tmpConfig.read( options.config )
                    general = tmpConfig.getGeneral()
                    tmpConfig.add_section("IOV")
                    tmpConfig.set("IOV", "iov", iov)
                    tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("internals","scriptsdir",os.path.join(outPath, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpOptions = copy.deepcopy(options)
                    tmpOptions.Name = os.path.join(options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov)
                    tmpOptions.config = tmpConfig
                    newOutPath = os.path.abspath( tmpOptions.Name )
                    if not os.path.exists( newOutPath ):
                        os.makedirs( newOutPath )
                    elif not os.path.isdir( newOutPath ):
                        raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
                    job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
                    validations.append(job)
 
            return validations
 
        if "preexisting" in self.valType:
            preexistingValType = self.valType
            preexistingValSection = self.valSection
            preexistingEosdir = self.config.get( self.valSection, "eosdirName" )
            originalValType = preexistingValType.replace('preexisting', '')
            originalValName = self.config.get( self.valSection, "originalValName" )
            self.valSection = originalValType + ":" + originalValName
            originalAlignment = self.valName
 
        datasetList = self.config.get( self.valSection, "dataset" )
        datasetList = re.sub(r"\s+", "", datasetList, flags=re.UNICODE).split( "," )
        for dataset in datasetList:
            sectionMultiIOV = "multiIOV:%s"%dataset
            if not self.config.has_section(sectionMultiIOV):
                raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
            else:
                datasetBaseName = self.config.get( sectionMultiIOV, "dataset" )
                iovList = self.config.get( sectionMultiIOV, "iovs" )
                iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
                for iov in iovList:
                    datasetName = datasetBaseName+"_since%s"%iov
                    tmpConfig = BetterConfigParser()
                    tmpConfig.read( options.config )
                    general = tmpConfig.getGeneral()
                    if "preexisting" in self.valType:
                        valType = originalValType
                        valName = originalValName
                    else:
                        valType = self.valType
                        valName = self.valName
                    tmpConfig.add_section("IOV")
                    tmpConfig.set("IOV", "iov", iov)
                    tmpConfig.set( self.valSection, "dataset", datasetName )
                    tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("internals","scriptsdir",os.path.join(outPath, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    if "preexisting" in self.valType:
                        if self.valType == "preexistingoffline":
                            validationClassName = "AlignmentValidation"
                        #elif self.valType == "preexistingmcValidate":
                        #    validationClassName = "MonteCarloValidation"
                        #elif self.valType == "preexistingsplit":
                        #    validationClassName = "TrackSplittingValidation"
                        #elif self.valType == "preexistingprimaryvertex":
                        #    validationClassName = "PrimaryVertexValidation"
                        else:
                            raise AllInOneError("Unknown validation mode for preexisting option:'%s'"%self.valType)
                        preexistingEosdirPath = os.path.join("AlignmentValidation", preexistingEosdir, valType + "_" + valName + "_%s"%iov)
                        file = "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/" + "%s"%preexistingEosdirPath + "/%s"%validationClassName + "_%s"%originalValName + "_%s"%originalAlignment + ".root"
                        tmpConfig.set(preexistingValSection, "file", file)
                    tmpOptions = copy.deepcopy(options)
                    tmpOptions.Name = os.path.join(options.Name, valType + "_" + valName + "_%s"%iov)
                    tmpOptions.config = tmpConfig
                    newOutPath = os.path.abspath( tmpOptions.Name )
                    if not os.path.exists( newOutPath ):
                        os.makedirs( newOutPath )
                    elif not os.path.isdir( newOutPath ):
                        raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
                    job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
                    validations.append(job)
 
        return validations
 

createJob()

def validateAlignments.ValidationJobMultiIOV.createJob

(

self

)

Reimplemented from validateAlignments.ValidationBase.

Definition at line 413 of file validateAlignments.py.

    def createJob( self ):
        lmap( lambda validation: validation.createJob(), self.validations )
 

References validateAlignments.ValidationJobMultiIOV.validations.

getValidation()

def validateAlignments.ValidationJobMultiIOV.getValidation

(

self

)

Reimplemented from validateAlignments.ValidationBase.

Definition at line 467 of file validateAlignments.py.

    def getValidation( self ):
        return [validation.getValidation() for validation in self.validations]
 

References validateAlignments.ValidationJobMultiIOV.validations.

needsproxy()

def validateAlignments.ValidationJobMultiIOV.needsproxy

(

self

)

Reimplemented from validateAlignments.ValidationBase.

Definition at line 470 of file validateAlignments.py.

    def needsproxy( self ):
        return [validation.needsproxy() for validation in self.validations].join("and") and not self.preexisting and not self.commandLineOptions.dryRun
 

References validateAlignments.ValidationBase.commandLineOptions, join(), validateAlignments.ValidationBase.preexisting, and validateAlignments.ValidationJobMultiIOV.validations.

runCondorJobs()

def validateAlignments.ValidationJobMultiIOV.runCondorJobs (   outdir )

static

Definition at line 420 of file validateAlignments.py.

    def runCondorJobs(outdir):
        dagmanLog = "{}/daglogs".format(outdir)
        os.system("mkdir -p {}".format(dagmanLog))
 
 
        with open("{}/validation.condor".format(outdir), "w") as condor:
            condor.write("universe = vanilla" + "\n")
            condor.write("executable = $(scriptName).sh" + "\n")
            condor.write("log = $(scriptName).log" + "\n")
            condor.write("error = $(scriptName).stderr" + "\n")
            condor.write("output = $(scriptName).stdout" + "\n")
            condor.write('requirements = (OpSysAndVer =?= "CentOS7")' + '\n')
            condor.write('+JobFlavour = "tomorrow"' + "\n")
            condor.write('+RequestMemory = {}'.format(1540) + "\n")
            condor.write('+FileTransferDownloadBytes = {}'.format(1540) + "\n")
            condor.write('+AccountingGroup     = "group_u_CMS.CAF.ALCA"' + '\n')
            condor.write("queue")
 
        with open("{}/validation.dagman".format(outdir), "w") as dagman:
            parents = {}
            for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
 
                parents[(valType, valName, iov)] = []
                for jobInfo in alignments:
                    if not "preexisting" in jobInfo[0]:
                        dagman.write("JOB {}_{} {}/validation.condor".format(jobInfo[0], iov, outdir) + "\n")
                        dagman.write('VARS {}_{} '.format(jobInfo[0], iov) + 'scriptName="{}"'.format('.'.join(jobInfo[1].split('.')[:-1])) + "\n")
                        parents[(valType, valName, iov)].append('{}_{}'.format(jobInfo[0], iov))
                        dagman.write("\n")
 
                path =  os.path.join(jobInfo[2], "TkAlMerge.sh")
                if os.path.exists( path ):
                    dagman.write("JOB Merge_{}_{}_{} {}/validation.condor".format(valType, valName, iov, outdir) + "\n")
                    dagman.write("VARS Merge_{}_{}_{} ".format(valType, valName, iov) + 'scriptName="{}"'.format(os.path.join(jobInfo[2], "TkAlMerge")) + "\n")
                    dagman.write("\n")
                else:
                    raise AllInOneError("Merge script '[%s]' not found!"%path)
 
            for (valType, valName, iov), alignments in ValidationJob.condorConf.items():
                if len(parents[(valType, valName, iov)]) != 0:
                    dagman.write('PARENT {} '.format(" ".join([parent for parent in parents[(valType, valName, iov)]])) + 'CHILD Merge_{}_{}_{}'.format(valType, valName, iov) + "\n")
 
        submitCommands = ["condor_submit_dag -no_submit -outfile_dir {} {}/validation.dagman".format(dagmanLog, outdir), "condor_submit {}/validation.dagman.condor.sub".format(outdir)]
 
        for command in submitCommands:
            subprocess.call(command.split(" "))
 

References mps_setup.append, join(), and submitPVValidationJobs.split().

runJob()

def validateAlignments.ValidationJobMultiIOV.runJob

(

self

)

Reimplemented from validateAlignments.ValidationBase.

Definition at line 416 of file validateAlignments.py.

    def runJob( self ):
        return [validation.runJob() for validation in self.validations]
 

References validateAlignments.ValidationJobMultiIOV.validations.

Member Data Documentation

config

validateAlignments.ValidationJobMultiIOV.config

Definition at line 303 of file validateAlignments.py.

Referenced by confdb.HLTProcess.addEras(), confdb.HLTProcess.addGlobalOptions(), confdb.HLTProcess.build_source(), confdb.HLTProcess.buildOptions(), confdb.HLTProcess.customize(), confdb.HLTProcess.dump(), confdb.HLTProcess.fixPrescales(), pkg.AbstractPkg.generate(), pkg.AbstractPkg.get_kwds(), confdb.HLTProcess.getPathList(), confdb.HLTProcess.getRawConfigurationFromDB(), confdb.HLTProcess.getSetupConfigurationFromDB(), confdb.HLTProcess.instrumentDQM(), confdb.HLTProcess.instrumentErrorEventType(), confdb.HLTProcess.instrumentOpenMode(), confdb.HLTProcess.instrumentTiming(), confdb.HLTProcess.loadCffCommand(), confdb.HLTProcess.loadSetupCff(), confdb.HLTProcess.overrideGlobalTag(), confdb.HLTProcess.overrideL1MenuXml(), confdb.HLTProcess.overrideOutput(), confdb.HLTProcess.overrideProcessName(), pkg.AbstractPkg.parse_etags(), confdb.HLTProcess.runL1Emulator(), and confdb.HLTProcess.specificCustomize().

end

validateAlignments.ValidationJobMultiIOV.end

Definition at line 298 of file validateAlignments.py.

Referenced by OrderedSet.OrderedSet.__iter__(), validateAlignments.ValidationJobMultiIOV.__next__(), OrderedSet.OrderedSet.__reversed__(), OrderedSet.OrderedSet.add(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), o2olib.O2ORunMgr.endJob(), esMonitoring.FDOutputListener.handle_line(), and OrderedSet.OrderedSet.pop().

optionMultiIOV

validateAlignments.ValidationJobMultiIOV.optionMultiIOV

Definition at line 300 of file validateAlignments.py.

options

validateAlignments.ValidationJobMultiIOV.options

Definition at line 304 of file validateAlignments.py.

Referenced by betterConfigParser.BetterConfigParser.__updateDict(), submitPVValidationJobs.BetterConfigParser.__updateDict(), DTWorkflow.DTWorkflow.add_local_calib_db(), DTWorkflow.DTWorkflow.add_local_custom_db(), DTWorkflow.DTWorkflow.add_local_t0_db(), DTWorkflow.DTWorkflow.add_local_vdrift_db(), DTWorkflow.DTWorkflow.add_preselection(), DTWorkflow.DTWorkflow.all(), confdb.HLTProcess.buildOptions(), CrabHelper.CrabHelper.check_crabtask(), DTWorkflow.DTWorkflow.check_missing_options(), betterConfigParser.BetterConfigParser.checkInput(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_config_filepath(), CrabHelper.CrabHelper.crab_taskname(), DTWorkflow.DTWorkflow.dump_options(), CrabHelper.CrabHelper.fill_options_from_crab_config(), confdb.HLTProcess.fixPrescales(), DTWorkflow.DTWorkflow.get_config_name(), confdb.HLTProcess.getRawConfigurationFromDB(), confdb.HLTProcess.getSetupConfigurationFromDB(), edmIntegrityCheck.IntegrityCheck.listFiles(), DTWorkflow.DTWorkflow.load_options(), DTWorkflow.DTWorkflow.load_options_command(), DTWorkflow.DTWorkflow.local_path(), production_tasks.MonitorJobs.monitor(), DTWorkflow.DTWorkflow.outpath_workflow_mode_tag(), DTWorkflow.DTWorkflow.prepare_common_submit(), DTWorkflow.DTWorkflow.prepare_common_write(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_meantimer_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_residuals_submit(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_dump(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_segment_write(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_correction(), DTTtrigWorkflow.DTttrigWorkflow.prepare_timeboxes_dump(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_submit(), DTTtrigWorkflow.DTttrigWorkflow.prepare_validation_write(), DTT0WireWorkflow.DTT0WireWorkflow.prepare_workflow(), DTVdriftWorkflow.DTvdriftWorkflow.prepare_workflow(), DTTtrigWorkflow.DTttrigWorkflow.prepare_workflow(), edmIntegrityCheck.IntegrityCheck.query(), production_tasks.BaseDataset.query(), DTWorkflow.DTWorkflow.remote_out_path(), edmIntegrityCheck.IntegrityCheck.report(), DTCalibrationWorker.DTCalibrationWorker.run(), cmsswPreprocessor.CmsswPreprocessor.run(), DTWorkflow.DTWorkflow.run(), production_tasks.CheckDatasetExists.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.CreateJobDirectory.run(), production_tasks.SourceCFG.run(), production_tasks.FullCFG.run(), production_tasks.WriteToDatasets.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.WriteJobReport.run(), production_tasks.CleanJobFiles.run(), DTWorkflow.DTWorkflow.runCMSSWtask(), edmIntegrityCheck.IntegrityCheck.structured(), CrabHelper.CrabHelper.submit_crab_task(), DTWorkflow.DTWorkflow.tag(), edmIntegrityCheck.IntegrityCheck.test(), DTWorkflow.DTWorkflow.user(), and CrabHelper.CrabHelper.write_crabConfig().

outPath

validateAlignments.ValidationJobMultiIOV.outPath

Definition at line 305 of file validateAlignments.py.

start

validateAlignments.ValidationJobMultiIOV.start

Definition at line 297 of file validateAlignments.py.

Referenced by progressbar.ProgressBar.__next__(), validateAlignments.ValidationJobMultiIOV.__next__(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), o2olib.O2ORunMgr.endJob(), and esMonitoring.FDOutputListener.handle_line().

validation

validateAlignments.ValidationJobMultiIOV.validation

Definition at line 302 of file validateAlignments.py.

validations

validateAlignments.ValidationJobMultiIOV.validations

Definition at line 306 of file validateAlignments.py.

Referenced by validateAlignments.ValidationJobMultiIOV.createJob(), validateAlignments.ValidationJobMultiIOV.getValidation(), validateAlignments.ValidationJobMultiIOV.needsproxy(), and validateAlignments.ValidationJobMultiIOV.runJob().

valSection

validateAlignments.ValidationJobMultiIOV.valSection

Definition at line 354 of file validateAlignments.py.

valType

validateAlignments.ValidationJobMultiIOV.valType

Definition at line 312 of file validateAlignments.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().