validateAlignments.ValidationJobMultiIOV Class Reference

Inheritance diagram for validateAlignments.ValidationJobMultiIOV:

Public Member Functions
def	__init__ (self, validation, config, options, outPath, args, kwargs)
def	__iter__ (self)
def	__next__ (self)
def	createJob (self)
def	getValidation (self)
def	needsproxy (self)
def	runJob (self)
Public Member Functions inherited from validateAlignments.ValidationBase
def	__init__ (self, validation, config, options)
def	createJob (self)
def	getValidation (self)
def	needsproxy (self)
def	runJob (self)

Static Public Member Functions
def	runCondorJobs (outdir)

Public Attributes
	config
	end
	optionMultiIOV
	options
	outPath
	start
	validation
	validations
	valSection
	valType
Public Attributes inherited from validateAlignments.ValidationBase
	alignments
	commandLineOptions
	config
	preexisting
	valName
	valSection
	valType

Private Member Functions
def	__performMultiIOV (self, validation, alignments, config, options, outPath)

Detailed Description

Definition at line 288 of file validateAlignments.py.

Constructor & Destructor Documentation

def validateAlignments.ValidationJobMultiIOV.__init__	(		self,
			validation,
			config,
			options,
			outPath,
			args,
			kwargs
	)

Definition at line 290 of file validateAlignments.py.

    def __init__( self, validation, config, options, outPath, *args, **kwargs):
        self.start = 0
        self.end = args
        super(ValidationJobMultiIOV, self).__init__( validation, config, options )
        self.optionMultiIOV = self.config.getboolean( self.valSection, "multiIOV" )
        if self.optionMultiIOV == True:
            self.validation = validation
            self.config = config
            self.options = options
            self.outPath = outPath
            self.validations = self.__performMultiIOV(self.validation, self.alignments, self.config,
                                                  self.options, self.outPath)

Member Function Documentation

def validateAlignments.ValidationJobMultiIOV.__iter__

(

self

)

Definition at line 467 of file validateAlignments.py.

    def __iter__(self):
        yield self

def validateAlignments.ValidationJobMultiIOV.__next__

(

self

)

Definition at line 470 of file validateAlignments.py.

References validateAlignments.ValidationJob.end, validateAlignments.ValidationJobMultiIOV.end, svgfig.LineAxis.end, validateAlignments.ValidationJob.start, validateAlignments.ValidationJobMultiIOV.start, and svgfig.LineAxis.start.

    def __next__(self):
        if self.start >= len(self.end):
            raise StopIteration
        else:
            self.start += 1
            return self.end[self.start-1]

def validateAlignments.ValidationJobMultiIOV.__performMultiIOV (   self,   validation,   alignments,   config,   options,   outPath  )

private

Definition at line 304 of file validateAlignments.py.

    def __performMultiIOV(self, validation, alignments, config, options, outPath):
        validations = []
        if self.valType == "compare":
            alignmentsList = alignments.split( "," )
            firstAlignList = alignmentsList[0].split()
            firstAlignName = firstAlignList[0].strip()
            secondAlignList = alignmentsList[1].split()
            secondAlignName = secondAlignList[0].strip()
            compareAlignments = "%s"%firstAlignName + "_vs_%s"%secondAlignName
            sectionMultiIOV = "multiIOV:compare"
            if not self.config.has_section(sectionMultiIOV):
                raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
            iovList = self.config.get( sectionMultiIOV, "iovs" )
            iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
            for iov in iovList:
                    tmpConfig = BetterConfigParser()
                    tmpConfig.read( options.config )
                    general = tmpConfig.getGeneral()
                    tmpConfig.add_section("IOV")
                    tmpConfig.set("IOV", "iov", iov)
                    tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("internals","scriptsdir",os.path.join(outPath, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov) )
                    tmpOptions = copy.deepcopy(options)
                    tmpOptions.Name = os.path.join(options.Name, self.valType + "_%s"%compareAlignments + "_%s"%iov)
                    tmpOptions.config = tmpConfig
                    newOutPath = os.path.abspath( tmpOptions.Name )
                    if not os.path.exists( newOutPath ):
                        os.makedirs( newOutPath )
                    elif not os.path.isdir( newOutPath ):
                        raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
                    job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
                    validations.append(job)

            return validations

        if "preexisting" in self.valType:
            preexistingValType = self.valType
            preexistingValSection = self.valSection
            preexistingEosdir = self.config.get( self.valSection, "eosdirName" )
            originalValType = preexistingValType.replace('preexisting', '')
            originalValName = self.config.get( self.valSection, "originalValName" )
            self.valSection = originalValType + ":" + originalValName
            originalAlignment = self.valName

        datasetList = self.config.get( self.valSection, "dataset" )
        datasetList = re.sub(r"\s+", "", datasetList, flags=re.UNICODE).split( "," )
        for dataset in datasetList:
            sectionMultiIOV = "multiIOV:%s"%dataset
            if not self.config.has_section(sectionMultiIOV):
                raise AllInOneError("section'[%s]' not found. Please define the dataset"%sectionMultiIOV)
            else:
                datasetBaseName = self.config.get( sectionMultiIOV, "dataset" )
                iovList = self.config.get( sectionMultiIOV, "iovs" )
                iovList = re.sub(r"\s+", "", iovList, flags=re.UNICODE).split( "," )
                for iov in iovList:
                    datasetName = datasetBaseName+"_since%s"%iov
                    tmpConfig = BetterConfigParser()
                    tmpConfig.read( options.config )
                    general = tmpConfig.getGeneral()
                    if "preexisting" in self.valType:
                        valType = originalValType
                        valName = originalValName
                    else:
                        valType = self.valType
                        valName = self.valName
                    tmpConfig.add_section("IOV")
                    tmpConfig.set("IOV", "iov", iov)
                    tmpConfig.set( self.valSection, "dataset", datasetName )
                    tmpConfig.set("internals","workdir",os.path.join(general["workdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("internals","scriptsdir",os.path.join(outPath, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","datadir",os.path.join(general["datadir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","logdir",os.path.join(general["logdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    tmpConfig.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name, valType + "_" + valName + "_%s"%iov) )
                    if "preexisting" in self.valType:
                        if self.valType == "preexistingoffline":
                            validationClassName = "AlignmentValidation"
                        #elif self.valType == "preexistingmcValidate":
                        #    validationClassName = "MonteCarloValidation"
                        #elif self.valType == "preexistingsplit":
                        #    validationClassName = "TrackSplittingValidation"
                        #elif self.valType == "preexistingprimaryvertex":
                        #    validationClassName = "PrimaryVertexValidation"
                        else:
                            raise AllInOneError("Unknown validation mode for preexisting option:'%s'"%self.valType)
                        preexistingEosdirPath = os.path.join("AlignmentValidation", preexistingEosdir, valType + "_" + valName + "_%s"%iov)
                        file = "/eos/cms/store/group/alca_trackeralign/AlignmentValidation/" + "%s"%preexistingEosdirPath + "/%s"%validationClassName + "_%s"%originalValName + "_%s"%originalAlignment + ".root"
                        tmpConfig.set(preexistingValSection, "file", file)
                    tmpOptions = copy.deepcopy(options)
                    tmpOptions.Name = os.path.join(options.Name, valType + "_" + valName + "_%s"%iov)
                    tmpOptions.config = tmpConfig
                    newOutPath = os.path.abspath( tmpOptions.Name )
                    if not os.path.exists( newOutPath ):
                        os.makedirs( newOutPath )
                    elif not os.path.isdir( newOutPath ):
                        raise AllInOneError("the file %s is in the way rename the Job or move it away"%newOutPath)
                    job = ValidationJob( validation, tmpConfig, tmpOptions, len(iovList) )
                    validations.append(job)

        return validations

def validateAlignments.ValidationJobMultiIOV.createJob

(

self

)

Definition at line 407 of file validateAlignments.py.

References genParticles_cff.map, and validateAlignments.ValidationJobMultiIOV.validations.

    def createJob( self ):
        map( lambda validation: validation.createJob(), self.validations )

def validateAlignments.ValidationJobMultiIOV.getValidation

(

self

)

Definition at line 461 of file validateAlignments.py.

References validateAlignments.ValidationJobMultiIOV.validations.

    def getValidation( self ):
        return [validation.getValidation() for validation in self.validations]

def validateAlignments.ValidationJobMultiIOV.needsproxy

(

self

)

Definition at line 464 of file validateAlignments.py.

References join(), validateAlignments.ValidationBase.preexisting, and validateAlignments.ValidationJobMultiIOV.validations.

    def needsproxy( self ):
        return [validation.needsproxy() for validation in self.validations].join("and") and not self.preexisting and not self.commandLineOptions.dryRun

def validateAlignments.ValidationJobMultiIOV.runCondorJobs (   outdir )

static

Definition at line 414 of file validateAlignments.py.

References mps_setup.append, join(), and split.

    def runCondorJobs(outdir):
        dagmanLog = "{}/daglogs".format(outdir)
        os.system("mkdir -p {}".format(dagmanLog))


        with open("{}/validation.condor".format(outdir), "w") as condor:
            condor.write("universe = vanilla" + "\n")
            condor.write("executable = $(scriptName).sh" + "\n")
            condor.write("log = $(scriptName).log" + "\n")
            condor.write("error = $(scriptName).stderr" + "\n")
            condor.write("output = $(scriptName).stdout" + "\n")
            condor.write('requirements = (OpSysAndVer =?= "CentOS7")' + '\n')
            condor.write('+JobFlavour = "tomorrow"' + "\n")
            condor.write('+RequestMemory = {}'.format(1540) + "\n")
            condor.write('+FileTransferDownloadBytes = {}'.format(1540) + "\n")
            condor.write('+AccountingGroup     = "group_u_CMS.CAF.ALCA"' + '\n')
            condor.write("queue")

        with open("{}/validation.dagman".format(outdir), "w") as dagman:
            parents = {}
            for (valType, valName, iov), alignments in six.iteritems(ValidationJob.condorConf):

                parents[(valType, valName, iov)] = []
                for jobInfo in alignments:
                    if not "preexisting" in jobInfo[0]:
                        dagman.write("JOB {}_{} {}/validation.condor".format(jobInfo[0], iov, outdir) + "\n")
                        dagman.write('VARS {}_{} '.format(jobInfo[0], iov) + 'scriptName="{}"'.format('.'.join(jobInfo[1].split('.')[:-1])) + "\n")
                        parents[(valType, valName, iov)].append('{}_{}'.format(jobInfo[0], iov))
                        dagman.write("\n")

                path =  os.path.join(jobInfo[2], "TkAlMerge.sh")
                if os.path.exists( path ):
                    dagman.write("JOB Merge_{}_{}_{} {}/validation.condor".format(valType, valName, iov, outdir) + "\n")
                    dagman.write("VARS Merge_{}_{}_{} ".format(valType, valName, iov) + 'scriptName="{}"'.format(os.path.join(jobInfo[2], "TkAlMerge")) + "\n")
                    dagman.write("\n")
                else:
                    raise AllInOneError("Merge script '[%s]' not found!"%path)

            for (valType, valName, iov), alignments in six.iteritems(ValidationJob.condorConf):
                if len(parents[(valType, valName, iov)]) != 0:
                    dagman.write('PARENT {} '.format(" ".join([parent for parent in parents[(valType, valName, iov)]])) + 'CHILD Merge_{}_{}_{}'.format(valType, valName, iov) + "\n")

        submitCommands = ["condor_submit_dag -no_submit -outfile_dir {} {}/validation.dagman".format(dagmanLog, outdir), "condor_submit {}/validation.dagman.condor.sub".format(outdir)]

        for command in submitCommands:
            subprocess.call(command.split(" "))

def validateAlignments.ValidationJobMultiIOV.runJob

(

self

)

Definition at line 410 of file validateAlignments.py.

References validateAlignments.ValidationJobMultiIOV.validations.

    def runJob( self ):
        return [validation.runJob() for validation in self.validations]

Member Data Documentation

validateAlignments.ValidationJobMultiIOV.config

Definition at line 297 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.end

Definition at line 292 of file validateAlignments.py.

Referenced by OrderedSet.OrderedSet.__iter__(), validateAlignments.ValidationJobMultiIOV.__next__(), OrderedSet.OrderedSet.__reversed__(), OrderedSet.OrderedSet.add(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), o2olib.O2ORunMgr.endJob(), OrderedSet.OrderedSet.pop(), and cmsPerfSuite.PerfSuiteTimer.set_end().

validateAlignments.ValidationJobMultiIOV.optionMultiIOV

Definition at line 294 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.options

Definition at line 298 of file validateAlignments.py.

Referenced by DOTExport.DotProducer.__call__(), betterConfigParser.BetterConfigParser.__updateDict(), DTWorkflow.DTWorkflow.add_local_custom_db(), confdb.HLTProcess.buildOptions(), DTWorkflow.DTWorkflow.check_missing_options(), betterConfigParser.BetterConfigParser.checkInput(), DOTExport.DotProducer.connectTags(), CrabHelper.CrabHelper.crab_config_filename(), CrabHelper.CrabHelper.crab_taskname(), DTWorkflow.DTWorkflow.dump_options(), DOTExport.DotExport.export(), CrabHelper.CrabHelper.fill_options_from_crab_config(), confdb.HLTProcess.fixPrescales(), confdb.HLTProcess.getRawConfigurationFromDB(), confdb.HLTProcess.getSetupConfigurationFromDB(), DOTExport.DotProducer.getTopLevel(), DTWorkflow.DTWorkflow.load_options(), DOTExport.DotProducer.makePath(), DOTExport.DotProducer.nodeLabel(), DOTExport.DotProducer.nodeURL(), DTWorkflow.DTWorkflow.prepare_common_submit(), DOTExport.DotProducer.produceLegend(), DOTExport.DotProducer.produceNodes(), DOTExport.DotProducer.producePaths(), DOTExport.DotProducer.produceServices(), DOTExport.DotProducer.produceSource(), edmIntegrityCheck.IntegrityCheck.query(), DTCalibrationWorker.DTCalibrationWorker.run(), cmsswPreprocessor.CmsswPreprocessor.run(), DTWorkflow.DTWorkflow.run(), production_tasks.BaseDataset.run(), production_tasks.GenerateMask.run(), production_tasks.RunCMSBatch.run(), production_tasks.MonitorJobs.run(), production_tasks.CheckJobStatus.run(), production_tasks.CleanJobFiles.run(), DOTExport.DotProducer.seqRecurseChildren(), FileExportPlugin.FileExportPlugin.setOption(), DTWorkflow.DTWorkflow.user(), and DOTExport.DotExport.write_output().

validateAlignments.ValidationJobMultiIOV.outPath

Definition at line 299 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.start

Definition at line 291 of file validateAlignments.py.

Referenced by progressbar.ProgressBar.__next__(), validateAlignments.ValidationJobMultiIOV.__next__(), Types.LuminosityBlockRange.cppID(), Types.EventRange.cppID(), o2olib.O2ORunMgr.endJob(), cmsPerfSuite.PerfSuiteTimer.set_end(), and cmsPerfSuite.PerfSuiteTimer.set_start().

validateAlignments.ValidationJobMultiIOV.validation

Definition at line 296 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.validations

Definition at line 300 of file validateAlignments.py.

Referenced by validateAlignments.ValidationJobMultiIOV.createJob(), validateAlignments.ValidationJobMultiIOV.getValidation(), validateAlignments.ValidationJobMultiIOV.needsproxy(), and validateAlignments.ValidationJobMultiIOV.runJob().

validateAlignments.ValidationJobMultiIOV.valSection

Definition at line 348 of file validateAlignments.py.

validateAlignments.ValidationJobMultiIOV.valType

Definition at line 306 of file validateAlignments.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().