genericValidation.GenericValidation Class Reference

Inheritance diagram for genericValidation.GenericValidation:

Public Member Functions
def	__init__ (self, valName, alignment, config)
def	createConfiguration (self, fileContents, path, schedule=None, repMap=None, repMaps=None)
def	createCrabCfg (self, fileContents, path)
def	createFiles (self, fileContents, path, repMap=None, repMaps=None)
def	createScript (self, fileContents, path, downloadFiles=[], repMap=None, repMaps=None)
def	filesToCompare (self)
def	getCompareStrings (self, requestId=None, plain=False)
def	getRepMap (self, alignment=None)

Public Attributes
	alignmentToValidate
	AutoAlternates
	cmssw
	cmsswreleasebase
	config
	configFiles
	crabConfigFiles
	general
	jobid
	jobmode
	name
	needsproxy
	NJobs
	packages
	randomWorkdirPart
	scramarch
	scriptFiles

Static Public Attributes
string	defaultReferenceName = "DEFAULT"
dictionary	defaults
	mandatories = set()
dictionary	needpackages = {"Alignment/OfflineValidation"}
dictionary	optionals = {"jobmode"}

Static Private Attributes
	__metaclass__ = ValidationMetaClass

Detailed Description

Definition at line 38 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__	(		self,
			valName,
			alignment,
			config
	)

Definition at line 51 of file genericValidation.py.

    def __init__(self, valName, alignment, config):
        import random
        self.name = valName
        self.alignmentToValidate = alignment
        self.general = config.getGeneral()
        self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
        self.configFiles = []
        self.config = config
        self.jobid = ""

        theUpdate = config.getResultingSection(self.valType+":"+self.name,
                                               defaultDict = self.defaults,
                                               demandPars = self.mandatories)
        self.general.update(theUpdate)
        self.jobmode = self.general["jobmode"]
        self.NJobs = int(self.general["parallelJobs"])
        self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")

        # limit maximum number of parallel jobs to 40
        # (each output file is approximately 20MB)
        maximumNumberJobs = 40
        if self.NJobs > maximumNumberJobs:
            msg = ("Maximum allowed number of parallel jobs "
                   +str(maximumNumberJobs)+" exceeded!!!")
            raise AllInOneError(msg)
        if self.NJobs > 1 and not isinstance(self, ParallelValidation):
            raise AllInOneError("Parallel jobs not implemented for {}!\n"
                                "Please set parallelJobs = 1.".format(type(self).__name__))

        self.jobid = self.general["jobid"]
        if self.jobid:
            try:  #make sure it's actually a valid jobid
                output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
                if "is not found" in output: raise RuntimeError
            except RuntimeError:
                raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)

        self.cmssw = self.general["cmssw"]
        badcharacters = r"\'"
        for character in badcharacters:
            if character in self.cmssw:
                raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
                                    "path name.  If you really have it in such a ridiculously named location,\n"
                                    "try making a symbolic link somewhere with a decent name.")
        try:
            os.listdir(self.cmssw)
        except OSError:
            raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')

        if self.cmssw == os.environ["CMSSW_BASE"]:
            self.scramarch = os.environ["SCRAM_ARCH"]
            self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
        else:
            command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
                       ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
            commandoutput = getCommandOutput2(command).split('\n')
            self.cmssw = commandoutput[0]
            self.scramarch = commandoutput[1]
            self.cmsswreleasebase = commandoutput[2]

        self.packages = {}
        for package in self.needpackages:
            for placetolook in self.cmssw, self.cmsswreleasebase:
                pkgpath = os.path.join(placetolook, "src", package)
                if os.path.exists(pkgpath):
                    self.packages[package] = pkgpath
                    break
            else:
                raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))

        self.AutoAlternates = True
        if config.has_option("alternateTemplates","AutoAlternates"):
            try:
                self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
            except ValueError:
                raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))

        knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
        ignoreOpts = []
        config.checkInput(self.valType+":"+self.name,
                          knownSimpleOptions = knownOpts,
                          ignoreOptions = ignoreOpts)

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration	(		self,
			fileContents,
			path,
			schedule = None,
			repMap = None,
			repMaps = None
	)

Definition at line 213 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles, and genericValidation.GenericValidation.createFiles().

    def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
        self.configFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        if not schedule == None:
            schedule = [os.path.join( path, cfgName) for cfgName in schedule]
            for cfgName in schedule:
                if not cfgName in self.configFiles:
                    msg = ("scheduled %s missing in generated configfiles: %s"
                           %(cfgName, self.configFiles))
                    raise AllInOneError(msg)
            for cfgName in self.configFiles:
                if not cfgName in schedule:
                    msg = ("generated configuration %s not scheduled: %s"
                           %(cfgName, schedule))
                    raise AllInOneError(msg)
            self.configFiles = schedule
        return self.configFiles

def genericValidation.GenericValidation.createCrabCfg	(		self,
			fileContents,
			path
	)

Definition at line 239 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

    def createCrabCfg(self, fileContents, path ):
        if self.NJobs > 1:
            msg =  ("jobmode 'crab' not supported for parallel validation."
                    " Please set parallelJobs = 1.")
            raise AllInOneError(msg)
        self.crabConfigFiles = self.createFiles(fileContents, path)
        return self.crabConfigFiles

def genericValidation.GenericValidation.createFiles	(		self,
			fileContents,
			path,
			repMap = None,
			repMaps = None
	)

repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 190 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, helperFunctions.replaceByMap(), and harvestTrackValidationPlots.str.

Referenced by genericValidation.GenericValidation.createConfiguration(), and geometryComparison.GeometryComparison.createScript().

    def createFiles(self, fileContents, path, repMap = None, repMaps = None):
        """repMap: single map for all files
           repMaps: a dict, with the filenames as the keys"""
        if repMap is not None and repMaps is not None:
            raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
        result = []
        for fileName in fileContents:
            filePath = os.path.join(path, fileName)
            result.append(filePath)

            for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
                theFile = open( filePathi, "w" )
                fileContentsi = fileContents[ fileName ]
                if repMaps is not None:
                    repMap = repMaps[fileName]
                if repMap is not None:
                    repMap.update({"nIndex": str(i)})
                    fileContentsi = replaceByMap(fileContentsi, repMap)
                theFile.write( fileContentsi )
                theFile.close()

        return result

def genericValidation.GenericValidation.createScript	(		self,
			fileContents,
			path,
			downloadFiles = [],
			repMap = None,
			repMaps = None
	)

Definition at line 231 of file genericValidation.py.

    def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
        self.scriptFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        for script in self.scriptFiles:
            for scriptwithindex in addIndex(script, self.NJobs):
                os.chmod(scriptwithindex,0o755)
        return self.scriptFiles

def genericValidation.GenericValidation.filesToCompare

(

self

)

Definition at line 163 of file genericValidation.py.

    def filesToCompare(self):
        pass

def genericValidation.GenericValidation.getCompareStrings	(		self,
			requestId = None,
			plain = False
	)

Definition at line 166 of file genericValidation.py.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToPlots().

    def getCompareStrings( self, requestId = None, plain = False ):
        result = {}
        repMap = self.getRepMap().copy()
        for validationId in self.filesToCompare:
            repMap["file"] = self.filesToCompare[ validationId ]
            if repMap["file"].startswith( "/castor/" ):
                repMap["file"] = "rfio:%(file)s"%repMap
            elif repMap["file"].startswith( "/store/" ):
                repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
            if plain:
                result[validationId]=repMap["file"]
            else:
                result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
        if requestId == None:
            return result
        else:
            if not "." in requestId:
                requestId += ".%s"%self.defaultReferenceName
            if not requestId.split(".")[-1] in result:
                msg = ("could not find %s in reference Objects!"
                       %requestId.split(".")[-1])
                raise AllInOneError(msg)
            return result[ requestId.split(".")[-1] ]

def genericValidation.GenericValidation.getRepMap	(		self,
			alignment = None
	)

Definition at line 134 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, geometry.Alignables.config, genericValidation.GenericValidation.config, genericValidation.GenericValidation.general, genericValidation.GenericValidation.packages, plottingOptions.PlottingOptions(), genericValidation.GenericValidation.randomWorkdirPart, genericValidation.GenericValidation.scramarch, trackSplittingValidation.TrackSplittingValidation.valType, monteCarloValidation.MonteCarloValidation.valType, primaryVertexValidation.PrimaryVertexValidation.valType, offlineValidation.OfflineValidation.valType, zMuMuValidation.ZMuMuValidation.valType, and geometryComparison.GeometryComparison.valType.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToMerge(), primaryVertexValidation.PrimaryVertexValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.appendToPlots(), primaryVertexValidation.PrimaryVertexValidation.appendToPlots(), zMuMuValidation.ZMuMuValidation.appendToPlots(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), genericValidation.ParallelValidation.doMerge(), zMuMuValidation.ZMuMuValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), genericValidation.GenericValidation.getCompareStrings(), and plottingOptions.PlottingOptionsTrackSplitting.validsubdets().

    def getRepMap(self, alignment = None):
        from plottingOptions import PlottingOptions
        if alignment == None:
            alignment = self.alignmentToValidate
        try:
            result = PlottingOptions(self.config, self.valType)
        except KeyError:
            result = {}
        result.update(alignment.getRepMap())
        result.update(self.general)
        result.update({
                "workdir": os.path.join(self.general["workdir"],
                                        self.randomWorkdirPart),
                "datadir": self.general["datadir"],
                "logdir": self.general["logdir"],
                "CommandLineTemplate": ("#run configfile and post-proccess it\n"
                                        "cmsRun %(cfgFile)s\n"
                                        "%(postProcess)s "),
                "CMSSW_BASE": self.cmssw,
                "SCRAM_ARCH": self.scramarch,
                "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
                "alignmentName": alignment.name,
                "condLoad": alignment.getConditions(),
                "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
                })
        result.update(self.packages)
        return result

Member Data Documentation

genericValidation.GenericValidation.__metaclass__ = ValidationMetaClass

staticprivate

Definition at line 39 of file genericValidation.py.

genericValidation.GenericValidation.alignmentToValidate

Definition at line 54 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), geometryComparison.GeometryComparison.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 121 of file genericValidation.py.

genericValidation.GenericValidation.cmssw

Definition at line 88 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 102 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.config

Definition at line 58 of file genericValidation.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), DTTTrigProd.DTTTrigProd.initCrab(), DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab(), DTT0DBValidation.DTT0DBValidation.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), DTTTrigValid.DTTTrigValid.initProcess(), DTCalibrationWorker.DTCalibrationWorker.run(), and zMuMuValidation.ZMuMuValidation.trackcollection().

genericValidation.GenericValidation.configFiles

Definition at line 57 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 244 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"

static

Definition at line 40 of file genericValidation.py.

Referenced by zMuMuValidation.ZMuMuValidation.filesToCompare(), preexistingValidation.PreexistingValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), and genericValidation.GenericValidation.getCompareStrings().

dictionary genericValidation.GenericValidation.defaults

static

Initial value:

= {
                "cmssw":        os.environ['CMSSW_BASE'],
                "parallelJobs": "1",
                "jobid":        "",
                "needsproxy":   "false",
               }

Definition at line 42 of file genericValidation.py.

Referenced by tree.Tree.reset(), and tree.Tree.var().

genericValidation.GenericValidation.general

Definition at line 55 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), offlineValidation.OfflineValidation.__init__(), preexistingValidation.PreexistingOfflineValidation.__init__(), plottingOptions.PlottingOptionsTrackSplitting.__init__(), plottingOptions.PlottingOptionsZMuMu.__init__(), plottingOptions.PlottingOptionsOffline.__init__(), plottingOptions.PlottingOptionsPrimaryVertex.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidationData_CTSR.cosmics0T(), preexistingValidation.PreexistingValidation.filesToCompare(), monteCarloValidation.MonteCarloValidation.getRepMap(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), preexistingValidation.PreexistingOfflineValidation.getRepMap(), plottingOptions.BasePlottingOptions.getRepMap(), genericValidation.GenericValidation.getRepMap(), and genericValidation.GenericValidationData_CTSR.use_d0cut().

genericValidation.GenericValidation.jobid

Definition at line 59 of file genericValidation.py.

genericValidation.GenericValidation.jobmode

Definition at line 65 of file genericValidation.py.

genericValidation.GenericValidation.mandatories = set()

static

Definition at line 41 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 53 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), genericValidation.GenericValidationData.cfgName(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), validateAlignments.ParallelMergeJob.runJob(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

dictionary genericValidation.GenericValidation.needpackages = {"Alignment/OfflineValidation"}

static

Definition at line 48 of file genericValidation.py.

genericValidation.GenericValidation.needsproxy

Definition at line 67 of file genericValidation.py.

genericValidation.GenericValidation.NJobs

Definition at line 66 of file genericValidation.py.

Referenced by monteCarloValidation.MonteCarloValidation.__init__(), primaryVertexValidation.PrimaryVertexValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), offlineValidation.OfflineValidation.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

dictionary genericValidation.GenericValidation.optionals = {"jobmode"}

static

Definition at line 49 of file genericValidation.py.

genericValidation.GenericValidation.packages

Definition at line 111 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 56 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 101 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 232 of file genericValidation.py.