genericValidation.GenericValidation Class Reference

Inheritance diagram for genericValidation.GenericValidation:

Public Member Functions
def	__init__ (self, valName, alignment, config)
def	createConfiguration (self, fileContents, path, schedule=None, repMap=None, repMaps=None)
def	createCrabCfg (self, fileContents, path)
def	createFiles (self, fileContents, path, repMap=None, repMaps=None)
def	createScript (self, fileContents, path, downloadFiles=[], repMap=None, repMaps=None)
def	filesToCompare (self)
def	getCompareStrings (self, requestId=None, plain=False)
def	getRepMap (self, alignment=None)

Public Attributes
	alignmentToValidate
	AutoAlternates
	cmssw
	cmsswreleasebase
	config
	configFiles
	crabConfigFiles
	general
	jobid
	jobmode
	name
	needsproxy
	NJobs
	packages
	randomWorkdirPart
	scramarch
	scriptFiles

Static Public Attributes
string	defaultReferenceName = "DEFAULT"
dictionary	defaults
	mandatories = set()
dictionary	needpackages = {"Alignment/OfflineValidation"}
dictionary	optionals = {"jobmode"}

Static Private Attributes
	__metaclass__ = ValidationMetaClass

Detailed Description

Definition at line 41 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__	(		self,
			valName,
			alignment,
			config
	)

Definition at line 54 of file genericValidation.py.

    def __init__(self, valName, alignment, config):
        import random
        self.name = valName
        self.alignmentToValidate = alignment
        self.general = config.getGeneral()
        self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
        self.configFiles = []
        self.config = config
        self.jobid = ""

        theUpdate = config.getResultingSection(self.valType+":"+self.name,
                                               defaultDict = self.defaults,
                                               demandPars = self.mandatories)
        self.general.update(theUpdate)
        self.jobmode = self.general["jobmode"]
        self.NJobs = int(self.general["parallelJobs"])
        self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")

        # limit maximum number of parallel jobs to 40
        # (each output file is approximately 20MB)
        maximumNumberJobs = 40
        if self.NJobs > maximumNumberJobs:
            msg = ("Maximum allowed number of parallel jobs "
                   +str(maximumNumberJobs)+" exceeded!!!")
            raise AllInOneError(msg)
        if self.NJobs > 1 and not isinstance(self, ParallelValidation):
            raise AllInOneError("Parallel jobs not implemented for {}!\n"
                                "Please set parallelJobs = 1.".format(type(self).__name__))

        self.jobid = self.general["jobid"]
        if self.jobid:
            try:  #make sure it's actually a valid jobid
                output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
                if "is not found" in output: raise RuntimeError
            except RuntimeError:
                raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)

        self.cmssw = self.general["cmssw"]
        badcharacters = r"\'"
        for character in badcharacters:
            if character in self.cmssw:
                raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
                                    "path name.  If you really have it in such a ridiculously named location,\n"
                                    "try making a symbolic link somewhere with a decent name.")
        try:
            os.listdir(self.cmssw)
        except OSError:
            raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')

        if self.cmssw == os.environ["CMSSW_BASE"]:
            self.scramarch = os.environ["SCRAM_ARCH"]
            self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
        else:
            command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
                       ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
            commandoutput = getCommandOutput2(command).split('\n')
            self.cmssw = commandoutput[0]
            self.scramarch = commandoutput[1]
            self.cmsswreleasebase = commandoutput[2]

        self.packages = {}
        for package in self.needpackages:
            for placetolook in self.cmssw, self.cmsswreleasebase:
                pkgpath = os.path.join(placetolook, "src", package)
                if os.path.exists(pkgpath):
                    self.packages[package] = pkgpath
                    break
            else:
                raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))

        self.AutoAlternates = True
        if config.has_option("alternateTemplates","AutoAlternates"):
            try:
                self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
            except ValueError:
                raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))

        knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
        ignoreOpts = []
        config.checkInput(self.valType+":"+self.name,
                          knownSimpleOptions = knownOpts,
                          ignoreOptions = ignoreOpts)

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration	(		self,
			fileContents,
			path,
			schedule = None,
			repMap = None,
			repMaps = None
	)

Definition at line 216 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles, and genericValidation.GenericValidation.createFiles().

    def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
        self.configFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        if not schedule == None:
            schedule = [os.path.join( path, cfgName) for cfgName in schedule]
            for cfgName in schedule:
                if not cfgName in self.configFiles:
                    msg = ("scheduled %s missing in generated configfiles: %s"
                           %(cfgName, self.configFiles))
                    raise AllInOneError(msg)
            for cfgName in self.configFiles:
                if not cfgName in schedule:
                    msg = ("generated configuration %s not scheduled: %s"
                           %(cfgName, schedule))
                    raise AllInOneError(msg)
            self.configFiles = schedule
        return self.configFiles

def genericValidation.GenericValidation.createCrabCfg	(		self,
			fileContents,
			path
	)

Definition at line 242 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

    def createCrabCfg(self, fileContents, path ):
        if self.NJobs > 1:
            msg =  ("jobmode 'crab' not supported for parallel validation."
                    " Please set parallelJobs = 1.")
            raise AllInOneError(msg)
        self.crabConfigFiles = self.createFiles(fileContents, path)
        return self.crabConfigFiles

def genericValidation.GenericValidation.createFiles	(		self,
			fileContents,
			path,
			repMap = None,
			repMaps = None
	)

repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 193 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, helperFunctions.replaceByMap(), and str.

Referenced by genericValidation.GenericValidation.createConfiguration(), and geometryComparison.GeometryComparison.createScript().

    def createFiles(self, fileContents, path, repMap = None, repMaps = None):
        """repMap: single map for all files
           repMaps: a dict, with the filenames as the keys"""
        if repMap is not None and repMaps is not None:
            raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
        result = []
        for fileName in fileContents:
            filePath = os.path.join(path, fileName)
            result.append(filePath)

            for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
                theFile = open( filePathi, "w" )
                fileContentsi = fileContents[ fileName ]
                if repMaps is not None:
                    repMap = repMaps[fileName]
                if repMap is not None:
                    repMap.update({"nIndex": str(i)})
                    fileContentsi = replaceByMap(fileContentsi, repMap)
                theFile.write( fileContentsi )
                theFile.close()

        return result

def genericValidation.GenericValidation.createScript	(		self,
			fileContents,
			path,
			downloadFiles = [],
			repMap = None,
			repMaps = None
	)

Definition at line 234 of file genericValidation.py.

    def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
        self.scriptFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        for script in self.scriptFiles:
            for scriptwithindex in addIndex(script, self.NJobs):
                os.chmod(scriptwithindex,0o755)
        return self.scriptFiles

def genericValidation.GenericValidation.filesToCompare

(

self

)

Definition at line 166 of file genericValidation.py.

    def filesToCompare(self):
        pass

def genericValidation.GenericValidation.getCompareStrings	(		self,
			requestId = None,
			plain = False
	)

Definition at line 169 of file genericValidation.py.

Referenced by overlapValidation.OverlapValidation.appendToPlots(), and trackSplittingValidation.TrackSplittingValidation.appendToPlots().

    def getCompareStrings( self, requestId = None, plain = False ):
        result = {}
        repMap = self.getRepMap().copy()
        for validationId in self.filesToCompare:
            repMap["file"] = self.filesToCompare[ validationId ]
            if repMap["file"].startswith( "/castor/" ):
                repMap["file"] = "rfio:%(file)s"%repMap
            elif repMap["file"].startswith( "/store/" ):
                repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
            if plain:
                result[validationId]=repMap["file"]
            else:
                result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
        if requestId == None:
            return result
        else:
            if not "." in requestId:
                requestId += ".%s"%self.defaultReferenceName
            if not requestId.split(".")[-1] in result:
                msg = ("could not find %s in reference Objects!"
                       %requestId.split(".")[-1])
                raise AllInOneError(msg)
            return result[ requestId.split(".")[-1] ]

def genericValidation.GenericValidation.getRepMap	(		self,
			alignment = None
	)

Definition at line 137 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, geometry.Alignables.config, genericValidation.GenericValidation.config, genericValidation.GenericValidation.general, genericValidation.GenericValidation.packages, plottingOptions.PlottingOptions(), genericValidation.GenericValidation.randomWorkdirPart, genericValidation.GenericValidation.scramarch, trackSplittingValidation.TrackSplittingValidation.valType, monteCarloValidation.MonteCarloValidation.valType, overlapValidation.OverlapValidation.valType, primaryVertexValidation.PrimaryVertexValidation.valType, offlineValidation.OfflineValidation.valType, zMuMuValidation.ZMuMuValidation.valType, and geometryComparison.GeometryComparison.valType.

Referenced by overlapValidation.OverlapValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), primaryVertexValidation.PrimaryVertexValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), overlapValidation.OverlapValidation.appendToPlots(), trackSplittingValidation.TrackSplittingValidation.appendToPlots(), primaryVertexValidation.PrimaryVertexValidation.appendToPlots(), zMuMuValidation.ZMuMuValidation.appendToPlots(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), genericValidation.ParallelValidation.doMerge(), zMuMuValidation.ZMuMuValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), genericValidation.GenericValidation.getCompareStrings(), and plottingOptions.PlottingOptionsTrackSplitting.validsubdets().

    def getRepMap(self, alignment = None):
        from .plottingOptions import PlottingOptions
        if alignment == None:
            alignment = self.alignmentToValidate
        try:
            result = PlottingOptions(self.config, self.valType)
        except KeyError:
            result = {}
        result.update(alignment.getRepMap())
        result.update(self.general)
        result.update({
                "workdir": os.path.join(self.general["workdir"],
                                        self.randomWorkdirPart),
                "datadir": self.general["datadir"],
                "logdir": self.general["logdir"],
                "CommandLineTemplate": ("#run configfile and post-proccess it\n"
                                        "cmsRun %(cfgFile)s\n"
                                        "%(postProcess)s "),
                "CMSSW_BASE": self.cmssw,
                "SCRAM_ARCH": self.scramarch,
                "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
                "alignmentName": alignment.name,
                "condLoad": alignment.getConditions(),
                "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
                })
        result.update(self.packages)
        return result

Member Data Documentation

genericValidation.GenericValidation.__metaclass__ = ValidationMetaClass

staticprivate

Definition at line 42 of file genericValidation.py.

genericValidation.GenericValidation.alignmentToValidate

Definition at line 57 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.getRepMap(), geometryComparison.GeometryComparison.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 124 of file genericValidation.py.

genericValidation.GenericValidation.cmssw

Definition at line 91 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 105 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.config

Definition at line 61 of file genericValidation.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), and zMuMuValidation.ZMuMuValidation.trackcollection().

genericValidation.GenericValidation.configFiles

Definition at line 60 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 247 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"

static

Definition at line 43 of file genericValidation.py.

Referenced by zMuMuValidation.ZMuMuValidation.filesToCompare(), preexistingValidation.PreexistingValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), and genericValidation.GenericValidation.getCompareStrings().

dictionary genericValidation.GenericValidation.defaults

static

Initial value:

= {
                "cmssw":        os.environ['CMSSW_BASE'],
                "parallelJobs": "1",
                "jobid":        "",
                "needsproxy":   "false",
               }

Definition at line 45 of file genericValidation.py.

Referenced by tree.Tree.reset(), and tree.Tree.var().

genericValidation.GenericValidation.general

Definition at line 58 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.__init__(), offlineValidation.OfflineValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), preexistingValidation.PreexistingOfflineValidation.__init__(), plottingOptions.PlottingOptionsTrackSplitting.__init__(), plottingOptions.PlottingOptionsZMuMu.__init__(), plottingOptions.PlottingOptionsOffline.__init__(), plottingOptions.PlottingOptionsPrimaryVertex.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidationData_CTSR.cosmics0T(), preexistingValidation.PreexistingValidation.filesToCompare(), overlapValidation.OverlapValidation.getRepMap(), monteCarloValidation.MonteCarloValidation.getRepMap(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), preexistingValidation.PreexistingOfflineValidation.getRepMap(), plottingOptions.BasePlottingOptions.getRepMap(), preexistingValidation.PreexistingPrimaryVertexValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), and genericValidation.GenericValidationData_CTSR.use_d0cut().

genericValidation.GenericValidation.jobid

Definition at line 62 of file genericValidation.py.

genericValidation.GenericValidation.jobmode

Definition at line 68 of file genericValidation.py.

genericValidation.GenericValidation.mandatories = set()

static

Definition at line 44 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 56 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), genericValidation.GenericValidationData.cfgName(), crabFunctions.CrabTask.crabConfig(), crabFunctions.CrabTask.crabFolder(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), crabFunctions.CrabTask.handleNoState(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), crabFunctions.CrabTask.resubmit_failed(), production_tasks.MonitorJobs.run(), validateAlignments.ParallelMergeJob.runJob(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), crabFunctions.CrabTask.update(), crabFunctions.CrabTask.updateJobStats(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

dictionary genericValidation.GenericValidation.needpackages = {"Alignment/OfflineValidation"}

static

Definition at line 51 of file genericValidation.py.

genericValidation.GenericValidation.needsproxy

Definition at line 70 of file genericValidation.py.

genericValidation.GenericValidation.NJobs

Definition at line 69 of file genericValidation.py.

Referenced by monteCarloValidation.MonteCarloValidation.__init__(), offlineValidation.OfflineValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

dictionary genericValidation.GenericValidation.optionals = {"jobmode"}

static

Definition at line 52 of file genericValidation.py.

genericValidation.GenericValidation.packages

Definition at line 114 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 59 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 104 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 235 of file genericValidation.py.