validation Namespace Reference

Classes
class	Sample
class	SeparateValidation
class	SimpleSample
class	SimpleValidation
class	Validation

Functions
def	_copyDir (src, dst)
def	_copySubDir (oldfile, newfile, basenames, dirname)
def	_doPlotsForPlotter (self, plotter, sample, limitSubFoldersOnlyTo=None)
def	_findDuplicates (lst)
def	_getGlobalTag (sample, release)
def	_getRelValUrl (release)
def	_processPlotsForSample (plotterFolder, sample)
def	_stripRelease (release)

Variables
	_doBHadronSamples
	_doConversionSamples
	_doElectronSamples
	_globalTags
	_releasePostfixes
	_relvalUrls
	IgnoreCommandLineOptions

Function Documentation

_copyDir()

def validation._copyDir (   src,   dst  )

private

Copy non-TTree objects from src TDirectory to dst TDirectory.

Definition at line 1166 of file validation.py.

Referenced by _copySubDir().

def _copyDir(src, dst):
    """Copy non-TTree objects from src TDirectory to dst TDirectory."""
    keys = src.GetListOfKeys()
    for key in keys:
        classname = key.GetClassName()
        cl = ROOT.TClass.GetClass(classname)
        if not cl:
            continue
        if not (cl.InheritsFrom("TTree") and cl.InheritsFrom("TDirectory")):
            dst.cd()
            obj = key.ReadObj()
            obj.Write()
            obj.Delete()

_copySubDir()

def validation._copySubDir (   oldfile,   newfile,   basenames,   dirname  )

private

Copy a subdirectory from oldfile to newfile.

Arguments:
oldfile   -- String for source TFile
newfile   -- String for destination TFile
basenames -- List of strings for base directories, first existing one is picked
dirname   -- String for directory name under the base directory

Definition at line 1131 of file validation.py.

References _copyDir(), join(), and submitPVValidationJobs.split().

Referenced by validation.Validation._doPlots().

def _copySubDir(oldfile, newfile, basenames, dirname):
    """Copy a subdirectory from oldfile to newfile.

    Arguments:
    oldfile   -- String for source TFile
    newfile   -- String for destination TFile
    basenames -- List of strings for base directories, first existing one is picked
    dirname   -- String for directory name under the base directory
    """
    oldf = ROOT.TFile.Open(oldfile)

    dirold = None
    for basename in basenames:
        dirold = oldf.GetDirectory(basename)
        if dirold:
            break
    if not dirold:
        raise Exception("Did not find any of %s directories from file %s" % (",".join(basenames), oldfile))
    if dirname:
        d = dirold.Get(dirname)
        if not d:
            raise Exception("Did not find directory %s under %s" % (dirname, dirold.GetPath()))
        dirold = d

    newf = ROOT.TFile.Open(newfile, "RECREATE")
    dirnew = newf
    for d in basenames[0].split("/"):
        dirnew = dirnew.mkdir(d)
    if dirname:
        dirnew = dirnew.mkdir(dirname)
    _copyDir(dirold, dirnew)

    oldf.Close()
    return newf

_doPlotsForPlotter()

def validation._doPlotsForPlotter (   self,   plotter,   sample,   limitSubFoldersOnlyTo = None  )

private

Definition at line 1190 of file validation.py.

References _processPlotsForSample(), join(), and isotrackApplyRegressor.range.

def _doPlotsForPlotter(self, plotter, sample, limitSubFoldersOnlyTo=None):
    plottingProcess = self._nProc
    if plottingProcess<=0: plottingProcess = os.cpu_count()
    plotterInstance = plotter.readDirs(*self._openFiles)
    manager = multiprocessing.Manager()
    return_dict = manager.dict()
    proc = []
    iProc = 0
    active_proc = []

    for plotterFolder, dqmSubFolder in plotterInstance.iterFolders(limitSubFoldersOnlyTo=limitSubFoldersOnlyTo):
        if sample is not None and not _processPlotsForSample(plotterFolder, sample):
            continue
        newsubdir = self._subdirprefix+plotterFolder.getSelectionName(dqmSubFolder)
        newdir = os.path.join(self._newdir, newsubdir)
        if not os.path.exists(newdir):
            os.makedirs(newdir, exist_ok=True)

        plotterFolder.create(self._openFiles, self._labels, dqmSubFolder)
        while len(active_proc)>=plottingProcess:
          time.sleep(0.1)
          active_proc = [p for p in active_proc if p.is_alive()]
        p = multiprocessing.Process(target=self._doPlots, args=(plotterFolder, dqmSubFolder, newsubdir, newdir, iProc, return_dict))
        proc.append((plotterFolder, dqmSubFolder, p))
        active_proc.append(p)
        p.start()
        iProc += 1

    for i in range(iProc):
        proc[i][2].join()
        if len(return_dict[i]) > 0:
            self._htmlReport.addPlots(proc[i][0], proc[i][1], return_dict[i])

_findDuplicates()

def validation._findDuplicates (   lst )

private

Definition at line 1180 of file validation.py.

Referenced by validation.Validation._doPlots(), validation.SimpleValidation._doPlots(), validation.SeparateValidation._doPlots(), validation.Validation._doPlotsFastFull(), and validation.Validation._doPlotsPileup().

def _findDuplicates(lst):
    found = set()
    found2 = set()
    for x in lst:
        if x in found:
            found2.add(x)
        else:
            found.add(x)
    return list(found2)

_getGlobalTag()

def validation._getGlobalTag (   sample,   release  )

private

Get a GlobalTag.

Arguments:
sample  -- Sample object
release -- CMSSW release string

Definition at line 361 of file validation.py.

References print().

Referenced by validation.Validation._doPlots(), validation.Validation._doPlotsFastFull(), validation.Validation._doPlotsPileup(), validation.Validation._getRefFileAndSelection(), validation.Sample.datasetpattern(), and validation.Sample.filename().

def _getGlobalTag(sample, release):
    """Get a GlobalTag.

    Arguments:
    sample  -- Sample object
    release -- CMSSW release string
    """
    if not release in _globalTags:
        print("Release %s not found from globaltag map in validation.py" % release)
        sys.exit(1)
    gtmap = _globalTags[release]
    selectedGT = None
    if sample.hasOverrideGlobalTag():
        ogt = sample.overrideGlobalTag()
        if release in ogt:
            gtmap = _globalTags[ogt[release]]
    scenario = ""
    if sample.hasScenario():
        scenario = sample.scenario()
    sims = []
    if sample.fullsim():
        if sample.pileupEnabled():
            sim = "fullsim_"+sample.pileupType()
            sims.extend([
                sim+"_PU%d"%sample.pileupNumber(),
                sim
            ])
    elif sample.fastsim():
        sim = "fastsim"
        if sample.pileupEnabled():
            sim += "_"+sample.pileupType()
            sims.append(sim+"_PU%d"%sample.pileupNumber())
        sims.append(sim)

    selectedGT = None
    # First try with scenario+simulation
    if scenario != "":
        for sim in sims:
            selectedGT = gtmap.get(scenario+"_"+sim, None)
            if selectedGT is not None:
                break
        # Then with scenario (but only if sample specifies a scenario)
        if selectedGT is None:
            selectedGT = gtmap.get(scenario, None)
    # Then with simulation
    if selectedGT is None:
        for sim in sims:
            selectedGT = gtmap.get(sim, None)
            if selectedGT is not None:
                break
    # Finally default
    if selectedGT is None:
        selectedGT = gtmap["default"]

    if isinstance(selectedGT, dict):
        return selectedGT.get(sample.name(), selectedGT["default"])
    else:
        return selectedGT

# Mapping from release series to RelVal download URLs

_getRelValUrl()

def validation._getRelValUrl (   release )

private

Get RelVal download URL for a given release.

Definition at line 453 of file validation.py.

References print().

Referenced by validation.Validation.download().

def _getRelValUrl(release):
    """Get RelVal download URL for a given release."""
    version_re = re.compile("CMSSW_(?P<X>\\d+)_(?P<Y>\\d+)")
    m = version_re.search(release)
    if not m:
        raise Exception("Regex %s does not match to release version %s" % (version_re.pattern, release))
    version = "%s_%s_X" % (m.group("X"), m.group("Y"))
    if not version in _relvalUrls:
        print("No RelVal URL for version %s, please update _relvalUrls" % version)
        sys.exit(1)
    return _relvalUrls[version]

_processPlotsForSample()

def validation._processPlotsForSample (   plotterFolder,   sample  )

private

Definition at line 465 of file validation.py.

Referenced by validation.Validation._doFastsimFastVsFullPlots(), validation.Validation._doPhase2PileupPlots(), and _doPlotsForPlotter().

def _processPlotsForSample(plotterFolder, sample):
    if plotterFolder.onlyForPileup() and not sample.pileupEnabled():
        return False
    if plotterFolder.onlyForElectron() and not sample.doElectron():
        return False
    if plotterFolder.onlyForConversion() and not sample.doConversion():
        return False
    if plotterFolder.onlyForBHadron() and not sample.doBHadron():
        return False
    return True

_stripRelease()

def validation._stripRelease (   release )

private

Definition at line 354 of file validation.py.

Referenced by validation.Validation._doPlots(), validation.Validation._doPlotsFastFull(), validation.Validation._doPlotsPileup(), and validation.Sample.filename().

def _stripRelease(release):
    for pf in _releasePostfixes:
        if pf in release:
            return _stripRelease(release.replace(pf, ""))
    return release

Variable Documentation

_doBHadronSamples

validation._doBHadronSamples

private

Definition at line 449 of file validation.py.

_doConversionSamples

validation._doConversionSamples

private

Definition at line 445 of file validation.py.

_doElectronSamples

validation._doElectronSamples

private

Definition at line 440 of file validation.py.

_globalTags

validation._globalTags

private

Definition at line 20 of file validation.py.

_releasePostfixes

validation._releasePostfixes

private

Definition at line 352 of file validation.py.

_relvalUrls

validation._relvalUrls

private

Definition at line 421 of file validation.py.

IgnoreCommandLineOptions

validation.IgnoreCommandLineOptions

Definition at line 14 of file validation.py.