genericValidation.py

Go to the documentation of this file.

from abc import ABCMeta, abstractmethod, abstractproperty
import os
import re
import json
import globalDictionaries
import configTemplates
from dataset import Dataset
from helperFunctions import replaceByMap, addIndex, getCommandOutput2, boolfromstring, pythonboolstring
from TkAlExceptions import AllInOneError

class ValidationMetaClass(ABCMeta):
    sets = ["mandatories", "optionals", "needpackages"]
    dicts = ["defaults"]
    def __new__(cls, clsname, bases, dct):
        for setname in cls.sets:
            if setname not in dct: dct[setname] = set()
            dct[setname] = set.union(dct[setname], *(getattr(base, setname) for base in bases if hasattr(base, setname)))

        for dictname in cls.dicts:
            if dictname not in dct: dct[dictname] = {}
            for base in bases:
                if not hasattr(base, dictname): continue
                newdict = getattr(base, dictname)
                for key in set(newdict) & set(dct[dictname]):
                    if newdict[key] != dct[dictname][key]:
                        raise ValueError("Inconsistent values of defaults[{}]: {}, {}".format(key, newdict[key], dct[dictname][key]))
                dct[dictname].update(newdict)

        for setname in cls.sets:      #e.g. removemandatories, used in preexistingvalidation
                                      #use with caution
            if "remove"+setname not in dct: dct["remove"+setname] = set()
            dct["remove"+setname] = set.union(dct["remove"+setname], *(getattr(base, "remove"+setname) for base in bases if hasattr(base, "remove"+setname)))

            dct[setname] -= dct["remove"+setname]

        return super(ValidationMetaClass, cls).__new__(cls, clsname, bases, dct)

class GenericValidation(object):
    __metaclass__ = ValidationMetaClass
    defaultReferenceName = "DEFAULT"
    mandatories = set()
    defaults = {
                "cmssw":        os.environ['CMSSW_BASE'],
                "parallelJobs": "1",
                "jobid":        "",
                "needsproxy":   "false",
               }
    needpackages = {"Alignment/OfflineValidation"}
    optionals = {"jobmode"}

    def __init__(self, valName, alignment, config):
        import random
        self.name = valName
        self.alignmentToValidate = alignment
        self.general = config.getGeneral()
        self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
        self.configFiles = []
        self.config = config
        self.jobid = ""

        theUpdate = config.getResultingSection(self.valType+":"+self.name,
                                               defaultDict = self.defaults,
                                               demandPars = self.mandatories)
        self.general.update(theUpdate)
        self.jobmode = self.general["jobmode"]
        self.NJobs = int(self.general["parallelJobs"])
        self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")

        # limit maximum number of parallel jobs to 40
        # (each output file is approximately 20MB)
        maximumNumberJobs = 40
        if self.NJobs > maximumNumberJobs:
            msg = ("Maximum allowed number of parallel jobs "
                   +str(maximumNumberJobs)+" exceeded!!!")
            raise AllInOneError(msg)
        if self.NJobs > 1 and not isinstance(self, ParallelValidation):
            raise AllInOneError("Parallel jobs not implemented for {}!\n"
                                "Please set parallelJobs = 1.".format(type(self).__name__))

        self.jobid = self.general["jobid"]
        if self.jobid:
            try:  #make sure it's actually a valid jobid
                output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
                if "is not found" in output: raise RuntimeError
            except RuntimeError:
                raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)

        self.cmssw = self.general["cmssw"]
        badcharacters = r"\'"
        for character in badcharacters:
            if character in self.cmssw:
                raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
                                    "path name.  If you really have it in such a ridiculously named location,\n"
                                    "try making a symbolic link somewhere with a decent name.")
        try:
            os.listdir(self.cmssw)
        except OSError:
            raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')

        if self.cmssw == os.environ["CMSSW_BASE"]:
            self.scramarch = os.environ["SCRAM_ARCH"]
            self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
        else:
            command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
                       ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
            commandoutput = getCommandOutput2(command).split('\n')
            self.cmssw = commandoutput[0]
            self.scramarch = commandoutput[1]
            self.cmsswreleasebase = commandoutput[2]

        self.packages = {}
        for package in self.needpackages:
            for placetolook in self.cmssw, self.cmsswreleasebase:
                pkgpath = os.path.join(placetolook, "src", package)
                if os.path.exists(pkgpath):
                    self.packages[package] = pkgpath
                    break
            else:
                raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))

        self.AutoAlternates = True
        if config.has_option("alternateTemplates","AutoAlternates"):
            try:
                self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
            except ValueError:
                raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))

        knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
        ignoreOpts = []
        config.checkInput(self.valType+":"+self.name,
                          knownSimpleOptions = knownOpts,
                          ignoreOptions = ignoreOpts)

    def getRepMap(self, alignment = None):
        from plottingOptions import PlottingOptions
        if alignment == None:
            alignment = self.alignmentToValidate
        try:
            result = PlottingOptions(self.config, self.valType)
        except KeyError:
            result = {}
        result.update(alignment.getRepMap())
        result.update(self.general)
        result.update({
                "workdir": os.path.join(self.general["workdir"],
                                        self.randomWorkdirPart),
                "datadir": self.general["datadir"],
                "logdir": self.general["logdir"],
                "CommandLineTemplate": ("#run configfile and post-proccess it\n"
                                        "cmsRun %(cfgFile)s\n"
                                        "%(postProcess)s "),
                "CMSSW_BASE": self.cmssw,
                "SCRAM_ARCH": self.scramarch,
                "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
                "alignmentName": alignment.name,
                "condLoad": alignment.getConditions(),
                "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
                })
        result.update(self.packages)
        return result

    @abstractproperty
    def filesToCompare(self):
        pass

    def getCompareStrings( self, requestId = None, plain = False ):
        result = {}
        repMap = self.getRepMap().copy()
        for validationId in self.filesToCompare:
            repMap["file"] = self.filesToCompare[ validationId ]
            if repMap["file"].startswith( "/castor/" ):
                repMap["file"] = "rfio:%(file)s"%repMap
            elif repMap["file"].startswith( "/store/" ):
                repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
            if plain:
                result[validationId]=repMap["file"]
            else:
                result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
        if requestId == None:
            return result
        else:
            if not "." in requestId:
                requestId += ".%s"%self.defaultReferenceName
            if not requestId.split(".")[-1] in result:
                msg = ("could not find %s in reference Objects!"
                       %requestId.split(".")[-1])
                raise AllInOneError(msg)
            return result[ requestId.split(".")[-1] ]

    def createFiles(self, fileContents, path, repMap = None, repMaps = None):
        """repMap: single map for all files
           repMaps: a dict, with the filenames as the keys"""
        if repMap is not None and repMaps is not None:
            raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
        result = []
        for fileName in fileContents:
            filePath = os.path.join(path, fileName)
            result.append(filePath)

            for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
                theFile = open( filePathi, "w" )
                fileContentsi = fileContents[ fileName ]
                if repMaps is not None:
                    repMap = repMaps[fileName]
                if repMap is not None:
                    repMap.update({"nIndex": str(i)})
                    fileContentsi = replaceByMap(fileContentsi, repMap)
                theFile.write( fileContentsi )
                theFile.close()

        return result

    def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
        self.configFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        if not schedule == None:
            schedule = [os.path.join( path, cfgName) for cfgName in schedule]
            for cfgName in schedule:
                if not cfgName in self.configFiles:
                    msg = ("scheduled %s missing in generated configfiles: %s"
                           %(cfgName, self.configFiles))
                    raise AllInOneError(msg)
            for cfgName in self.configFiles:
                if not cfgName in schedule:
                    msg = ("generated configuration %s not scheduled: %s"
                           %(cfgName, schedule))
                    raise AllInOneError(msg)
            self.configFiles = schedule
        return self.configFiles

    def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
        self.scriptFiles = self.createFiles(fileContents,
                                            path, repMap = repMap, repMaps = repMaps)
        for script in self.scriptFiles:
            for scriptwithindex in addIndex(script, self.NJobs):
                os.chmod(scriptwithindex,0o755)
        return self.scriptFiles

    def createCrabCfg(self, fileContents, path ):
        if self.NJobs > 1:
            msg =  ("jobmode 'crab' not supported for parallel validation."
                    " Please set parallelJobs = 1.")
            raise AllInOneError(msg)
        self.crabConfigFiles = self.createFiles(fileContents, path)
        return self.crabConfigFiles


class GenericValidationData(GenericValidation):
    """
    Subclass of `GenericValidation` which is the base for validations using
    datasets.
    """
    needParentFiles = False
    mandatories = {"dataset", "maxevents"}
    defaults = {
                "runRange": "",
                "firstRun": "",
                "lastRun": "",
                "begin": "",
                "end": "",
                "JSON": "",
                "dasinstance": "prod/global",
                "ttrhbuilder":"WithAngleAndTemplate",
                "usepixelqualityflag": "True",
               }
    optionals = {"magneticfield"}
    
    def __init__(self, valName, alignment, config):
        """
        This method adds additional items to the `self.general` dictionary
        which are only needed for validations using datasets.
        
        Arguments:
        - `valName`: String which identifies individual validation instances
        - `alignment`: `Alignment` instance to validate
        - `config`: `BetterConfigParser` instance which includes the
                    configuration of the validations
        """

        super(GenericValidationData, self).__init__(valName, alignment, config)

        # if maxevents is not specified, cannot calculate number of events for
        # each parallel job, and therefore running only a single job
        if int( self.general["maxevents"] ) < 0 and self.NJobs > 1:
            msg = ("Maximum number of events (maxevents) not specified: "
                   "cannot use parallel jobs.")
            raise AllInOneError(msg)
        if int( self.general["maxevents"] ) / self.NJobs != float( self.general["maxevents"] ) / self.NJobs:
            msg = ("maxevents has to be divisible by parallelJobs")
            raise AllInOneError(msg)

        tryPredefinedFirst = (not self.jobmode.split( ',' )[0] == "crab" and self.general["JSON"]    == ""
                              and self.general["firstRun"] == ""         and self.general["lastRun"] == ""
                              and self.general["begin"]    == ""         and self.general["end"]     == "")

        if self.general["dataset"] not in globalDictionaries.usedDatasets:
            globalDictionaries.usedDatasets[self.general["dataset"]] = {}

        if self.cmssw not in globalDictionaries.usedDatasets[self.general["dataset"]]:
            if globalDictionaries.usedDatasets[self.general["dataset"]] != {}:
                print ("Warning: you use the same dataset '%s' in multiple cmssw releases.\n"
                       "This is allowed, but make sure it's not a mistake") % self.general["dataset"]
            globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw] = {False: None, True: None}

        Bfield = self.general.get("magneticfield", None)
        if globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] is None:
            dataset = Dataset(
                self.general["dataset"], tryPredefinedFirst = tryPredefinedFirst,
                cmssw = self.cmssw, cmsswrelease = self.cmsswreleasebase, magneticfield = Bfield,
                dasinstance = self.general["dasinstance"])
            globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst] = dataset
            if tryPredefinedFirst and not dataset.predefined():                              #No point finding the data twice in that case
                globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][False] = dataset

        self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]][self.cmssw][tryPredefinedFirst]
        self.general["magneticField"] = self.dataset.magneticField()
        self.general["defaultMagneticField"] = "MagneticField"
        if self.general["magneticField"] == "unknown":
            print "Could not get the magnetic field for this dataset."
            print "Using the default: ", self.general["defaultMagneticField"]
            self.general["magneticField"] = '.oO[defaultMagneticField]Oo.'
        
        if not self.jobmode.split( ',' )[0] == "crab":
            try:
                self.general["datasetDefinition"] = self.dataset.datasetSnippet(
                    jsonPath = self.general["JSON"],
                    firstRun = self.general["firstRun"],
                    lastRun = self.general["lastRun"],
                    begin = self.general["begin"],
                    end = self.general["end"],
                    parent = self.needParentFiles )
            except AllInOneError as e:
                msg = "In section [%s:%s]: "%(self.valType, self.name)
                msg += str(e)
                raise AllInOneError(msg)
        else:
            if self.dataset.predefined():
                msg = ("For jobmode 'crab' you cannot use predefined datasets "
                       "(in your case: '%s')."%( self.dataset.name() ))
                raise AllInOneError( msg )
            try:
                theUpdate = config.getResultingSection(self.valType+":"+self.name,
                                                       demandPars = ["parallelJobs"])
            except AllInOneError as e:
                msg = str(e)[:-1]+" when using 'jobmode: crab'."
                raise AllInOneError(msg)
            self.general.update(theUpdate)
            if self.general["begin"] or self.general["end"]:
                ( self.general["begin"],
                  self.general["end"],
                  self.general["firstRun"],
                  self.general["lastRun"] ) = self.dataset.convertTimeToRun(
                    firstRun = self.general["firstRun"],
                    lastRun = self.general["lastRun"],
                    begin = self.general["begin"],
                    end = self.general["end"],
                    shortTuple = False)
                if self.general["begin"] == None:
                    self.general["begin"] = ""
                if self.general["end"] == None:
                    self.general["end"] = ""
                self.general["firstRun"] = str( self.general["firstRun"] )
                self.general["lastRun"] = str( self.general["lastRun"] )
            if ( not self.general["firstRun"] ) and \
                   ( self.general["end"] or self.general["lastRun"] ):
                self.general["firstRun"] = str(
                    self.dataset.runList()[0]["run_number"])
            if ( not self.general["lastRun"] ) and \
                   ( self.general["begin"] or self.general["firstRun"] ):
                self.general["lastRun"] = str(
                    self.dataset.runList()[-1]["run_number"])
            if self.general["firstRun"] and self.general["lastRun"]:
                if int(self.general["firstRun"]) > int(self.general["lastRun"]):
                    msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
                            "chosen is greater than the upper time/runrange limit "
                            "('end'/'lastRun').")
                    raise AllInOneError( msg )
                self.general["runRange"] = (self.general["firstRun"]
                                            + '-' + self.general["lastRun"])
            try:
                self.general["datasetDefinition"] = self.dataset.datasetSnippet(
                    jsonPath = self.general["JSON"],
                    firstRun = self.general["firstRun"],
                    lastRun = self.general["lastRun"],
                    begin = self.general["begin"],
                    end = self.general["end"],
                    crab = True )
            except AllInOneError as e:
                msg = "In section [%s:%s]: "%(self.valType, self.name)
                msg += str( e )
                raise AllInOneError( msg )

        self.general["usepixelqualityflag"] = pythonboolstring(self.general["usepixelqualityflag"], "usepixelqualityflag")

    def getRepMap(self, alignment = None):
        result = super(GenericValidationData, self).getRepMap(alignment)
        outputfile = os.path.expandvars(replaceByMap(
                           "%s_%s_.oO[name]Oo..root" % (self.outputBaseName, self.name)
                                 , result))
        resultfile = os.path.expandvars(replaceByMap(("/store/caf/user/$USER/.oO[eosdir]Oo./" +
                           "%s_%s_.oO[name]Oo..root" % (self.resultBaseName, self.name))
                                 , result))
        result.update({
                "resultFile": ".oO[resultFiles[.oO[nIndex]Oo.]]Oo.",
                "resultFiles": addIndex(resultfile, self.NJobs),
                "finalResultFile": resultfile,
                "outputFile": ".oO[outputFiles[.oO[nIndex]Oo.]]Oo.",
                "outputFiles": addIndex(outputfile, self.NJobs),
                "finalOutputFile": outputfile,
                "ProcessName": self.ProcessName,
                "Bookkeeping": self.Bookkeeping,
                "LoadBasicModules": self.LoadBasicModules,
                "TrackSelectionRefitting": self.TrackSelectionRefitting,
                "ValidationConfig": self.ValidationTemplate,
                "FileOutputTemplate": self.FileOutputTemplate,
                "DefinePath": self.DefinePath,
                })
        return result

    @property
    def cfgName(self):
        return "%s.%s.%s_cfg.py"%( self.configBaseName, self.name,
                                   self.alignmentToValidate.name )
    @abstractproperty
    def ProcessName(self):
        pass

    @property
    def cfgTemplate(self):
        return configTemplates.cfgTemplate

    @abstractproperty
    def ValidationTemplate(self):
        pass

    @property
    def filesToCompare(self):
        return {self.defaultReferenceName: self.getRepMap()["finalResultFile"]}

    def createConfiguration(self, path ):
        repMap = self.getRepMap()
        cfgs = {self.cfgName: self.cfgTemplate}
        super(GenericValidationData, self).createConfiguration(cfgs, path, repMap=repMap)

    def createScript(self, path, template = configTemplates.scriptTemplate, downloadFiles=[], repMap = None, repMaps = None):
        scriptName = "%s.%s.%s.sh"%(self.scriptBaseName, self.name,
                                    self.alignmentToValidate.name )
        if repMap is None and repMaps is None:
            repMap = self.getRepMap()
            repMap["CommandLine"]=""
            for cfg in self.configFiles:
                repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":addIndex(cfg, self.NJobs, ".oO[nIndex]Oo."),
                                                      "postProcess":""
                                                     }
        scripts = {scriptName: template}
        return super(GenericValidationData, self).createScript(scripts, path, downloadFiles = downloadFiles,
                                                               repMap = repMap, repMaps = repMaps)

    def createCrabCfg(self, path, crabCfgBaseName):
        """
        Method which creates a `crab.cfg` for a validation on datasets.
        
        Arguments:
        - `path`: Path at which the file will be stored.
        - `crabCfgBaseName`: String which depends on the actual type of
                             validation calling this method.
        """
        crabCfgName = "crab.%s.%s.%s.cfg"%( crabCfgBaseName, self.name,
                                            self.alignmentToValidate.name )
        repMap = self.getRepMap()
        repMap["script"] = "dummy_script.sh"
        # repMap["crabOutputDir"] = os.path.basename( path )
        repMap["crabWorkingDir"] = crabCfgName.split( '.cfg' )[0]
        self.crabWorkingDir = repMap["crabWorkingDir"]
        repMap["numberOfJobs"] = self.general["parallelJobs"]
        repMap["cfgFile"] = self.configFiles[0]
        repMap["queue"] = self.jobmode.split( ',' )[1].split( '-q' )[1]
        if self.dataset.dataType() == "mc":
            repMap["McOrData"] = "events = .oO[nEvents]Oo."
        elif self.dataset.dataType() == "data":
            repMap["McOrData"] = "lumis = -1"
            if self.jobmode.split( ',' )[0] == "crab":
                print ("For jobmode 'crab' the parameter 'maxevents' will be "
                       "ignored and all events will be processed.")
        else:
            raise AllInOneError("Unknown data type!  Can't run in crab mode")
        crabCfg = {crabCfgName: replaceByMap( configTemplates.crabCfgTemplate,
                                              repMap ) }
        return super(GenericValidationData, self).createCrabCfg( crabCfg, path )

    @property
    def Bookkeeping(self):
        return configTemplates.Bookkeeping
    @property
    def LoadBasicModules(self):
        return configTemplates.LoadBasicModules
    @abstractproperty
    def TrackSelectionRefitting(self):
        pass
    @property
    def FileOutputTemplate(self):
        return configTemplates.FileOutputTemplate
    @abstractproperty
    def DefinePath(self):
        pass

class GenericValidationData_CTSR(GenericValidationData):
    #common track selection and refitting
    defaults = {
        "momentumconstraint": "None",
        "openmasswindow": "False",
        "cosmicsdecomode": "True",
        "removetrackhitfiltercommands": "",
        "appendtrackhitfiltercommands": "",
    }
    def getRepMap(self, alignment=None):
        result = super(GenericValidationData_CTSR, self).getRepMap(alignment)

        from trackSplittingValidation import TrackSplittingValidation
        result.update({
            "ValidationSequence": self.ValidationSequence,
            "istracksplitting": str(isinstance(self, TrackSplittingValidation)),
            "cosmics0T": str(self.cosmics0T),
            "use_d0cut": str(self.use_d0cut),
        })

        commands = []
        for removeorappend in "remove", "append":
            optionname = removeorappend + "trackhitfiltercommands"
            if result[optionname]:
                for command in result[optionname].split(","):
                    command = command.strip()
                    commands.append('process.TrackerTrackHitFilter.commands.{}("{}")'.format(removeorappend, command))
        result["trackhitfiltercommands"] = "\n".join(commands)

        return result
    @property
    def use_d0cut(self):
        return "Cosmics" not in self.general["trackcollection"]  #use it for collisions only
    @property
    def TrackSelectionRefitting(self):
        return configTemplates.CommonTrackSelectionRefitting
    @property
    def DefinePath(self):
        return configTemplates.DefinePath_CommonSelectionRefitting
    @abstractproperty
    def ValidationSequence(self):
        pass
    @property
    def cosmics0T(self):
        if "Cosmics" not in self.general["trackcollection"]: return False
        Bfield = self.dataset.magneticFieldForRun()
        if Bfield < 0.5: return True
        if isinstance(Bfield, str):
            if "unknown " in Bfield:
                msg = Bfield.replace("unknown ","",1)
            elif Bfield == "unknown":
                msg = "Can't get the B field for %s." % self.dataset.name()
            else:
                msg = "B field = {}???".format(Bfield)
            raise AllInOneError(msg + "\n"
                                "To use this dataset, specify magneticfield = [value] in your .ini config file.")
        return False

class ParallelValidation(GenericValidation):
    @classmethod
    def initMerge(cls):
        return ""
    @abstractmethod
    def appendToMerge(self):
        pass

    @classmethod
    def doInitMerge(cls):
        from plottingOptions import PlottingOptions
        result = cls.initMerge()
        result = replaceByMap(result, PlottingOptions(None, cls))
        if result and result[-1] != "\n": result += "\n"
        return result
    def doMerge(self):
        result = self.appendToMerge()
        if result[-1] != "\n": result += "\n"
        result += ("if [[ tmpMergeRetCode -eq 0 ]]; then\n"
                   "  xrdcp -f .oO[finalOutputFile]Oo. root://eoscms//eos/cms.oO[finalResultFile]Oo.\n"
                   "fi\n"
                   "if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then\n"
                   "  mergeRetCode=${tmpMergeRetCode}\n"
                   "fi\n")
        result = replaceByMap(result, self.getRepMap())
        return result

class ValidationWithPlots(GenericValidation):
    @classmethod
    def runPlots(cls, validations):
        return ("rfcp .oO[plottingscriptpath]Oo. .\n"
                "root -x -b -q .oO[plottingscriptname]Oo.++")
    @abstractmethod
    def appendToPlots(self):
        pass
    @abstractmethod
    def plottingscriptname(cls):
        """override with a classmethod"""
    @abstractmethod
    def plottingscripttemplate(cls):
        """override with a classmethod"""
    @abstractmethod
    def plotsdirname(cls):
        """override with a classmethod"""

    @classmethod
    def doRunPlots(cls, validations):
        from plottingOptions import PlottingOptions
        cls.createPlottingScript(validations)
        result = cls.runPlots(validations)
        result = replaceByMap(result, PlottingOptions(None, cls))
        if result and result[-1] != "\n": result += "\n"
        return result
    @classmethod
    def createPlottingScript(cls, validations):
        from plottingOptions import PlottingOptions
        repmap = PlottingOptions(None, cls).copy()
        filename = replaceByMap(".oO[plottingscriptpath]Oo.", repmap)
        repmap["PlottingInstantiation"] = "\n".join(
                                                    replaceByMap(v.appendToPlots(), v.getRepMap()).rstrip("\n")
                                                         for v in validations
                                                   )
        plottingscript = replaceByMap(cls.plottingscripttemplate(), repmap)
        with open(filename, 'w') as f:
            f.write(plottingscript)

class ValidationWithPlotsSummaryBase(ValidationWithPlots):
    class SummaryItem(object):
        def __init__(self, name, values, format=None, latexname=None, latexformat=None):
            """
            name:        name of the summary item, goes on top of the column
            values:      value for each alignment (in order of rows)
            format:      python format string (default: {:.3g}, meaning up to 3 significant digits)
            latexname:   name in latex form, e.g. if name=sigma you might want latexname=\sigma (default: name)
            latexformat: format for latex (default: format)
            """
            if format is None: format = "{:.3g}"
            if latexname is None: latexname = name
            if latexformat is None: latexformat = format

            self.__name = name
            self.__values = values
            self.__format = format
            self.__latexname = latexname
            self.__latexformat = latexformat

        def name(self, latex=False):
            if latex:
                return self.__latexname
            else:
                return self.__name

        def format(self, value, latex=False):
            if latex:
                fmt = self.__latexformat
            else:
                fmt = self.__format
            if re.match(".*[{][^}]*[fg][}].*", fmt):
                value = float(value)
            return fmt.format(value)

        def values(self, latex=False):
            result = [self.format(v, latex=latex) for v in self.__values]
            return result

        def value(self, i, latex):
            return self.values(latex)[i]

    @abstractmethod
    def getsummaryitems(cls, folder):
        """override with a classmethod that returns a list of SummaryItems
           based on the plots saved in folder"""

    __summaryitems = None
    __lastfolder = None

    @classmethod
    def summaryitemsstring(cls, folder=None, latex=False, transpose=True):
        if folder is None: folder = cls.plotsdirname()
        if folder.startswith( "/castor/" ):
            folder = "rfio:%(file)s"%repMap
        elif folder.startswith( "/store/" ):
            folder = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap

        if cls.__summaryitems is None or cls.__lastfolder != folder:
            cls.__lastfolder = folder
            cls.__summaryitems = cls.getsummaryitems(folder)

        summaryitems = cls.__summaryitems

        if not summaryitems:
            raise AllInOneError("No summary items!")
        size = {len(_.values(latex)) for _ in summaryitems}
        if len(size) != 1:
            raise AllInOneError("Some summary items have different numbers of values\n{}".format(size))
        size = size.pop()

        if transpose:
            columnwidths = ([max(len(_.name(latex)) for _ in summaryitems)]
                          + [max(len(_.value(i, latex)) for _ in summaryitems) for i in range(size)])
        else:
            columnwidths = [max(len(entry) for entry in [_.name(latex)] + _.values(latex)) for _ in summaryitems]

        if latex:
            join = " & "
        else:
            join = " "
        row = join.join("{{:{}}}".format(width) for width in columnwidths)

        if transpose:
            rows = [row.format(*[_.name(latex)]+_.values(latex)) for _ in summaryitems]
        else:
            rows = []
            rows.append(row.format(*(_.name for _ in summaryitems)))
            for i in range(size):
                rows.append(row.format(*(_.value(i, latex) for _ in summaryitems)))

        if latex:
            join = " \\\\\n"
        else:
            join = "\n"
        result = join.join(rows)
        if latex:
            result = (r"\begin{{tabular}}{{{}}}".format("|" + "|".join("c"*(len(columnwidths))) + "|") + "\n"
                         + result + "\n"
                         + r"\end{tabular}")
        return result

    @classmethod
    def printsummaryitems(cls, *args, **kwargs):
        print cls.summaryitemsstring(*args, **kwargs)
    @classmethod
    def writesummaryitems(cls, filename, *args, **kwargs):
        with open(filename, "w") as f:
            f.write(cls.summaryitemsstring(*args, **kwargs)+"\n")

class ValidationWithPlotsSummary(ValidationWithPlotsSummaryBase):
    @classmethod
    def getsummaryitems(cls, folder):
        result = []
        with open(os.path.join(folder, "{}Summary.txt".format(cls.__name__))) as f:
            for line in f:
                split = line.rstrip("\n").split("\t")
                kwargs = {}
                for thing in split[:]:
                    if thing.startswith("format="):
                        kwargs["format"] = thing.replace("format=", "", 1)
                        split.remove(thing)
                    if thing.startswith("latexname="):
                        kwargs["latexname"] = thing.replace("latexname=", "", 1)
                        split.remove(thing)
                    if thing.startswith("latexformat="):
                        kwargs["latexformat"] = thing.replace("latexformat=", "", 1)
                        split.remove(thing)

                name = split[0]
                values = split[1:]
                result.append(cls.SummaryItem(name, values, **kwargs))
        return result

class ValidationWithComparison(GenericValidation):
    @classmethod
    def doComparison(cls, validations):
        from plottingOptions import PlottingOptions
        repmap = PlottingOptions(None, cls).copy()
        repmap["compareStrings"] = " , ".join(v.getCompareStrings("OfflineValidation") for v in validations)
        repmap["compareStringsPlain"] = " , ".join(v.getCompareStrings("OfflineValidation", True) for v in validations)
        comparison = replaceByMap(cls.comparisontemplate(), repmap)
        return comparison

    @classmethod
    def comparisontemplate(cls):
        return configTemplates.compareAlignmentsExecution
    @classmethod
    def comparealignmentspath(cls):
        return ".oO[Alignment/OfflineValidation]Oo./scripts/.oO[compareAlignmentsName]Oo."
    @abstractmethod
    def comparealignmentsname(cls):
        """classmethod"""

class ValidationForPresentation(ValidationWithPlots):
    @abstractmethod
    def presentationsubsections(cls):
        """classmethod"""