00001 import os
00002 import globalDictionaries
00003 import configTemplates
00004 from dataset import Dataset
00005 from helperFunctions import replaceByMap
00006 from TkAlExceptions import AllInOneError
00007
00008
00009 class GenericValidation:
00010 defaultReferenceName = "DEFAULT"
00011 def __init__(self, valName, alignment, config):
00012 import random
00013 self.name = valName
00014 self.alignmentToValidate = alignment
00015 self.general = config.getGeneral()
00016 self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
00017 self.configFiles = []
00018 self.filesToCompare = {}
00019 self.jobmode = self.general["jobmode"]
00020 self.config = config
00021
00022 def getRepMap(self, alignment = None):
00023 if alignment == None:
00024 alignment = self.alignmentToValidate
00025 result = alignment.getRepMap()
00026 result.update( self.general )
00027 result.update({
00028 "workdir": os.path.join(self.general["workdir"],
00029 self.randomWorkdirPart),
00030 "datadir": self.general["datadir"],
00031 "logdir": self.general["logdir"],
00032 "CommandLineTemplate": ("#run configfile and post-proccess it\n"
00033 "cmsRun %(cfgFile)s\n"
00034 "%(postProcess)s "),
00035 "CMSSW_BASE": os.environ['CMSSW_BASE'],
00036 "SCRAM_ARCH": os.environ['SCRAM_ARCH'],
00037 "alignmentName": alignment.name,
00038 "condLoad": alignment.getConditions()
00039 })
00040 return result
00041
00042 def getCompareStrings( self, requestId = None ):
00043 result = {}
00044 repMap = self.alignmentToValidate.getRepMap()
00045 for validationId in self.filesToCompare:
00046 repMap["file"] = self.filesToCompare[ validationId ]
00047 if repMap["file"].startswith( "/castor/" ):
00048 repMap["file"] = "rfio:%(file)s"%repMap
00049 elif repMap["file"].startswith( "/store/" ):
00050 repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
00051 result[validationId]= "%(file)s=%(name)s|%(color)s|%(style)s"%repMap
00052 if requestId == None:
00053 return result
00054 else:
00055 if not "." in requestId:
00056 requestId += ".%s"%GenericValidation.defaultReferenceName
00057 if not requestId.split(".")[-1] in result:
00058 msg = ("could not find %s in reference Objects!"
00059 %requestId.split(".")[-1])
00060 raise AllInOneError(msg)
00061 return result[ requestId.split(".")[-1] ]
00062
00063 def createFiles( self, fileContents, path ):
00064 result = []
00065 for fileName in fileContents:
00066 filePath = os.path.join( path, fileName)
00067 theFile = open( filePath, "w" )
00068 theFile.write( fileContents[ fileName ] )
00069 theFile.close()
00070 result.append( filePath )
00071 return result
00072
00073 def createConfiguration(self, fileContents, path, schedule= None):
00074 self.configFiles = GenericValidation.createFiles(self, fileContents,
00075 path)
00076 if not schedule == None:
00077 schedule = [os.path.join( path, cfgName) for cfgName in schedule]
00078 for cfgName in schedule:
00079 if not cfgName in self.configFiles:
00080 msg = ("scheduled %s missing in generated configfiles: %s"
00081 %(cfgName, self.configFiles))
00082 raise AllInOneError(msg)
00083 for cfgName in self.configFiles:
00084 if not cfgName in schedule:
00085 msg = ("generated configuration %s not scheduled: %s"
00086 %(cfgName, schedule))
00087 raise AllInOneError(msg)
00088 self.configFiles = schedule
00089 return self.configFiles
00090
00091 def createScript(self, fileContents, path, downloadFiles=[] ):
00092 self.scriptFiles = GenericValidation.createFiles(self, fileContents,
00093 path)
00094 for script in self.scriptFiles:
00095 os.chmod(script,0755)
00096 return self.scriptFiles
00097
00098 def createCrabCfg(self, fileContents, path ):
00099 self.crabConfigFiles = GenericValidation.createFiles(self, fileContents,
00100 path)
00101 return self.crabConfigFiles
00102
00103
00104 class GenericValidationData(GenericValidation):
00105 """
00106 Subclass of `GenericValidation` which is the base for validations using
00107 datasets.
00108 """
00109
00110 def __init__(self, valName, alignment, config, valType,
00111 addDefaults = {}, addMandatories=[]):
00112 """
00113 This method adds additional items to the `self.general` dictionary
00114 which are only needed for validations using datasets.
00115
00116 Arguments:
00117 - `valName`: String which identifies individual validation instances
00118 - `alignment`: `Alignment` instance to validate
00119 - `config`: `BetterConfigParser` instance which includes the
00120 configuration of the validations
00121 - `valType`: String which specifies the type of validation
00122 - `addDefaults`: Dictionary which contains default values for individual
00123 validations in addition to the general default values
00124 - `addMandatories`: List which contains mandatory parameters for
00125 individual validations in addition to the general
00126 mandatory parameters
00127 (currently there are no general mandatories)
00128 """
00129
00130 GenericValidation.__init__(self, valName, alignment, config)
00131 defaults = {"jobmode": self.jobmode,
00132 "runRange": "",
00133 "firstRun": "",
00134 "lastRun": "",
00135 "begin": "",
00136 "end": "",
00137 "JSON": ""
00138 }
00139 defaults.update(addDefaults)
00140 mandatories = []
00141 mandatories += addMandatories
00142 theUpdate = config.getResultingSection(valType+":"+self.name,
00143 defaultDict = defaults,
00144 demandPars = mandatories)
00145 self.general.update(theUpdate)
00146 self.jobmode = self.general["jobmode"]
00147
00148 knownOpts = defaults.keys()+mandatories
00149 ignoreOpts = []
00150 if self.jobmode.split(",")[0] == "crab" \
00151 or self.__class__.__name__=="OfflineValidationParallel":
00152 knownOpts.append("parallelJobs")
00153 else:
00154 ignoreOpts.append("parallelJobs")
00155 config.checkInput(valType+":"+self.name,
00156 knownSimpleOptions = knownOpts,
00157 ignoreOptions = ignoreOpts)
00158
00159 if self.general["dataset"] not in globalDictionaries.usedDatasets:
00160 globalDictionaries.usedDatasets[self.general["dataset"]] = Dataset(
00161 self.general["dataset"] )
00162 self.dataset = globalDictionaries.usedDatasets[self.general["dataset"]]
00163
00164 if not self.jobmode.split( ',' )[0] == "crab":
00165 try:
00166 self.general["datasetDefinition"] = self.dataset.datasetSnippet(
00167 jsonPath = self.general["JSON"],
00168 nEvents = self.general["maxevents"],
00169 firstRun = self.general["firstRun"],
00170 lastRun = self.general["lastRun"],
00171 begin = self.general["begin"],
00172 end = self.general["end"] )
00173 except AllInOneError, e:
00174 msg = "In section [%s:%s]: "%(valType, self.name)
00175 msg += str(e)
00176 raise AllInOneError(msg)
00177 else:
00178 if self.dataset.predefined():
00179 msg = ("For jobmode 'crab' you cannot use predefined datasets "
00180 "(in your case: '%s')."%( self.dataset.name() ))
00181 raise AllInOneError( msg )
00182 try:
00183 theUpdate = config.getResultingSection(valType+":"+self.name,
00184 demandPars = ["parallelJobs"])
00185 except AllInOneError, e:
00186 msg = str(e)[:-1]+" when using 'jobmode: crab'."
00187 raise AllInOneError(msg)
00188 self.general.update(theUpdate)
00189 if self.general["begin"] or self.general["end"]:
00190 ( self.general["begin"],
00191 self.general["end"],
00192 self.general["firstRun"],
00193 self.general["lastRun"] ) = self.dataset.convertTimeToRun(
00194 firstRun = self.general["firstRun"],
00195 lastRun = self.general["lastRun"],
00196 begin = self.general["begin"],
00197 end = self.general["end"],
00198 shortTuple = False)
00199 if self.general["begin"] == None:
00200 self.general["begin"] = ""
00201 if self.general["end"] == None:
00202 self.general["end"] = ""
00203 self.general["firstRun"] = str( self.general["firstRun"] )
00204 self.general["lastRun"] = str( self.general["lastRun"] )
00205 if ( not self.general["firstRun"] ) and \
00206 ( self.general["end"] or self.general["lastRun"] ):
00207 self.general["firstRun"] = str(
00208 self.dataset.runList()[0]["run_number"])
00209 if ( not self.general["lastRun"] ) and \
00210 ( self.general["begin"] or self.general["firstRun"] ):
00211 self.general["lastRun"] = str(
00212 self.dataset.runList()[-1]["run_number"])
00213 if self.general["firstRun"] and self.general["lastRun"]:
00214 if int(self.general["firstRun"]) > int(self.general["lastRun"]):
00215 msg = ( "The lower time/runrange limit ('begin'/'firstRun') "
00216 "chosen is greater than the upper time/runrange limit "
00217 "('end'/'lastRun').")
00218 raise AllInOneError( msg )
00219 self.general["runRange"] = (self.general["firstRun"]
00220 + '-' + self.general["lastRun"])
00221 try:
00222 self.general["datasetDefinition"] = self.dataset.datasetSnippet(
00223 jsonPath = self.general["JSON"],
00224 nEvents = self.general["maxevents"],
00225 firstRun = self.general["firstRun"],
00226 lastRun = self.general["lastRun"],
00227 begin = self.general["begin"],
00228 end = self.general["end"],
00229 crab = True )
00230 except AllInOneError, e:
00231 msg = "In section [%s:%s]: "%(valType, self.name)
00232 msg += str( e )
00233 raise AllInOneError( msg )
00234
00235 def createCrabCfg(self, path, crabCfgBaseName):
00236 """
00237 Method which creates a `crab.cfg` for a validation on datasets.
00238
00239 Arguments:
00240 - `path`: Path at which the file will be stored.
00241 - `crabCfgBaseName`: String which depends on the actual type of
00242 validation calling this method.
00243 """
00244 crabCfgName = "crab.%s.%s.%s.cfg"%( crabCfgBaseName, self.name,
00245 self.alignmentToValidate.name )
00246 repMap = self.getRepMap()
00247 repMap["script"] = "dummy_script.sh"
00248
00249 repMap["crabWorkingDir"] = crabCfgName.split( '.cfg' )[0]
00250 self.crabWorkingDir = repMap["crabWorkingDir"]
00251 repMap["numberOfJobs"] = self.general["parallelJobs"]
00252 repMap["cfgFile"] = self.configFiles[0]
00253 repMap["queue"] = self.jobmode.split( ',' )[1].split( '-q' )[1]
00254 if self.dataset.dataType() == "mc":
00255 repMap["McOrData"] = "events = .oO[nEvents]Oo."
00256 elif self.dataset.dataType() == "data":
00257 repMap["McOrData"] = "lumis = -1"
00258 if self.jobmode.split( ',' )[0] == "crab":
00259 print ("For jobmode 'crab' the parameter 'maxevents' will be "
00260 "ignored and all events will be processed.")
00261 crabCfg = {crabCfgName: replaceByMap( configTemplates.crabCfgTemplate,
00262 repMap ) }
00263 return GenericValidation.createCrabCfg( self, crabCfg, path )
1.7.3