CMS 3D CMS Logo

List of all members | Public Member Functions | Public Attributes | Static Public Attributes
genericValidation.GenericValidation Class Reference
Inheritance diagram for genericValidation.GenericValidation:
genericValidation.ValidationMetaClass genericValidation.GenericValidationData genericValidation.ParallelValidation genericValidation.ValidationWithComparison genericValidation.ValidationWithPlots geometryComparison.GeometryComparison preexistingValidation.PreexistingValidation genericValidation.GenericValidationData_CTSR monteCarloValidation.MonteCarloValidation primaryVertexResolution.PrimaryVertexResolution zMuMuValidation.ZMuMuValidation offlineValidation.OfflineValidation overlapValidation.OverlapValidation primaryVertexValidation.PrimaryVertexValidation trackSplittingValidation.TrackSplittingValidation offlineValidation.OfflineValidation genericValidation.ValidationForPresentation genericValidation.ValidationWithPlotsSummaryBase overlapValidation.OverlapValidation primaryVertexResolution.PrimaryVertexResolution primaryVertexValidation.PrimaryVertexValidation zMuMuValidation.ZMuMuValidation preexistingValidation.PreexistingGeometryComparison preexistingValidation.PreexistingMonteCarloValidation preexistingValidation.PreexistingOfflineValidation preexistingValidation.PreexistingPrimaryVertexValidation preexistingValidation.PreexistingTrackSplittingValidation preexistingValidation.PreexistingZMuMuValidation

Public Member Functions

def __init__ (self, valName, alignment, config)
 
def createConfiguration (self, fileContents, path, schedule=None, repMap=None, repMaps=None)
 
def createCrabCfg (self, fileContents, path)
 
def createFiles (self, fileContents, path, repMap=None, repMaps=None)
 
def createScript (self, fileContents, path, downloadFiles=[], repMap=None, repMaps=None)
 
def filesToCompare (self)
 
def getCompareStrings (self, requestId=None, plain=False)
 
def getRepMap (self, alignment=None)
 
- Public Member Functions inherited from genericValidation.ValidationMetaClass
def __new__ (cls, clsname, bases, dct)
 

Public Attributes

 alignmentToValidate
 
 AutoAlternates
 
 cmssw
 
 cmsswreleasebase
 
 config
 
 configFiles
 
 crabConfigFiles
 
 general
 
 jobid
 
 jobmode
 
 name
 
 needsproxy
 
 NJobs
 
 packages
 
 randomWorkdirPart
 
 scramarch
 
 scriptFiles
 

Static Public Attributes

string defaultReferenceName = "DEFAULT"
 
dictionary defaults
 
 mandatories = set()
 
dictionary needpackages = {"Alignment/OfflineValidation"}
 
dictionary optionals = {"jobmode"}
 
- Static Public Attributes inherited from genericValidation.ValidationMetaClass
list dicts = ["defaults"]
 
list sets = ["mandatories", "optionals", "needpackages"]
 

Detailed Description

Definition at line 41 of file genericValidation.py.

Constructor & Destructor Documentation

◆ __init__()

def genericValidation.GenericValidation.__init__ (   self,
  valName,
  alignment,
  config 
)

Definition at line 53 of file genericValidation.py.

53  def __init__(self, valName, alignment, config):
54  import random
55  self.name = valName
56  self.alignmentToValidate = alignment
57  self.general = config.getGeneral()
58  self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
59  self.configFiles = []
60  self.config = config
61  self.jobid = ""
62 
63  theUpdate = config.getResultingSection(self.valType+":"+self.name,
64  defaultDict = self.defaults,
65  demandPars = self.mandatories)
66  self.general.update(theUpdate)
67  self.jobmode = self.general["jobmode"]
68  self.NJobs = int(self.general["parallelJobs"])
69  self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")
70 
71  # limit maximum number of parallel jobs to 40
72  # (each output file is approximately 20MB)
73  maximumNumberJobs = 40
74  if self.NJobs > maximumNumberJobs:
75  msg = ("Maximum allowed number of parallel jobs "
76  +str(maximumNumberJobs)+" exceeded!!!")
77  raise AllInOneError(msg)
78  if self.NJobs > 1 and not isinstance(self, ParallelValidation):
79  raise AllInOneError("Parallel jobs not implemented for {}!\n"
80  "Please set parallelJobs = 1.".format(type(self).__name__))
81 
82  self.jobid = self.general["jobid"]
83  if self.jobid:
84  try: #make sure it's actually a valid jobid
85  output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
86  if "is not found" in output: raise RuntimeError
87  except RuntimeError:
88  raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)
89 
90  self.cmssw = self.general["cmssw"]
91  badcharacters = r"\'"
92  for character in badcharacters:
93  if character in self.cmssw:
94  raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
95  "path name. If you really have it in such a ridiculously named location,\n"
96  "try making a symbolic link somewhere with a decent name.")
97  try:
98  os.listdir(self.cmssw)
99  except OSError:
100  raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
101 
102  if self.cmssw == os.environ["CMSSW_BASE"]:
103  self.scramarch = os.environ["SCRAM_ARCH"]
104  self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
105  else:
106  command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
107  ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
108  commandoutput = getCommandOutput2(command).split('\n')
109  self.cmssw = commandoutput[0]
110  self.scramarch = commandoutput[1]
111  self.cmsswreleasebase = commandoutput[2]
112 
113  self.packages = {}
114  for package in self.needpackages:
115  for placetolook in self.cmssw, self.cmsswreleasebase:
116  pkgpath = os.path.join(placetolook, "src", package)
117  if os.path.exists(pkgpath):
118  self.packages[package] = pkgpath
119  break
120  else:
121  raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))
122 
123  self.AutoAlternates = True
124  if config.has_option("alternateTemplates","AutoAlternates"):
125  try:
126  self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
127  except ValueError:
128  raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
129 
130  knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
131  ignoreOpts = []
132  config.checkInput(self.valType+":"+self.name,
133  knownSimpleOptions = knownOpts,
134  ignoreOptions = ignoreOpts)
135 
def __init__(self, dataset, job_number, job_id, job_name, isDA, isMC, applyBOWS, applyEXTRACOND, extraconditions, runboundary, lumilist, intlumi, maxevents, gt, allFromGT, alignmentDB, alignmentTAG, apeDB, apeTAG, bowDB, bowTAG, vertextype, tracktype, refittertype, ttrhtype, applyruncontrol, ptcut, CMSSW_dir, the_dir)
def getCommandOutput2(command)
def boolfromstring(string, name)
#define update(a, b)
#define str(s)

Member Function Documentation

◆ createConfiguration()

def genericValidation.GenericValidation.createConfiguration (   self,
  fileContents,
  path,
  schedule = None,
  repMap = None,
  repMaps = None 
)

Definition at line 215 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles, and genericValidation.GenericValidation.createFiles().

215  def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
216  self.configFiles = self.createFiles(fileContents,
217  path, repMap = repMap, repMaps = repMaps)
218  if not schedule == None:
219  schedule = [os.path.join( path, cfgName) for cfgName in schedule]
220  for cfgName in schedule:
221  if not cfgName in self.configFiles:
222  msg = ("scheduled %s missing in generated configfiles: %s"
223  %(cfgName, self.configFiles))
224  raise AllInOneError(msg)
225  for cfgName in self.configFiles:
226  if not cfgName in schedule:
227  msg = ("generated configuration %s not scheduled: %s"
228  %(cfgName, schedule))
229  raise AllInOneError(msg)
230  self.configFiles = schedule
231  return self.configFiles
232 

◆ createCrabCfg()

def genericValidation.GenericValidation.createCrabCfg (   self,
  fileContents,
  path 
)

Definition at line 241 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

241  def createCrabCfg(self, fileContents, path ):
242  if self.NJobs > 1:
243  msg = ("jobmode 'crab' not supported for parallel validation."
244  " Please set parallelJobs = 1.")
245  raise AllInOneError(msg)
246  self.crabConfigFiles = self.createFiles(fileContents, path)
247  return self.crabConfigFiles
248 
249 

◆ createFiles()

def genericValidation.GenericValidation.createFiles (   self,
  fileContents,
  path,
  repMap = None,
  repMaps = None 
)
repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 192 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, helperFunctions.replaceByMap(), and str.

Referenced by genericValidation.GenericValidation.createConfiguration(), and geometryComparison.GeometryComparison.createScript().

192  def createFiles(self, fileContents, path, repMap = None, repMaps = None):
193  """repMap: single map for all files
194  repMaps: a dict, with the filenames as the keys"""
195  if repMap is not None and repMaps is not None:
196  raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
197  result = []
198  for fileName in fileContents:
199  filePath = os.path.join(path, fileName)
200  result.append(filePath)
201 
202  for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
203  theFile = open( filePathi, "w" )
204  fileContentsi = fileContents[ fileName ]
205  if repMaps is not None:
206  repMap = repMaps[fileName]
207  if repMap is not None:
208  repMap.update({"nIndex": str(i)})
209  fileContentsi = replaceByMap(fileContentsi, repMap)
210  theFile.write( fileContentsi )
211  theFile.close()
212 
213  return result
214 
def addIndex(filename, njobs, index=None)
def replaceByMap(target, the_map)
— Helpers —############################
#define str(s)

◆ createScript()

def genericValidation.GenericValidation.createScript (   self,
  fileContents,
  path,
  downloadFiles = [],
  repMap = None,
  repMaps = None 
)

Definition at line 233 of file genericValidation.py.

233  def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
234  self.scriptFiles = self.createFiles(fileContents,
235  path, repMap = repMap, repMaps = repMaps)
236  for script in self.scriptFiles:
237  for scriptwithindex in addIndex(script, self.NJobs):
238  os.chmod(scriptwithindex,0o755)
239  return self.scriptFiles
240 
def addIndex(filename, njobs, index=None)

◆ filesToCompare()

def genericValidation.GenericValidation.filesToCompare (   self)

Definition at line 165 of file genericValidation.py.

165  def filesToCompare(self):
166  pass
167 

◆ getCompareStrings()

def genericValidation.GenericValidation.getCompareStrings (   self,
  requestId = None,
  plain = False 
)

Definition at line 168 of file genericValidation.py.

Referenced by overlapValidation.OverlapValidation.appendToPlots(), and trackSplittingValidation.TrackSplittingValidation.appendToPlots().

168  def getCompareStrings( self, requestId = None, plain = False ):
169  result = {}
170  repMap = self.getRepMap().copy()
171  for validationId in self.filesToCompare:
172  repMap["file"] = self.filesToCompare[ validationId ]
173  if repMap["file"].startswith( "/castor/" ):
174  repMap["file"] = "rfio:%(file)s"%repMap
175  elif repMap["file"].startswith( "/store/" ):
176  repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
177  if plain:
178  result[validationId]=repMap["file"]
179  else:
180  result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
181  if requestId == None:
182  return result
183  else:
184  if not "." in requestId:
185  requestId += ".%s"%self.defaultReferenceName
186  if not requestId.split(".")[-1] in result:
187  msg = ("could not find %s in reference Objects!"
188  %requestId.split(".")[-1])
189  raise AllInOneError(msg)
190  return result[ requestId.split(".")[-1] ]
191 

◆ getRepMap()

def genericValidation.GenericValidation.getRepMap (   self,
  alignment = None 
)

Definition at line 136 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, geometry.Alignables.config, genericValidation.GenericValidation.config, genericValidation.GenericValidation.general, genericValidation.GenericValidation.packages, plottingOptions.PlottingOptions(), genericValidation.GenericValidation.randomWorkdirPart, genericValidation.GenericValidation.scramarch, trackSplittingValidation.TrackSplittingValidation.valType, monteCarloValidation.MonteCarloValidation.valType, overlapValidation.OverlapValidation.valType, primaryVertexValidation.PrimaryVertexValidation.valType, primaryVertexResolution.PrimaryVertexResolution.valType, offlineValidation.OfflineValidation.valType, zMuMuValidation.ZMuMuValidation.valType, geometryComparison.GeometryComparison.valType, validateAlignments.ValidationBase.valType, and validateAlignments.ValidationJobMultiIOV.valType.

Referenced by overlapValidation.OverlapValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), primaryVertexResolution.PrimaryVertexResolution.appendToMerge(), primaryVertexValidation.PrimaryVertexValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), overlapValidation.OverlapValidation.appendToPlots(), trackSplittingValidation.TrackSplittingValidation.appendToPlots(), primaryVertexResolution.PrimaryVertexResolution.appendToPlots(), primaryVertexValidation.PrimaryVertexValidation.appendToPlots(), zMuMuValidation.ZMuMuValidation.appendToPlots(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), genericValidation.ParallelValidation.doMerge(), zMuMuValidation.ZMuMuValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), and plottingOptions.PlottingOptionsTrackSplitting.validsubdets().

136  def getRepMap(self, alignment = None):
137  from .plottingOptions import PlottingOptions
138  if alignment == None:
139  alignment = self.alignmentToValidate
140  try:
141  result = PlottingOptions(self.config, self.valType)
142  except KeyError:
143  result = {}
144  result.update(alignment.getRepMap())
145  result.update(self.general)
146  result.update({
147  "workdir": os.path.join(self.general["workdir"],
148  self.randomWorkdirPart),
149  "datadir": self.general["datadir"],
150  "logdir": self.general["logdir"],
151  "CommandLineTemplate": ("#run configfile and post-proccess it\n"
152  "cmsRun %(cfgFile)s\n"
153  "%(postProcess)s "),
154  "CMSSW_BASE": self.cmssw,
155  "SCRAM_ARCH": self.scramarch,
156  "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
157  "alignmentName": alignment.name,
158  "condLoad": alignment.getConditions(),
159  "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
160  })
161  result.update(self.packages)
162  return result
163 
def PlottingOptions(config, valType)

Member Data Documentation

◆ alignmentToValidate

genericValidation.GenericValidation.alignmentToValidate

◆ AutoAlternates

genericValidation.GenericValidation.AutoAlternates

Definition at line 123 of file genericValidation.py.

◆ cmssw

genericValidation.GenericValidation.cmssw

◆ cmsswreleasebase

genericValidation.GenericValidation.cmsswreleasebase

◆ config

genericValidation.GenericValidation.config

◆ configFiles

genericValidation.GenericValidation.configFiles

◆ crabConfigFiles

genericValidation.GenericValidation.crabConfigFiles

Definition at line 246 of file genericValidation.py.

◆ defaultReferenceName

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"
static

◆ defaults

dictionary genericValidation.GenericValidation.defaults
static
Initial value:
= {
"cmssw": os.environ['CMSSW_BASE'],
"parallelJobs": "1",
"jobid": "",
"needsproxy": "false",
}

Definition at line 44 of file genericValidation.py.

Referenced by tree.Tree.reset(), and tree.Tree.var().

◆ general

genericValidation.GenericValidation.general

◆ jobid

genericValidation.GenericValidation.jobid

Definition at line 61 of file genericValidation.py.

◆ jobmode

genericValidation.GenericValidation.jobmode

◆ mandatories

genericValidation.GenericValidation.mandatories = set()
static

Definition at line 43 of file genericValidation.py.

◆ name

genericValidation.GenericValidation.name

Definition at line 55 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dataset.CMSDataset.buildListOfFiles(), dataset.LocalDataset.buildListOfFiles(), dataset.CMSDataset.buildListOfFilesDBS(), dirstructure.Directory.calcStats(), genericValidation.GenericValidationData.cfgName(), genericValidation.GenericValidationData_CTSR.cosmics0T(), crabFunctions.CrabTask.crabConfig(), crabFunctions.CrabTask.crabFolder(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexResolution.PrimaryVertexResolution.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), crabFunctions.CrabTask.handleNoState(), VIDSelectorBase.VIDSelectorBase.initialize(), crabFunctions.CrabTask.isData(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), crabFunctions.CrabTask.resubmit_failed(), production_tasks.MonitorJobs.run(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), crabFunctions.CrabTask.update(), crabFunctions.CrabTask.updateJobStats(), counter.Counter.write(), and average.Average.write().

◆ needpackages

dictionary genericValidation.GenericValidation.needpackages = {"Alignment/OfflineValidation"}
static

Definition at line 50 of file genericValidation.py.

◆ needsproxy

genericValidation.GenericValidation.needsproxy

Definition at line 69 of file genericValidation.py.

◆ NJobs

genericValidation.GenericValidation.NJobs

◆ optionals

dictionary genericValidation.GenericValidation.optionals = {"jobmode"}
static

Definition at line 51 of file genericValidation.py.

◆ packages

genericValidation.GenericValidation.packages

Definition at line 113 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

◆ randomWorkdirPart

genericValidation.GenericValidation.randomWorkdirPart

◆ scramarch

genericValidation.GenericValidation.scramarch

◆ scriptFiles

genericValidation.GenericValidation.scriptFiles

Definition at line 234 of file genericValidation.py.