CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
genericValidation.GenericValidation Class Reference
Inheritance diagram for genericValidation.GenericValidation:
genericValidation.ValidationMetaClass genericValidation.GenericValidationData genericValidation.ParallelValidation genericValidation.ValidationWithComparison genericValidation.ValidationWithPlots genericValidation.GenericValidationData_CTSR genericValidation.ValidationForPresentation genericValidation.ValidationWithPlotsSummaryBase genericValidation.ValidationWithPlotsSummary

Public Member Functions

def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def filesToCompare
 
def getCompareStrings
 
def getRepMap
 
- Public Member Functions inherited from genericValidation.ValidationMetaClass
def __new__
 

Public Attributes

 alignmentToValidate
 
 AutoAlternates
 
 cmssw
 
 cmsswreleasebase
 
 config
 
 configFiles
 
 crabConfigFiles
 
 general
 
 jobid
 
 jobmode
 
 name
 
 needsproxy
 
 NJobs
 
 packages
 
 randomWorkdirPart
 
 scramarch
 
 scriptFiles
 

Static Public Attributes

string defaultReferenceName = "DEFAULT"
 
dictionary defaults
 
tuple mandatories = set()
 
dictionary needpackages = {"Alignment/OfflineValidation"}
 
dictionary optionals = {"jobmode"}
 
- Static Public Attributes inherited from genericValidation.ValidationMetaClass
list dicts = ["defaults"]
 
list sets = ["mandatories", "optionals", "needpackages"]
 

Detailed Description

Definition at line 41 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__ (   self,
  valName,
  alignment,
  config 
)

Definition at line 53 of file genericValidation.py.

53 
54  def __init__(self, valName, alignment, config):
55  import random
56  self.name = valName
57  self.alignmentToValidate = alignment
58  self.general = config.getGeneral()
59  self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
60  self.configFiles = []
61  self.config = config
62  self.jobid = ""
63 
64  theUpdate = config.getResultingSection(self.valType+":"+self.name,
65  defaultDict = self.defaults,
66  demandPars = self.mandatories)
67  self.general.update(theUpdate)
68  self.jobmode = self.general["jobmode"]
69  self.NJobs = int(self.general["parallelJobs"])
70  self.needsproxy = boolfromstring(self.general["needsproxy"], "needsproxy")
71 
72  # limit maximum number of parallel jobs to 40
73  # (each output file is approximately 20MB)
74  maximumNumberJobs = 40
75  if self.NJobs > maximumNumberJobs:
76  msg = ("Maximum allowed number of parallel jobs "
77  +str(maximumNumberJobs)+" exceeded!!!")
78  raise AllInOneError(msg)
79  if self.NJobs > 1 and not isinstance(self, ParallelValidation):
80  raise AllInOneError("Parallel jobs not implemented for {}!\n"
81  "Please set parallelJobs = 1.".format(type(self).__name__))
82 
83  self.jobid = self.general["jobid"]
84  if self.jobid:
85  try: #make sure it's actually a valid jobid
86  output = getCommandOutput2("bjobs %(jobid)s 2>&1"%self.general)
87  if "is not found" in output: raise RuntimeError
88  except RuntimeError:
89  raise AllInOneError("%s is not a valid jobid.\nMaybe it finished already?"%self.jobid)
90 
91  self.cmssw = self.general["cmssw"]
92  badcharacters = r"\'"
93  for character in badcharacters:
94  if character in self.cmssw:
95  raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
96  "path name. If you really have it in such a ridiculously named location,\n"
97  "try making a symbolic link somewhere with a decent name.")
98  try:
99  os.listdir(self.cmssw)
100  except OSError:
101  raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
102 
103  if self.cmssw == os.environ["CMSSW_BASE"]:
104  self.scramarch = os.environ["SCRAM_ARCH"]
105  self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
106  else:
107  command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
108  ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
109  commandoutput = getCommandOutput2(command).split('\n')
110  self.cmssw = commandoutput[0]
111  self.scramarch = commandoutput[1]
112  self.cmsswreleasebase = commandoutput[2]
114  self.packages = {}
115  for package in self.needpackages:
116  for placetolook in self.cmssw, self.cmsswreleasebase:
117  pkgpath = os.path.join(placetolook, "src", package)
118  if os.path.exists(pkgpath):
119  self.packages[package] = pkgpath
120  break
121  else:
122  raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))
124  self.AutoAlternates = True
125  if config.has_option("alternateTemplates","AutoAlternates"):
126  try:
127  self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
128  except ValueError:
129  raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
130 
131  knownOpts = set(self.defaults.keys())|self.mandatories|self.optionals
132  ignoreOpts = []
133  config.checkInput(self.valType+":"+self.name,
134  knownSimpleOptions = knownOpts,
135  ignoreOptions = ignoreOpts)
#define str(s)

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration (   self,
  fileContents,
  path,
  schedule = None,
  repMap = None,
  repMaps = None 
)

Definition at line 215 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles, and genericValidation.GenericValidation.createFiles().

216  def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
217  self.configFiles = self.createFiles(fileContents,
218  path, repMap = repMap, repMaps = repMaps)
219  if not schedule == None:
220  schedule = [os.path.join( path, cfgName) for cfgName in schedule]
221  for cfgName in schedule:
222  if not cfgName in self.configFiles:
223  msg = ("scheduled %s missing in generated configfiles: %s"
224  %(cfgName, self.configFiles))
225  raise AllInOneError(msg)
226  for cfgName in self.configFiles:
227  if not cfgName in schedule:
228  msg = ("generated configuration %s not scheduled: %s"
229  %(cfgName, schedule))
230  raise AllInOneError(msg)
231  self.configFiles = schedule
232  return self.configFiles
def genericValidation.GenericValidation.createCrabCfg (   self,
  fileContents,
  path 
)

Definition at line 241 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

242  def createCrabCfg(self, fileContents, path ):
243  if self.NJobs > 1:
244  msg = ("jobmode 'crab' not supported for parallel validation."
245  " Please set parallelJobs = 1.")
246  raise AllInOneError(msg)
247  self.crabConfigFiles = self.createFiles(fileContents, path)
248  return self.crabConfigFiles
249 
def genericValidation.GenericValidation.createFiles (   self,
  fileContents,
  path,
  repMap = None,
  repMaps = None 
)
repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 192 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, helperFunctions.replaceByMap(), and str.

Referenced by genericValidation.GenericValidation.createConfiguration(), and geometryComparison.GeometryComparison.createScript().

193  def createFiles(self, fileContents, path, repMap = None, repMaps = None):
194  """repMap: single map for all files
195  repMaps: a dict, with the filenames as the keys"""
196  if repMap is not None and repMaps is not None:
197  raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
198  result = []
199  for fileName in fileContents:
200  filePath = os.path.join(path, fileName)
201  result.append(filePath)
202 
203  for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
204  theFile = open( filePathi, "w" )
205  fileContentsi = fileContents[ fileName ]
206  if repMaps is not None:
207  repMap = repMaps[fileName]
208  if repMap is not None:
209  repMap.update({"nIndex": str(i)})
210  fileContentsi = replaceByMap(fileContentsi, repMap)
211  theFile.write( fileContentsi )
212  theFile.close()
213 
214  return result
def replaceByMap
— Helpers —############################
#define str(s)
def genericValidation.GenericValidation.createScript (   self,
  fileContents,
  path,
  downloadFiles = [],
  repMap = None,
  repMaps = None 
)

Definition at line 233 of file genericValidation.py.

234  def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
235  self.scriptFiles = self.createFiles(fileContents,
236  path, repMap = repMap, repMaps = repMaps)
237  for script in self.scriptFiles:
238  for scriptwithindex in addIndex(script, self.NJobs):
239  os.chmod(scriptwithindex,0o755)
240  return self.scriptFiles
def genericValidation.GenericValidation.filesToCompare (   self)

Definition at line 165 of file genericValidation.py.

166  def filesToCompare(self):
167  pass
def genericValidation.GenericValidation.getCompareStrings (   self,
  requestId = None,
  plain = False 
)

Definition at line 168 of file genericValidation.py.

Referenced by overlapValidation.OverlapValidation.appendToPlots(), and trackSplittingValidation.TrackSplittingValidation.appendToPlots().

169  def getCompareStrings( self, requestId = None, plain = False ):
170  result = {}
171  repMap = self.getRepMap().copy()
172  for validationId in self.filesToCompare:
173  repMap["file"] = self.filesToCompare[ validationId ]
174  if repMap["file"].startswith( "/castor/" ):
175  repMap["file"] = "rfio:%(file)s"%repMap
176  elif repMap["file"].startswith( "/store/" ):
177  repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
178  if plain:
179  result[validationId]=repMap["file"]
180  else:
181  result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
182  if requestId == None:
183  return result
184  else:
185  if not "." in requestId:
186  requestId += ".%s"%self.defaultReferenceName
187  if not requestId.split(".")[-1] in result:
188  msg = ("could not find %s in reference Objects!"
189  %requestId.split(".")[-1])
190  raise AllInOneError(msg)
191  return result[ requestId.split(".")[-1] ]
def genericValidation.GenericValidation.getRepMap (   self,
  alignment = None 
)

Definition at line 136 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, geometry.Alignables.config, genericValidation.GenericValidation.config, genericValidation.GenericValidation.general, genericValidation.GenericValidation.packages, plottingOptions.PlottingOptions(), genericValidation.GenericValidation.randomWorkdirPart, genericValidation.GenericValidation.scramarch, trackSplittingValidation.TrackSplittingValidation.valType, monteCarloValidation.MonteCarloValidation.valType, overlapValidation.OverlapValidation.valType, primaryVertexValidation.PrimaryVertexValidation.valType, primaryVertexResolution.PrimaryVertexResolution.valType, offlineValidation.OfflineValidation.valType, zMuMuValidation.ZMuMuValidation.valType, geometryComparison.GeometryComparison.valType, validateAlignments.ValidationBase.valType, and validateAlignments.ValidationJobMultiIOV.valType.

Referenced by overlapValidation.OverlapValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), primaryVertexResolution.PrimaryVertexResolution.appendToMerge(), primaryVertexValidation.PrimaryVertexValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), overlapValidation.OverlapValidation.appendToPlots(), trackSplittingValidation.TrackSplittingValidation.appendToPlots(), primaryVertexResolution.PrimaryVertexResolution.appendToPlots(), primaryVertexValidation.PrimaryVertexValidation.appendToPlots(), zMuMuValidation.ZMuMuValidation.appendToPlots(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), genericValidation.ParallelValidation.doMerge(), zMuMuValidation.ZMuMuValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), genericValidation.GenericValidation.getCompareStrings(), and plottingOptions.PlottingOptionsTrackSplitting.validsubdets().

137  def getRepMap(self, alignment = None):
138  from .plottingOptions import PlottingOptions
139  if alignment == None:
140  alignment = self.alignmentToValidate
141  try:
142  result = PlottingOptions(self.config, self.valType)
143  except KeyError:
144  result = {}
145  result.update(alignment.getRepMap())
146  result.update(self.general)
147  result.update({
148  "workdir": os.path.join(self.general["workdir"],
149  self.randomWorkdirPart),
150  "datadir": self.general["datadir"],
151  "logdir": self.general["logdir"],
152  "CommandLineTemplate": ("#run configfile and post-proccess it\n"
153  "cmsRun %(cfgFile)s\n"
154  "%(postProcess)s "),
155  "CMSSW_BASE": self.cmssw,
156  "SCRAM_ARCH": self.scramarch,
157  "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
158  "alignmentName": alignment.name,
159  "condLoad": alignment.getConditions(),
160  "LoadGlobalTagTemplate": configTemplates.loadGlobalTagTemplate,
161  })
162  result.update(self.packages)
163  return result

Member Data Documentation

genericValidation.GenericValidation.alignmentToValidate

Definition at line 56 of file genericValidation.py.

Referenced by primaryVertexResolution.PrimaryVertexResolution.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), geometryComparison.GeometryComparison.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 123 of file genericValidation.py.

genericValidation.GenericValidation.cmssw

Definition at line 90 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 104 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.config

Definition at line 60 of file genericValidation.py.

Referenced by preexistingValidation.PreexistingValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), and zMuMuValidation.ZMuMuValidation.trackcollection().

genericValidation.GenericValidation.configFiles

Definition at line 59 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 246 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"
static

Definition at line 42 of file genericValidation.py.

Referenced by zMuMuValidation.ZMuMuValidation.filesToCompare(), preexistingValidation.PreexistingValidation.filesToCompare(), genericValidation.GenericValidationData.filesToCompare(), and genericValidation.GenericValidation.getCompareStrings().

dictionary genericValidation.GenericValidation.defaults
static
Initial value:
1 = {
2  "cmssw": os.environ['CMSSW_BASE'],
3  "parallelJobs": "1",
4  "jobid": "",
5  "needsproxy": "false",
6  }

Definition at line 44 of file genericValidation.py.

Referenced by tree.Tree.reset(), and tree.Tree.var().

genericValidation.GenericValidation.general

Definition at line 57 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.__init__(), primaryVertexResolution.PrimaryVertexResolution.__init__(), offlineValidation.OfflineValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), preexistingValidation.PreexistingOfflineValidation.__init__(), plottingOptions.PlottingOptionsTrackSplitting.__init__(), plottingOptions.PlottingOptionsZMuMu.__init__(), plottingOptions.PlottingOptionsOffline.__init__(), plottingOptions.PlottingOptionsPrimaryVertex.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidationData_CTSR.cosmics0T(), preexistingValidation.PreexistingValidation.filesToCompare(), overlapValidation.OverlapValidation.getRepMap(), monteCarloValidation.MonteCarloValidation.getRepMap(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), primaryVertexResolution.PrimaryVertexResolution.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), preexistingValidation.PreexistingOfflineValidation.getRepMap(), plottingOptions.BasePlottingOptions.getRepMap(), preexistingValidation.PreexistingPrimaryVertexValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), and genericValidation.GenericValidationData_CTSR.use_d0cut().

genericValidation.GenericValidation.jobid

Definition at line 61 of file genericValidation.py.

genericValidation.GenericValidation.jobmode

Definition at line 67 of file genericValidation.py.

tuple genericValidation.GenericValidation.mandatories = set()
static

Definition at line 43 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 55 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), FWLite.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), DisplayManager.DisplayManager.__del__(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), FWLite.WorkingPoints._reformat_cut_definitions(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), genericValidation.GenericValidationData.cfgName(), crabFunctions.CrabTask.crabConfig(), crabFunctions.CrabTask.crabFolder(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), DisplayManager.DisplayManager.Draw(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexResolution.PrimaryVertexResolution.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), crabFunctions.CrabTask.handleNoState(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), crabFunctions.CrabTask.resubmit_failed(), production_tasks.MonitorJobs.run(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), crabFunctions.CrabTask.update(), crabFunctions.CrabTask.updateJobStats(), counter.Counter.write(), and average.Average.write().

dictionary genericValidation.GenericValidation.needpackages = {"Alignment/OfflineValidation"}
static

Definition at line 50 of file genericValidation.py.

genericValidation.GenericValidation.needsproxy

Definition at line 69 of file genericValidation.py.

genericValidation.GenericValidation.NJobs

Definition at line 68 of file genericValidation.py.

Referenced by monteCarloValidation.MonteCarloValidation.__init__(), primaryVertexResolution.PrimaryVertexResolution.__init__(), offlineValidation.OfflineValidation.__init__(), zMuMuValidation.ZMuMuValidation.__init__(), genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

dictionary genericValidation.GenericValidation.optionals = {"jobmode"}
static

Definition at line 51 of file genericValidation.py.

genericValidation.GenericValidation.packages

Definition at line 113 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 58 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 103 of file genericValidation.py.

Referenced by plottingOptions.BasePlottingOptions.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 234 of file genericValidation.py.