test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
genericValidation.GenericValidation Class Reference
Inheritance diagram for genericValidation.GenericValidation:
genericValidation.GenericValidationData geometryComparison.GeometryComparison preexistingValidation.PreexistingValidation monteCarloValidation.MonteCarloValidation offlineValidation.OfflineValidation primaryVertexValidation.PrimaryVertexValidation trackSplittingValidation.TrackSplittingValidation zMuMuValidation.ZMuMuValidation preexistingValidation.PreexistingGeometryComparison preexistingValidation.PreexistingMonteCarloValidation preexistingValidation.PreexistingOfflineValidation preexistingValidation.PreexistingTrackSplittingValidation preexistingValidation.PreexistingZMuMuValidation

Public Member Functions

def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def getCompareStrings
 
def getRepMap
 

Public Attributes

 alignmentToValidate
 
 AutoAlternates
 
 cmssw
 
 cmsswreleasebase
 
 config
 
 configFiles
 
 crabConfigFiles
 
 filesToCompare
 
 general
 
 jobmode
 
 name
 
 NJobs
 
 packages
 
 randomWorkdirPart
 
 scramarch
 
 scriptFiles
 
 valType
 

Static Public Attributes

string defaultReferenceName = "DEFAULT"
 

Detailed Description

Definition at line 12 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__ (   self,
  valName,
  alignment,
  config,
  valType,
  addDefaults = {},
  addMandatories = [],
  addneedpackages = [] 
)

Definition at line 15 of file genericValidation.py.

15 
16  addDefaults = {}, addMandatories=[], addneedpackages=[]):
17  import random
18  self.name = valName
19  self.valType = valType
20  self.alignmentToValidate = alignment
21  self.general = config.getGeneral()
22  self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
23  self.configFiles = []
24  self.filesToCompare = {}
25  self.config = config
26 
27  defaults = {"jobmode": self.general["jobmode"],
28  "cmssw": os.environ['CMSSW_BASE'],
29  "parallelJobs": "1"
30  }
31  defaults.update(addDefaults)
32  mandatories = []
33  mandatories += addMandatories
34  needpackages = ["Alignment/OfflineValidation"]
35  needpackages += addneedpackages
36  theUpdate = config.getResultingSection(valType+":"+self.name,
37  defaultDict = defaults,
38  demandPars = mandatories)
39  self.general.update(theUpdate)
40  self.jobmode = self.general["jobmode"]
41  self.NJobs = int(self.general["parallelJobs"])
42 
43  # limit maximum number of parallel jobs to 40
44  # (each output file is approximately 20MB)
45  maximumNumberJobs = 40
46  if self.NJobs > maximumNumberJobs:
47  msg = ("Maximum allowed number of parallel jobs "
48  +str(maximumNumberJobs)+" exceeded!!!")
49  raise AllInOneError(msg)
50 
51  self.cmssw = self.general["cmssw"]
52  badcharacters = r"\'"
53  for character in badcharacters:
54  if character in self.cmssw:
55  raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
56  "path name. If you really have it in such a ridiculously named location,\n"
57  "try making a symbolic link somewhere with a decent name.")
58  try:
59  os.listdir(self.cmssw)
60  except OSError:
61  raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
62 
63  if self.cmssw == os.environ["CMSSW_BASE"]:
64  self.scramarch = os.environ["SCRAM_ARCH"]
65  self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
66  else:
67  command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
68  ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
69  commandoutput = getCommandOutput2(command).split('\n')
70  self.cmssw = commandoutput[0]
71  self.scramarch = commandoutput[1]
72  self.cmsswreleasebase = commandoutput[2]
73 
74  self.packages = {}
75  for package in needpackages:
76  for placetolook in self.cmssw, self.cmsswreleasebase:
77  pkgpath = os.path.join(placetolook, "src", package)
78  if os.path.exists(pkgpath):
79  self.packages[package] = pkgpath
80  break
81  else:
82  raise AllInOneError("Package {} does not exist in {} or {}!".format(package, self.cmssw, self.cmsswreleasebase))
83 
84  self.AutoAlternates = True
85  if config.has_option("alternateTemplates","AutoAlternates"):
86  try:
87  self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
88  except ValueError:
89  raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
90 
91  knownOpts = defaults.keys()+mandatories
92  ignoreOpts = []
93  config.checkInput(valType+":"+self.name,
94  knownSimpleOptions = knownOpts,
95  ignoreOptions = ignoreOpts)
double split
Definition: MVATrainer.cc:139

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration (   self,
  fileContents,
  path,
  schedule = None,
  repMap = None,
  repMaps = None 
)

Definition at line 170 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles.

171  def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
172  self.configFiles = GenericValidation.createFiles(self, fileContents,
173  path, repMap = repMap, repMaps = repMaps)
174  if not schedule == None:
175  schedule = [os.path.join( path, cfgName) for cfgName in schedule]
176  for cfgName in schedule:
177  if not cfgName in self.configFiles:
178  msg = ("scheduled %s missing in generated configfiles: %s"
179  %(cfgName, self.configFiles))
180  raise AllInOneError(msg)
181  for cfgName in self.configFiles:
182  if not cfgName in schedule:
183  msg = ("generated configuration %s not scheduled: %s"
184  %(cfgName, schedule))
185  raise AllInOneError(msg)
186  self.configFiles = schedule
187  return self.configFiles
def genericValidation.GenericValidation.createCrabCfg (   self,
  fileContents,
  path 
)

Definition at line 196 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

197  def createCrabCfg(self, fileContents, path ):
198  if self.NJobs > 1:
199  msg = ("jobmode 'crab' not supported for parallel validation."
200  " Please set parallelJobs = 1.")
201  raise AllInOneError(msg)
202  self.crabConfigFiles = GenericValidation.createFiles(self, fileContents,
203  path)
204  return self.crabConfigFiles
205 
def genericValidation.GenericValidation.createFiles (   self,
  fileContents,
  path,
  repMap = None,
  repMaps = None 
)
repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 147 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, and helperFunctions.replaceByMap().

148  def createFiles(self, fileContents, path, repMap = None, repMaps = None):
149  """repMap: single map for all files
150  repMaps: a dict, with the filenames as the keys"""
151  if repMap is not None and repMaps is not None:
152  raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
153  result = []
154  for fileName in fileContents:
155  filePath = os.path.join(path, fileName)
156  result.append(filePath)
157 
158  for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
159  theFile = open( filePathi, "w" )
160  fileContentsi = fileContents[ fileName ]
161  if repMaps is not None:
162  repMap = repMaps[fileName]
163  if repMap is not None:
164  repMap.update({"nIndex": str(i)})
165  fileContentsi = replaceByMap(fileContentsi, repMap)
166  theFile.write( fileContentsi )
167  theFile.close()
168 
169  return result
def replaceByMap
— Helpers —############################
def genericValidation.GenericValidation.createScript (   self,
  fileContents,
  path,
  downloadFiles = [],
  repMap = None,
  repMaps = None 
)

Definition at line 188 of file genericValidation.py.

189  def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
190  self.scriptFiles = GenericValidation.createFiles(self, fileContents,
191  path, repMap = repMap, repMaps = repMaps)
192  for script in self.scriptFiles:
193  for scriptwithindex in addIndex(script, self.NJobs):
194  os.chmod(scriptwithindex,0o755)
195  return self.scriptFiles
def genericValidation.GenericValidation.getCompareStrings (   self,
  requestId = None,
  plain = False 
)

Definition at line 123 of file genericValidation.py.

References genericValidation.GenericValidation.filesToCompare.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), and preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation().

124  def getCompareStrings( self, requestId = None, plain = False ):
125  result = {}
126  repMap = self.alignmentToValidate.getRepMap()
127  for validationId in self.filesToCompare:
128  repMap["file"] = self.filesToCompare[ validationId ]
129  if repMap["file"].startswith( "/castor/" ):
130  repMap["file"] = "rfio:%(file)s"%repMap
131  elif repMap["file"].startswith( "/store/" ):
132  repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
133  if plain:
134  result[validationId]=repMap["file"]
135  else:
136  result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
137  if requestId == None:
138  return result
139  else:
140  if not "." in requestId:
141  requestId += ".%s"%GenericValidation.defaultReferenceName
142  if not requestId.split(".")[-1] in result:
143  msg = ("could not find %s in reference Objects!"
144  %requestId.split(".")[-1])
145  raise AllInOneError(msg)
146  return result[ requestId.split(".")[-1] ]
def genericValidation.GenericValidation.getRepMap (   self,
  alignment = None 
)

Definition at line 96 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, geometry.Alignables.config, genericValidation.GenericValidation.config, genericValidation.GenericValidation.general, genericValidation.GenericValidation.packages, plottingOptions.PlottingOptions(), genericValidation.GenericValidation.randomWorkdirPart, genericValidation.GenericValidation.scramarch, and genericValidation.GenericValidation.valType.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), zMuMuValidation.ZMuMuValidation.appendToExtendedValidation(), primaryVertexValidation.PrimaryVertexValidation.appendToExtendedValidation(), offlineValidation.OfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), primaryVertexValidation.PrimaryVertexValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), primaryVertexValidation.PrimaryVertexValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), preexistingValidation.PreexistingValidation.getCompareStrings(), and plottingOptions.PlottingOptionsTrackSplitting.validsubdets().

96 
97  def getRepMap(self, alignment = None):
98  if alignment == None:
99  alignment = self.alignmentToValidate
100  try:
101  result = PlottingOptions(self.config, self.valType)
102  except KeyError:
103  result = {}
104  result.update(alignment.getRepMap())
105  result.update( self.general )
106  result.update({
107  "workdir": os.path.join(self.general["workdir"],
108  self.randomWorkdirPart),
109  "datadir": self.general["datadir"],
110  "logdir": self.general["logdir"],
111  "CommandLineTemplate": ("#run configfile and post-proccess it\n"
112  "cmsRun %(cfgFile)s\n"
113  "%(postProcess)s "),
114  "CMSSW_BASE": self.cmssw,
115  "SCRAM_ARCH": self.scramarch,
116  "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
117  "alignmentName": alignment.name,
118  "condLoad": alignment.getConditions(),
119  "condLoad": alignment.getConditions(),
120  })
121  result.update(self.packages)
122  return result

Member Data Documentation

genericValidation.GenericValidation.alignmentToValidate

Definition at line 19 of file genericValidation.py.

Referenced by primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), geometryComparison.GeometryComparison.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 83 of file genericValidation.py.

Referenced by offlineValidation.OfflineValidation.createConfiguration().

genericValidation.GenericValidation.cmssw

Definition at line 50 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.getRepMap(), plottingOptions.BasePlottingOptions.getRepMap(), and trackSplittingValidation.TrackSplittingValidation.validsubdets().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 64 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.getRepMap(), plottingOptions.BasePlottingOptions.getRepMap(), and trackSplittingValidation.TrackSplittingValidation.validsubdets().

genericValidation.GenericValidation.config

Definition at line 24 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap(), DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), DTT0DBValidation.DTT0DBValidation.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTTTrigValid.DTTTrigValid.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), and DTCalibrationWorker.DTCalibrationWorker.run().

genericValidation.GenericValidation.configFiles

Definition at line 22 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 201 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"
static

Definition at line 13 of file genericValidation.py.

genericValidation.GenericValidation.filesToCompare

Definition at line 23 of file genericValidation.py.

Referenced by trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), primaryVertexValidation.PrimaryVertexValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), preexistingValidation.PreexistingValidation.getCompareStrings(), and genericValidation.GenericValidation.getCompareStrings().

genericValidation.GenericValidation.general

Definition at line 20 of file genericValidation.py.

Referenced by preexistingValidation.PreexistingOfflineValidation.__init__(), plottingOptions.PlottingOptionsTrackSplitting.__init__(), genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), monteCarloValidation.MonteCarloValidation.getRepMap(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), genericValidation.GenericValidation.getRepMap(), and plottingOptions.BasePlottingOptions.getRepMap().

genericValidation.GenericValidation.jobmode

Definition at line 39 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 17 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), primaryVertexValidation.PrimaryVertexValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), primaryVertexValidation.PrimaryVertexValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

genericValidation.GenericValidation.NJobs

Definition at line 40 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

genericValidation.GenericValidation.packages

Definition at line 73 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 21 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap(), and plottingOptions.BasePlottingOptions.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 63 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap(), and plottingOptions.BasePlottingOptions.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 189 of file genericValidation.py.

genericValidation.GenericValidation.valType

Definition at line 18 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().