CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
genericValidation.GenericValidation Class Reference
Inheritance diagram for genericValidation.GenericValidation:
genericValidation.GenericValidationData geometryComparison.GeometryComparison preexistingValidation.PreexistingValidation monteCarloValidation.MonteCarloValidation offlineValidation.OfflineValidation trackSplittingValidation.TrackSplittingValidation zMuMuValidation.ZMuMuValidation preexistingValidation.PreexistingGeometryComparison preexistingValidation.PreexistingMonteCarloValidation preexistingValidation.PreexistingOfflineValidation preexistingValidation.PreexistingTrackSplittingValidation preexistingValidation.PreexistingZMuMuValidation

Public Member Functions

def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def getCompareStrings
 
def getRepMap
 

Public Attributes

 alignmentToValidate
 
 AutoAlternates
 
 cmssw
 
 cmsswreleasebase
 
 config
 
 configFiles
 
 crabConfigFiles
 
 filesToCompare
 
 general
 
 jobmode
 
 name
 
 NJobs
 
 randomWorkdirPart
 
 scramarch
 
 scriptFiles
 

Static Public Attributes

string defaultReferenceName = "DEFAULT"
 

Detailed Description

Definition at line 11 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__ (   self,
  valName,
  alignment,
  config,
  valType,
  addDefaults = {},
  addMandatories = [] 
)

Definition at line 14 of file genericValidation.py.

14 
15  addDefaults = {}, addMandatories=[]):
16  import random
17  self.name = valName
18  self.alignmentToValidate = alignment
19  self.general = config.getGeneral()
20  self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
21  self.configFiles = []
22  self.filesToCompare = {}
23  self.config = config
24 
25  defaults = {"jobmode": self.general["jobmode"],
26  "cmssw": os.environ['CMSSW_BASE'],
27  "parallelJobs": "1"
28  }
29  defaults.update(addDefaults)
30  mandatories = []
31  mandatories += addMandatories
32  theUpdate = config.getResultingSection(valType+":"+self.name,
33  defaultDict = defaults,
34  demandPars = mandatories)
35  self.general.update(theUpdate)
36  self.jobmode = self.general["jobmode"]
37  self.NJobs = int(self.general["parallelJobs"])
38 
39  # limit maximum number of parallel jobs to 40
40  # (each output file is approximately 20MB)
41  maximumNumberJobs = 40
42  if self.NJobs > maximumNumberJobs:
43  msg = ("Maximum allowed number of parallel jobs "
44  +str(maximumNumberJobs)+" exceeded!!!")
45  raise AllInOneError(msg)
46 
47  self.cmssw = self.general["cmssw"]
48  badcharacters = r"\'"
49  for character in badcharacters:
50  if character in self.cmssw:
51  raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
52  "path name. If you really have it in such a ridiculously named location,\n"
53  "try making a symbolic link somewhere with a decent name.")
54  try:
55  os.listdir(self.cmssw)
56  except OSError:
57  raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
58 
59  if self.cmssw == os.environ["CMSSW_BASE"]:
60  self.scramarch = os.environ["SCRAM_ARCH"]
61  self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
62  else:
63  command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
64  ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
65  commandoutput = getCommandOutput2(command).split('\n')
66  self.cmssw = commandoutput[0]
67  self.scramarch = commandoutput[1]
68  self.cmsswreleasebase = commandoutput[2]
69 
70  self.AutoAlternates = True
71  if config.has_option("alternateTemplates","AutoAlternates"):
72  try:
73  self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
74  except ValueError:
75  raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
76 
77  knownOpts = defaults.keys()+mandatories
78  ignoreOpts = []
79  config.checkInput(valType+":"+self.name,
80  knownSimpleOptions = knownOpts,
81  ignoreOptions = ignoreOpts)
double split
Definition: MVATrainer.cc:139

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration (   self,
  fileContents,
  path,
  schedule = None,
  repMap = None,
  repMaps = None 
)

Definition at line 150 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles.

151  def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
152  self.configFiles = GenericValidation.createFiles(self, fileContents,
153  path, repMap = repMap, repMaps = repMaps)
154  if not schedule == None:
155  schedule = [os.path.join( path, cfgName) for cfgName in schedule]
156  for cfgName in schedule:
157  if not cfgName in self.configFiles:
158  msg = ("scheduled %s missing in generated configfiles: %s"
159  %(cfgName, self.configFiles))
160  raise AllInOneError(msg)
161  for cfgName in self.configFiles:
162  if not cfgName in schedule:
163  msg = ("generated configuration %s not scheduled: %s"
164  %(cfgName, schedule))
165  raise AllInOneError(msg)
166  self.configFiles = schedule
167  return self.configFiles
def genericValidation.GenericValidation.createCrabCfg (   self,
  fileContents,
  path 
)

Definition at line 176 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

177  def createCrabCfg(self, fileContents, path ):
178  if self.NJobs > 1:
179  msg = ("jobmode 'crab' not supported for parallel validation."
180  " Please set parallelJobs = 1.")
181  raise AllInOneError(msg)
182  self.crabConfigFiles = GenericValidation.createFiles(self, fileContents,
183  path)
184  return self.crabConfigFiles
185 
def genericValidation.GenericValidation.createFiles (   self,
  fileContents,
  path,
  repMap = None,
  repMaps = None 
)
repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 127 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, and helperFunctions.replaceByMap().

128  def createFiles(self, fileContents, path, repMap = None, repMaps = None):
129  """repMap: single map for all files
130  repMaps: a dict, with the filenames as the keys"""
131  if repMap is not None and repMaps is not None:
132  raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
133  result = []
134  for fileName in fileContents:
135  filePath = os.path.join(path, fileName)
136  result.append(filePath)
137 
138  for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
139  theFile = open( filePathi, "w" )
140  fileContentsi = fileContents[ fileName ]
141  if repMaps is not None:
142  repMap = repMaps[fileName]
143  if repMap is not None:
144  repMap.update({"nIndex": str(i)})
145  fileContentsi = replaceByMap(fileContentsi, repMap)
146  theFile.write( fileContentsi )
147  theFile.close()
148 
149  return result
def replaceByMap
— Helpers —############################
def genericValidation.GenericValidation.createScript (   self,
  fileContents,
  path,
  downloadFiles = [],
  repMap = None,
  repMaps = None 
)

Definition at line 168 of file genericValidation.py.

169  def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
170  self.scriptFiles = GenericValidation.createFiles(self, fileContents,
171  path, repMap = repMap, repMaps = repMaps)
172  for script in self.scriptFiles:
173  for scriptwithindex in addIndex(script, self.NJobs):
174  os.chmod(scriptwithindex,0755)
175  return self.scriptFiles
def genericValidation.GenericValidation.getCompareStrings (   self,
  requestId = None,
  plain = False 
)

Definition at line 103 of file genericValidation.py.

References genericValidation.GenericValidation.filesToCompare.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), and preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation().

104  def getCompareStrings( self, requestId = None, plain = False ):
105  result = {}
106  repMap = self.alignmentToValidate.getRepMap()
107  for validationId in self.filesToCompare:
108  repMap["file"] = self.filesToCompare[ validationId ]
109  if repMap["file"].startswith( "/castor/" ):
110  repMap["file"] = "rfio:%(file)s"%repMap
111  elif repMap["file"].startswith( "/store/" ):
112  repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
113  if plain:
114  result[validationId]=repMap["file"]
115  else:
116  result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
117  if requestId == None:
118  return result
119  else:
120  if not "." in requestId:
121  requestId += ".%s"%GenericValidation.defaultReferenceName
122  if not requestId.split(".")[-1] in result:
123  msg = ("could not find %s in reference Objects!"
124  %requestId.split(".")[-1])
125  raise AllInOneError(msg)
126  return result[ requestId.split(".")[-1] ]
def genericValidation.GenericValidation.getRepMap (   self,
  alignment = None 
)

Definition at line 82 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, genericValidation.GenericValidation.general, genericValidation.GenericValidation.randomWorkdirPart, and genericValidation.GenericValidation.scramarch.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), offlineValidation.OfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), and preexistingValidation.PreexistingValidation.getCompareStrings().

82 
83  def getRepMap(self, alignment = None):
84  if alignment == None:
85  alignment = self.alignmentToValidate
86  result = alignment.getRepMap()
87  result.update( self.general )
88  result.update({
89  "workdir": os.path.join(self.general["workdir"],
90  self.randomWorkdirPart),
91  "datadir": self.general["datadir"],
92  "logdir": self.general["logdir"],
93  "CommandLineTemplate": ("#run configfile and post-proccess it\n"
94  "cmsRun %(cfgFile)s\n"
95  "%(postProcess)s "),
96  "CMSSW_BASE": self.cmssw,
97  "SCRAM_ARCH": self.scramarch,
98  "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
99  "alignmentName": alignment.name,
100  "condLoad": alignment.getConditions(),
101  })
102  return result

Member Data Documentation

genericValidation.GenericValidation.alignmentToValidate

Definition at line 17 of file genericValidation.py.

Referenced by geometryComparison.GeometryComparison.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 69 of file genericValidation.py.

Referenced by offlineValidation.OfflineValidation.createConfiguration().

genericValidation.GenericValidation.cmssw

Definition at line 46 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 60 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.config

Definition at line 22 of file genericValidation.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), DTT0DBValidation.DTT0DBValidation.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), DTTTrigValid.DTTTrigValid.initProcess(), and DTCalibrationWorker.DTCalibrationWorker.run().

genericValidation.GenericValidation.configFiles

Definition at line 20 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 181 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"
static

Definition at line 12 of file genericValidation.py.

genericValidation.GenericValidation.filesToCompare

Definition at line 21 of file genericValidation.py.

Referenced by trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), preexistingValidation.PreexistingValidation.getCompareStrings(), and genericValidation.GenericValidation.getCompareStrings().

genericValidation.GenericValidation.general

Definition at line 18 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), monteCarloValidation.MonteCarloValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.jobmode

Definition at line 35 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 16 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), genericValidation.GenericValidationData.getRepMap(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

genericValidation.GenericValidation.NJobs

Definition at line 36 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 19 of file genericValidation.py.

Referenced by geometryComparison.GeometryComparison.createConfiguration(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 59 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 169 of file genericValidation.py.