test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
genericValidation.GenericValidation Class Reference
Inheritance diagram for genericValidation.GenericValidation:
genericValidation.GenericValidationData geometryComparison.GeometryComparison preexistingValidation.PreexistingValidation monteCarloValidation.MonteCarloValidation offlineValidation.OfflineValidation trackSplittingValidation.TrackSplittingValidation zMuMuValidation.ZMuMuValidation preexistingValidation.PreexistingGeometryComparison preexistingValidation.PreexistingMonteCarloValidation preexistingValidation.PreexistingOfflineValidation preexistingValidation.PreexistingTrackSplittingValidation preexistingValidation.PreexistingZMuMuValidation

Public Member Functions

def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def getCompareStrings
 
def getRepMap
 

Public Attributes

 alignmentToValidate
 
 AutoAlternates
 
 cmssw
 
 cmsswreleasebase
 
 config
 
 configFiles
 
 crabConfigFiles
 
 filesToCompare
 
 general
 
 jobmode
 
 name
 
 NJobs
 
 randomWorkdirPart
 
 scramarch
 
 scriptFiles
 

Static Public Attributes

string defaultReferenceName = "DEFAULT"
 

Detailed Description

Definition at line 11 of file genericValidation.py.

Constructor & Destructor Documentation

def genericValidation.GenericValidation.__init__ (   self,
  valName,
  alignment,
  config,
  valType,
  addDefaults = {},
  addMandatories = [] 
)

Definition at line 14 of file genericValidation.py.

14 
15  addDefaults = {}, addMandatories=[]):
16  import random
17  self.name = valName
18  self.alignmentToValidate = alignment
19  self.general = config.getGeneral()
20  self.randomWorkdirPart = "%0i"%random.randint(1,10e9)
21  self.configFiles = []
22  self.filesToCompare = {}
23  self.config = config
24 
25  defaults = {"jobmode": self.general["jobmode"],
26  "cmssw": os.environ['CMSSW_BASE'],
27  "parallelJobs": "1"
28  }
29  defaults.update(addDefaults)
30  mandatories = []
31  mandatories += addMandatories
32  theUpdate = config.getResultingSection(valType+":"+self.name,
33  defaultDict = defaults,
34  demandPars = mandatories)
35  self.general.update(theUpdate)
36  self.jobmode = self.general["jobmode"]
37  self.NJobs = int(self.general["parallelJobs"])
38 
39  # limit maximum number of parallel jobs to 40
40  # (each output file is approximately 20MB)
41  maximumNumberJobs = 40
42  if self.NJobs > maximumNumberJobs:
43  msg = ("Maximum allowed number of parallel jobs "
44  +str(maximumNumberJobs)+" exceeded!!!")
45  raise AllInOneError(msg)
46 
47  self.cmssw = self.general["cmssw"]
48  badcharacters = r"\'"
49  for character in badcharacters:
50  if character in self.cmssw:
51  raise AllInOneError("The bad characters " + badcharacters + " are not allowed in the cmssw\n"
52  "path name. If you really have it in such a ridiculously named location,\n"
53  "try making a symbolic link somewhere with a decent name.")
54  try:
55  os.listdir(self.cmssw)
56  except OSError:
57  raise AllInOneError("Your cmssw release " + self.cmssw + ' does not exist')
58 
59  if self.cmssw == os.environ["CMSSW_BASE"]:
60  self.scramarch = os.environ["SCRAM_ARCH"]
61  self.cmsswreleasebase = os.environ["CMSSW_RELEASE_BASE"]
62  else:
63  command = ("cd '" + self.cmssw + "' && eval `scramv1 ru -sh 2> /dev/null`"
64  ' && echo "$CMSSW_BASE\n$SCRAM_ARCH\n$CMSSW_RELEASE_BASE"')
65  commandoutput = getCommandOutput2(command).split('\n')
66  self.cmssw = commandoutput[0]
67  self.scramarch = commandoutput[1]
68  self.cmsswreleasebase = commandoutput[2]
69 
70  self.AutoAlternates = True
71  if config.has_option("alternateTemplates","AutoAlternates"):
72  try:
73  self.AutoAlternates = json.loads(config.get("alternateTemplates","AutoAlternates").lower())
74  except ValueError:
75  raise AllInOneError("AutoAlternates needs to be true or false, not %s" % config.get("alternateTemplates","AutoAlternates"))
76 
77  knownOpts = defaults.keys()+mandatories
78  ignoreOpts = []
79  config.checkInput(valType+":"+self.name,
80  knownSimpleOptions = knownOpts,
81  ignoreOptions = ignoreOpts)
double split
Definition: MVATrainer.cc:139

Member Function Documentation

def genericValidation.GenericValidation.createConfiguration (   self,
  fileContents,
  path,
  schedule = None,
  repMap = None,
  repMaps = None 
)

Definition at line 151 of file genericValidation.py.

References genericValidation.GenericValidation.configFiles.

152  def createConfiguration(self, fileContents, path, schedule = None, repMap = None, repMaps = None):
153  self.configFiles = GenericValidation.createFiles(self, fileContents,
154  path, repMap = repMap, repMaps = repMaps)
155  if not schedule == None:
156  schedule = [os.path.join( path, cfgName) for cfgName in schedule]
157  for cfgName in schedule:
158  if not cfgName in self.configFiles:
159  msg = ("scheduled %s missing in generated configfiles: %s"
160  %(cfgName, self.configFiles))
161  raise AllInOneError(msg)
162  for cfgName in self.configFiles:
163  if not cfgName in schedule:
164  msg = ("generated configuration %s not scheduled: %s"
165  %(cfgName, schedule))
166  raise AllInOneError(msg)
167  self.configFiles = schedule
168  return self.configFiles
def genericValidation.GenericValidation.createCrabCfg (   self,
  fileContents,
  path 
)

Definition at line 177 of file genericValidation.py.

References genericValidation.GenericValidation.NJobs.

178  def createCrabCfg(self, fileContents, path ):
179  if self.NJobs > 1:
180  msg = ("jobmode 'crab' not supported for parallel validation."
181  " Please set parallelJobs = 1.")
182  raise AllInOneError(msg)
183  self.crabConfigFiles = GenericValidation.createFiles(self, fileContents,
184  path)
185  return self.crabConfigFiles
186 
def genericValidation.GenericValidation.createFiles (   self,
  fileContents,
  path,
  repMap = None,
  repMaps = None 
)
repMap: single map for all files
   repMaps: a dict, with the filenames as the keys

Definition at line 128 of file genericValidation.py.

References helperFunctions.addIndex(), genericValidation.GenericValidation.NJobs, and helperFunctions.replaceByMap().

129  def createFiles(self, fileContents, path, repMap = None, repMaps = None):
130  """repMap: single map for all files
131  repMaps: a dict, with the filenames as the keys"""
132  if repMap is not None and repMaps is not None:
133  raise AllInOneError("createFiles can only take repMap or repMaps (or neither), not both")
134  result = []
135  for fileName in fileContents:
136  filePath = os.path.join(path, fileName)
137  result.append(filePath)
138 
139  for (i, filePathi) in enumerate(addIndex(filePath, self.NJobs)):
140  theFile = open( filePathi, "w" )
141  fileContentsi = fileContents[ fileName ]
142  if repMaps is not None:
143  repMap = repMaps[fileName]
144  if repMap is not None:
145  repMap.update({"nIndex": str(i)})
146  fileContentsi = replaceByMap(fileContentsi, repMap)
147  theFile.write( fileContentsi )
148  theFile.close()
149 
150  return result
def replaceByMap
— Helpers —############################
def genericValidation.GenericValidation.createScript (   self,
  fileContents,
  path,
  downloadFiles = [],
  repMap = None,
  repMaps = None 
)

Definition at line 169 of file genericValidation.py.

170  def createScript(self, fileContents, path, downloadFiles=[], repMap = None, repMaps = None):
171  self.scriptFiles = GenericValidation.createFiles(self, fileContents,
172  path, repMap = repMap, repMaps = repMaps)
173  for script in self.scriptFiles:
174  for scriptwithindex in addIndex(script, self.NJobs):
175  os.chmod(scriptwithindex,0o755)
176  return self.scriptFiles
def genericValidation.GenericValidation.getCompareStrings (   self,
  requestId = None,
  plain = False 
)

Definition at line 104 of file genericValidation.py.

References genericValidation.GenericValidation.filesToCompare.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), and preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation().

105  def getCompareStrings( self, requestId = None, plain = False ):
106  result = {}
107  repMap = self.alignmentToValidate.getRepMap()
108  for validationId in self.filesToCompare:
109  repMap["file"] = self.filesToCompare[ validationId ]
110  if repMap["file"].startswith( "/castor/" ):
111  repMap["file"] = "rfio:%(file)s"%repMap
112  elif repMap["file"].startswith( "/store/" ):
113  repMap["file"] = "root://eoscms.cern.ch//eos/cms%(file)s"%repMap
114  if plain:
115  result[validationId]=repMap["file"]
116  else:
117  result[validationId]= "%(file)s=%(title)s|%(color)s|%(style)s"%repMap
118  if requestId == None:
119  return result
120  else:
121  if not "." in requestId:
122  requestId += ".%s"%GenericValidation.defaultReferenceName
123  if not requestId.split(".")[-1] in result:
124  msg = ("could not find %s in reference Objects!"
125  %requestId.split(".")[-1])
126  raise AllInOneError(msg)
127  return result[ requestId.split(".")[-1] ]
def genericValidation.GenericValidation.getRepMap (   self,
  alignment = None 
)

Definition at line 82 of file genericValidation.py.

References genericValidation.GenericValidation.alignmentToValidate, genericValidation.GenericValidation.cmssw, genericValidation.GenericValidation.cmsswreleasebase, genericValidation.GenericValidation.general, genericValidation.GenericValidation.randomWorkdirPart, and genericValidation.GenericValidation.scramarch.

Referenced by trackSplittingValidation.TrackSplittingValidation.appendToExtendedValidation(), zMuMuValidation.ZMuMuValidation.appendToExtendedValidation(), offlineValidation.OfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingOfflineValidation.appendToExtendedValidation(), preexistingValidation.PreexistingTrackSplittingValidation.appendToExtendedValidation(), trackSplittingValidation.TrackSplittingValidation.appendToMerge(), offlineValidation.OfflineValidation.appendToMerge(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), monteCarloValidation.MonteCarloValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), geometryComparison.GeometryComparison.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), and preexistingValidation.PreexistingValidation.getCompareStrings().

82 
83  def getRepMap(self, alignment = None):
84  if alignment == None:
85  alignment = self.alignmentToValidate
86  result = alignment.getRepMap()
87  result.update( self.general )
88  result.update({
89  "workdir": os.path.join(self.general["workdir"],
90  self.randomWorkdirPart),
91  "datadir": self.general["datadir"],
92  "logdir": self.general["logdir"],
93  "CommandLineTemplate": ("#run configfile and post-proccess it\n"
94  "cmsRun %(cfgFile)s\n"
95  "%(postProcess)s "),
96  "CMSSW_BASE": self.cmssw,
97  "SCRAM_ARCH": self.scramarch,
98  "CMSSW_RELEASE_BASE": self.cmsswreleasebase,
99  "alignmentName": alignment.name,
100  "condLoad": alignment.getConditions(),
101  "condLoad": alignment.getConditions(),
102  })
103  return result

Member Data Documentation

genericValidation.GenericValidation.alignmentToValidate

Definition at line 17 of file genericValidation.py.

Referenced by zMuMuValidation.ZMuMuValidation.getRepMap(), geometryComparison.GeometryComparison.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.AutoAlternates

Definition at line 69 of file genericValidation.py.

Referenced by offlineValidation.OfflineValidation.createConfiguration().

genericValidation.GenericValidation.cmssw

Definition at line 46 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.getRepMap(), and trackSplittingValidation.TrackSplittingValidation.validsubdets().

genericValidation.GenericValidation.cmsswreleasebase

Definition at line 60 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), genericValidation.GenericValidation.getRepMap(), and trackSplittingValidation.TrackSplittingValidation.validsubdets().

genericValidation.GenericValidation.config

Definition at line 22 of file genericValidation.py.

Referenced by DTNoiseCalibration.DTNoiseCalibration.initCrab(), DTTTrigProd.DTTTrigProd.initCrab(), DTDQMValidation.DTDQMValidation.initCrab(), DTTTrigValid.DTTTrigValid.initCrab(), DTAnalysisResiduals.DTAnalysisResiduals.initCrab(), DTResidualCalibration.DTResidualCalibration.initCrab(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initCrab(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initCrab(), DTT0DBValidation.DTT0DBValidation.initProcess(), DTDqm.DTDqm.initProcess(), DTDQMMerge.DTDQMMerge.initProcess(), DTTTrigProd.DTTTrigProd.initProcess(), DTAnalysisResiduals.DTAnalysisResiduals.initProcess(), DTVDriftSegmentCalibration.DTVDriftSegmentCalibration.initProcess(), DTVDriftMeanTimerCalibration.DTVDriftMeanTimerCalibration.initProcess(), DTDQMHarvesting.DTDQMHarvesting.initProcess(), DTResidualCalibration.DTResidualCalibration.initProcess(), DTNoiseCalibration.DTNoiseCalibration.initProcess(), DTDQMValidation.DTDQMValidation.initProcess(), DTTTrigValid.DTTTrigValid.initProcess(), and DTCalibrationWorker.DTCalibrationWorker.run().

genericValidation.GenericValidation.configFiles

Definition at line 20 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), and genericValidation.GenericValidationData.createScript().

genericValidation.GenericValidation.crabConfigFiles

Definition at line 182 of file genericValidation.py.

string genericValidation.GenericValidation.defaultReferenceName = "DEFAULT"
static

Definition at line 12 of file genericValidation.py.

genericValidation.GenericValidation.filesToCompare

Definition at line 21 of file genericValidation.py.

Referenced by monteCarloValidation.MonteCarloValidation.createConfiguration(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), geometryComparison.GeometryComparison.createScript(), preexistingValidation.PreexistingValidation.getCompareStrings(), and genericValidation.GenericValidation.getCompareStrings().

genericValidation.GenericValidation.general

Definition at line 18 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), trackSplittingValidation.TrackSplittingValidation.getRepMap(), monteCarloValidation.MonteCarloValidation.getRepMap(), zMuMuValidation.ZMuMuValidation.getRepMap(), preexistingValidation.PreexistingValidation.getRepMap(), offlineValidation.OfflineValidation.getRepMap(), and genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.jobmode

Definition at line 35 of file genericValidation.py.

genericValidation.GenericValidation.name

Definition at line 16 of file genericValidation.py.

Referenced by ElectronMVAID.ElectronMVAID.__call__(), dirstructure.Directory.__create_pie_image(), dqm_interfaces.DirID.__eq__(), BeautifulSoup.Tag.__eq__(), dirstructure.Directory.__get_full_path(), dirstructure.Comparison.__get_img_name(), dataset.Dataset.__getDataType(), dataset.Dataset.__getFileInfoList(), cuy.divideElement.__init__(), cuy.plotElement.__init__(), cuy.additionElement.__init__(), cuy.superimposeElement.__init__(), cuy.graphElement.__init__(), dirstructure.Comparison.__make_image(), core.autovars.NTupleVariable.__repr__(), core.autovars.NTupleObjectType.__repr__(), core.autovars.NTupleObject.__repr__(), core.autovars.NTupleCollection.__repr__(), dirstructure.Directory.__repr__(), dqm_interfaces.DirID.__repr__(), dirstructure.Comparison.__repr__(), config.Service.__setattr__(), config.CFG.__str__(), counter.Counter.__str__(), average.Average.__str__(), BeautifulSoup.Tag.__str__(), BeautifulSoup.SoupStrainer.__str__(), core.autovars.NTupleObjectType.addSubObjects(), core.autovars.NTupleObjectType.addVariables(), core.autovars.NTupleObjectType.allVars(), dirstructure.Directory.calcStats(), monteCarloValidation.MonteCarloValidation.createConfiguration(), trackSplittingValidation.TrackSplittingValidation.createConfiguration(), offlineValidation.OfflineValidation.createConfiguration(), zMuMuValidation.ZMuMuValidation.createConfiguration(), genericValidation.GenericValidationData.createCrabCfg(), geometryComparison.GeometryComparison.createScript(), genericValidation.GenericValidationData.createScript(), validation.Sample.digest(), python.rootplot.utilities.Hist.divide(), python.rootplot.utilities.Hist.divide_wilson(), TreeCrawler.Package.dump(), core.autovars.NTupleVariable.fillBranch(), core.autovars.NTupleObject.fillBranches(), core.autovars.NTupleCollection.fillBranchesScalar(), core.autovars.NTupleCollection.fillBranchesVector(), core.autovars.NTupleCollection.get_cpp_declaration(), core.autovars.NTupleCollection.get_cpp_wrapper_class(), core.autovars.NTupleCollection.get_py_wrapper_class(), utils.StatisticalTest.get_status(), production_tasks.Task.getname(), dataset.CMSDataset.getPrimaryDatasetEntries(), dataset.PrivateDataset.getPrimaryDatasetEntries(), zMuMuValidation.ZMuMuValidation.getRepMap(), genericValidation.GenericValidationData.getRepMap(), VIDSelectorBase.VIDSelectorBase.initialize(), personalPlayback.Applet.log(), core.autovars.NTupleVariable.makeBranch(), core.autovars.NTupleObject.makeBranches(), core.autovars.NTupleCollection.makeBranchesScalar(), core.autovars.NTupleCollection.makeBranchesVector(), dirstructure.Directory.print_report(), dataset.BaseDataset.printInfo(), dataset.Dataset.printInfo(), production_tasks.MonitorJobs.run(), BeautifulSoup.SoupStrainer.searchTag(), python.rootplot.utilities.Hist.TGraph(), python.rootplot.utilities.Hist.TH1F(), Vispa.Views.PropertyView.Property.valueChanged(), counter.Counter.write(), and average.Average.write().

genericValidation.GenericValidation.NJobs

Definition at line 36 of file genericValidation.py.

Referenced by genericValidation.GenericValidationData.__init__(), offlineValidation.OfflineValidation.createConfiguration(), genericValidation.GenericValidation.createCrabCfg(), genericValidation.GenericValidation.createFiles(), genericValidation.GenericValidationData.createScript(), and genericValidation.GenericValidationData.getRepMap().

genericValidation.GenericValidation.randomWorkdirPart

Definition at line 19 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scramarch

Definition at line 59 of file genericValidation.py.

Referenced by genericValidation.GenericValidation.getRepMap().

genericValidation.GenericValidation.scriptFiles

Definition at line 170 of file genericValidation.py.