test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Private Attributes
offlineValidation.OfflineValidationParallel Class Reference
Inheritance diagram for offlineValidation.OfflineValidationParallel:
offlineValidation.OfflineValidation genericValidation.GenericValidationData genericValidation.GenericValidation

Public Member Functions

def __init__
 
def appendToMergeParJobs
 
def createConfiguration
 
def createCrabCfg
 
def createScript
 
def getRepMap
 
- Public Member Functions inherited from offlineValidation.OfflineValidation
def __init__
 
def appendToExtendedValidation
 
def appendToMerge
 
def createConfiguration
 
def createCrabCfg
 
def createScript
 
def getRepMap
 
- Public Member Functions inherited from genericValidation.GenericValidationData
def __init__
 
def createCrabCfg
 
- Public Member Functions inherited from genericValidation.GenericValidation
def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def getCompareStrings
 
def getRepMap
 

Public Attributes

 outputFiles
 
- Public Attributes inherited from genericValidation.GenericValidationData
 crabWorkingDir
 
 dataset
 
 jobmode
 
- Public Attributes inherited from genericValidation.GenericValidation
 alignmentToValidate
 
 config
 
 configFiles
 
 crabConfigFiles
 
 filesToCompare
 
 general
 
 jobmode
 
 name
 
 randomWorkdirPart
 
 scriptFiles
 

Private Attributes

 __NJobs
 

Additional Inherited Members

- Static Public Attributes inherited from genericValidation.GenericValidation
string defaultReferenceName = "DEFAULT"
 

Detailed Description

Definition at line 98 of file offlineValidation.py.

Constructor & Destructor Documentation

def offlineValidation.OfflineValidationParallel.__init__ (   self,
  valName,
  alignment,
  config 
)

Definition at line 99 of file offlineValidation.py.

References genericValidation.GenericValidation.jobmode, entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

99 
100  def __init__(self, valName, alignment,config):
101  OfflineValidation.__init__(self, valName, alignment, config)
102  defaults = {
103  "parallelJobs":"1",
104  "jobmode":self.jobmode
105  }
106  offline = config.getResultingSection( "offline:"+self.name,
107  defaultDict = defaults )
108  self.general.update( offline )
109  self.__NJobs = self.general["parallelJobs"]
110  self.outputFiles = []
111  for index in range(int(self.general["parallelJobs"])):
112  fName = replaceByMap("AlignmentValidation_"+self.name
113  +"_.oO[name]Oo._%d.root"%(index),
114  self.getRepMap())
115  self.outputFiles.append(fName)
116 
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5

Member Function Documentation

def offlineValidation.OfflineValidationParallel.appendToMergeParJobs (   self,
  validationsSoFar = "" 
) 
if no argument or "" is passed a string with an instantiation is returned,
else the validation is appended to the list

Definition at line 203 of file offlineValidation.py.

References offlineValidation.OfflineValidationParallel.__NJobs, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

203 
204  def appendToMergeParJobs( self, validationsSoFar = "" ):
205  """
206  if no argument or "" is passed a string with an instantiation is returned,
207  else the validation is appended to the list
208  """
209  repMap = self.getRepMap()
210 
211  parameters = ""
212  fileToAdd = ""
213  for index in range(int(self.__NJobs)):
214  fileToAdd = '%(outputFile)s'%repMap
215  fileToAdd = fileToAdd.replace('.root','_'+str(index)+'.root')
216  if index < int( self.general["parallelJobs"] )-1:
217  parameters = parameters+fileToAdd+','
218  else:
219  parameters = parameters+fileToAdd
220 
221  mergedoutputfile = ("AlignmentValidation_" + self.name + "_"
222  + '%(name)s'%repMap + ".root")
223  validationsSoFar += ('root -x -b -q -l "TkAlOfflineJobsMerge.C(\\\"'
224  +parameters+'\\\",\\\"'+mergedoutputfile+'\\\")"'
225  +"\n")
226  return validationsSoFar
def offlineValidation.OfflineValidationParallel.createConfiguration (   self,
  path,
  configBaseName = "TkAlOfflineValidation" 
)

Definition at line 117 of file offlineValidation.py.

References genericValidation.GenericValidation.filesToCompare, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, plotscripts.SawTeethFunction.name, offlineValidation.OfflineValidationParallel.outputFiles, and helperFunctions.replaceByMap().

117 
118  def createConfiguration(self, path, configBaseName = "TkAlOfflineValidation"):
119  # if offline validation uses N parallel jobs, we create here N cfg files
120  numberParallelJobs = int( self.general["parallelJobs"] )
121  # limit maximum number of parallel jobs to 40
122  # (each output file is approximately 20MB)
123  maximumNumberJobs = 40
124  if numberParallelJobs > maximumNumberJobs:
125  msg = ("Maximum allowed number of parallel jobs "
126  +str(maximumNumberJobs)+" exceeded!!!")
127  raise AllInOneError(msg)
128  # if maxevents is not specified, cannot calculate number of events for
129  # each parallel job, and therefore running only a single job
130  if int( self.general["maxevents"] ) == -1:
131  msg = ("Maximum number of events (maxevents) not specified: "
132  "cannot use parallel jobs in offline validation")
133  raise AllInOneError(msg)
134  if numberParallelJobs > 1:
135  if self.general["offlineModuleLevelHistsTransient"] == "True":
136  msg = ("To be able to merge results when running parallel jobs,"
137  " set offlineModuleLevelHistsTransient to false.")
138  raise AllInOneError(msg)
139  for index in range(numberParallelJobs):
140  cfgName = "%s.%s.%s_%s_cfg.py"%(configBaseName, self.name,
141  self.alignmentToValidate.name,
142  str(index))
143  repMap = self.getRepMap()
144  # in this parallel job, skip index*(maxEvents/nJobs) events from
145  # the beginning
146  # (first index is zero, so no skipping for a single job)
147  # and use _index_ in the name of the output file
148  repMap.update({"nIndex": str(index)})
149  # Create the result file directly to datadir since should not use /tmp/
150  # see https://cern.service-now.com/service-portal/article.do?n=KB0000484
151  repMap.update({"outputFile": self.outputFiles[index]})
152  repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] )
153 
154  cfgs = {cfgName:replaceByMap(configTemplates.offlineParallelTemplate,
155  repMap)}
156  self.filesToCompare[GenericValidationData.defaultReferenceName] = repMap["resultFile"]
157  GenericValidationData.createConfiguration(self, cfgs, path)
158  # here is a small problem. only the last cfgs is saved
159  # it requires a bit ugly solution later
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def offlineValidation.OfflineValidationParallel.createCrabCfg (   self)

Definition at line 227 of file offlineValidation.py.

227 
228  def createCrabCfg( self ):
229  msg = ("jobmode 'crab' not supported for 'offlineParallel' validation."
230  " Please choose another 'jobmode'.")
231  raise AllInOneError(msg)
232 
def offlineValidation.OfflineValidationParallel.createScript (   self,
  path,
  scriptBaseName = "TkAlOfflineValidation" 
)

Definition at line 160 of file offlineValidation.py.

References genericValidation.GenericValidation.configFiles, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, plotscripts.SawTeethFunction.name, offlineValidation.OfflineValidationParallel.outputFiles, and helperFunctions.replaceByMap().

160 
161  def createScript(self, path, scriptBaseName = "TkAlOfflineValidation"):
162  # A separate script is created for each parallel jobs.
163  # Since only one cfg is saved a bit ugly solution is needed in the loop.
164  returnValue = []
165  numJobs = int( self.general["parallelJobs"] )
166  for index in range(numJobs):
167  scriptName = "%s.%s.%s_%s.sh"%(scriptBaseName, self.name,
168  self.alignmentToValidate.name,
169  str(index))
170  repMap = self.getRepMap()
171  repMap["nIndex"]=""
172  repMap["nIndex"]=str(index)
173  repMap["CommandLine"]=""
174  repMap.update({"outputFile": self.outputFiles[index]})
175  for cfg in self.configFiles:
176  # The ugly solution here is to change the name for each parallel job
177  cfgtemp = cfg.replace(str(numJobs-1)+"_cfg.py",
178  str(index)+"_cfg.py")
179  repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":cfgtemp,
180  "postProcess":""
181  }
182  scripts = {scriptName: replaceByMap(configTemplates.parallelScriptTemplate,
183  repMap ) }
184  returnValue.extend(GenericValidationData.createScript(self,
185  scripts,
186  path) )
187  return returnValue
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def offlineValidation.OfflineValidationParallel.getRepMap (   self,
  alignment = None 
)

Definition at line 188 of file offlineValidation.py.

References genericValidation.GenericValidation.general, entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

188 
189  def getRepMap(self, alignment = None):
190  repMap = OfflineValidation.getRepMap(self, alignment)
191  repMap.update({
192  "nJobs": self.general["parallelJobs"],
193  "offlineValidationFileOutput":
194  configTemplates.offlineParallelFileOutputTemplate,
195  "nameValidation": self.name
196  })
197  # In case maxevents==-1, set number of parallel jobs to 1
198  # since we cannot calculate number of events for each
199  # parallel job
200  if str(self.general["maxevents"]) == "-1":
201  repMap.update({ "nJobs": "1" })
202  return repMap

Member Data Documentation

offlineValidation.OfflineValidationParallel.__NJobs
private

Definition at line 108 of file offlineValidation.py.

Referenced by offlineValidation.OfflineValidationParallel.appendToMergeParJobs().

offlineValidation.OfflineValidationParallel.outputFiles

Definition at line 109 of file offlineValidation.py.

Referenced by offlineValidation.OfflineValidationParallel.createConfiguration(), and offlineValidation.OfflineValidationParallel.createScript().

Generated for CMSSW Reference Manual by doxygen 1.8.5