CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Private Attributes
offlineValidation.OfflineValidationParallel Class Reference
Inheritance diagram for offlineValidation.OfflineValidationParallel:
offlineValidation.OfflineValidation genericValidation.GenericValidationData genericValidation.GenericValidation

Public Member Functions

def __init__
 
def appendToMergeParJobs
 
def createConfiguration
 
def createCrabCfg
 
def createScript
 
def getRepMap
 
- Public Member Functions inherited from offlineValidation.OfflineValidation
def __init__
 
def appendToExtendedValidation
 
def appendToMerge
 
def createConfiguration
 
def createCrabCfg
 
def createScript
 
def getRepMap
 
- Public Member Functions inherited from genericValidation.GenericValidationData
def __init__
 
def createCrabCfg
 
- Public Member Functions inherited from genericValidation.GenericValidation
def __init__
 
def createConfiguration
 
def createCrabCfg
 
def createFiles
 
def createScript
 
def getCompareStrings
 
def getRepMap
 

Private Attributes

 __NJobs
 

Additional Inherited Members

- Public Attributes inherited from genericValidation.GenericValidationData
 crabWorkingDir
 
 dataset
 
 jobmode
 
- Public Attributes inherited from genericValidation.GenericValidation
 alignmentToValidate
 
 config
 
 configFiles
 
 crabConfigFiles
 
 filesToCompare
 
 general
 
 jobmode
 
 name
 
 randomWorkdirPart
 
 scriptFiles
 
- Static Public Attributes inherited from genericValidation.GenericValidation
string defaultReferenceName = "DEFAULT"
 

Detailed Description

Definition at line 98 of file offlineValidation.py.

Constructor & Destructor Documentation

def offlineValidation.OfflineValidationParallel.__init__ (   self,
  valName,
  alignment,
  config 
)

Definition at line 99 of file offlineValidation.py.

References genericValidation.GenericValidation.jobmode, entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

99 
100  def __init__(self, valName, alignment,config):
101  OfflineValidation.__init__(self, valName, alignment, config)
102  defaults = {
103  "parallelJobs":"1",
104  "jobmode":self.jobmode
105  }
106  offline = config.getResultingSection( "offline:"+self.name,
107  defaultDict = defaults )
108  self.general.update( offline )
109  self.__NJobs = self.general["parallelJobs"]

Member Function Documentation

def offlineValidation.OfflineValidationParallel.appendToMergeParJobs (   self,
  validationsSoFar = "" 
) 
if no argument or "" is passed a string with an instantiation is returned,
else the validation is appended to the list

Definition at line 206 of file offlineValidation.py.

References offlineValidation.OfflineValidationParallel.__NJobs, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

206 
207  def appendToMergeParJobs( self, validationsSoFar = "" ):
208  """
209  if no argument or "" is passed a string with an instantiation is returned,
210  else the validation is appended to the list
211  """
212  repMap = self.getRepMap()
213 
214  parameters = ""
215  fileToAdd = ""
216  for index in range(int(self.__NJobs)):
217  fileToAdd = '%(outputFile)s'%repMap
218  fileToAdd = fileToAdd.replace('.root','_'+str(index)+'.root')
219  if index < int( self.general["parallelJobs"] )-1:
220  parameters = parameters+fileToAdd+','
221  else:
222  parameters = parameters+fileToAdd
223 
224  mergedoutputfile = ("AlignmentValidation_" + self.name + "_"
225  + '%(name)s'%repMap + ".root")
226  validationsSoFar += ('root -x -b -q "TkAlOfflineJobsMerge.C(\\\"'
227  +parameters+'\\\",\\\"'+mergedoutputfile+'\\\")"'
228  +"\n")
229  return validationsSoFar
def offlineValidation.OfflineValidationParallel.createConfiguration (   self,
  path,
  configBaseName = "TkAlOfflineValidation" 
)

Definition at line 110 of file offlineValidation.py.

References genericValidation.GenericValidation.filesToCompare, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, plotscripts.SawTeethFunction.name, and helperFunctions.replaceByMap().

110 
111  def createConfiguration(self, path, configBaseName = "TkAlOfflineValidation"):
112  # if offline validation uses N parallel jobs, we create here N cfg files
113  numberParallelJobs = int( self.general["parallelJobs"] )
114  # limit maximum number of parallel jobs to 40
115  # (each output file is approximately 20MB)
116  maximumNumberJobs = 40
117  if numberParallelJobs > maximumNumberJobs:
118  msg = ("Maximum allowed number of parallel jobs "
119  +str(maximumNumberJobs)+" exceeded!!!")
120  raise AllInOneError(msg)
121  # if maxevents is not specified, cannot calculate number of events for
122  # each parallel job, and therefore running only a single job
123  if int( self.general["maxevents"] ) == -1:
124  msg = ("Maximum number of events (maxevents) not specified: "
125  "cannot use parallel jobs in offline validation")
126  raise AllInOneError(msg)
127  if numberParallelJobs > 1:
128  if self.general["offlineModuleLevelHistsTransient"] == "True":
129  msg = ("To be able to merge results when running parallel jobs,"
130  " set offlineModuleLevelHistsTransient to false.")
131  raise AllInOneError(msg)
132  for index in range(numberParallelJobs):
133  cfgName = "%s.%s.%s_%s_cfg.py"%(configBaseName, self.name,
134  self.alignmentToValidate.name,
135  str(index))
136  repMap = self.getRepMap()
137  # in this parallel job, skip index*(maxEvents/nJobs) events from
138  # the beginning
139  # (first index is zero, so no skipping for a single job)
140  # and use _index_ in the name of the output file
141  repMap.update({"nIndex": str(index)})
142  # Create the result file directly to datadir since should not use /tmp/
143  # see https://cern.service-now.com/service-portal/article.do?n=KB0000484
144  repMap.update({
145  "outputFile": replaceByMap("AlignmentValidation_"
146  + self.name +
147  "_.oO[name]Oo._.oO[nIndex]Oo..root",
148  repMap )
149  })
150  repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] )
151 
152  cfgs = {cfgName:replaceByMap(configTemplates.offlineParallelTemplate,
153  repMap)}
154  self.filesToCompare[GenericValidationData.defaultReferenceName] = repMap["resultFile"]
155  GenericValidationData.createConfiguration(self, cfgs, path)
156  # here is a small problem. only the last cfgs is saved
157  # it requires a bit ugly solution later
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def offlineValidation.OfflineValidationParallel.createCrabCfg (   self)

Definition at line 230 of file offlineValidation.py.

230 
231  def createCrabCfg( self ):
232  msg = ("jobmode 'crab' not supported for 'offlineParallel' validation."
233  " Please choose another 'jobmode'.")
234  raise AllInOneError(msg)
235 
def offlineValidation.OfflineValidationParallel.createScript (   self,
  path,
  scriptBaseName = "TkAlOfflineValidation" 
)

Definition at line 158 of file offlineValidation.py.

References genericValidation.GenericValidation.configFiles, genericValidation.GenericValidation.general, genericValidation.GenericValidation.getRepMap(), alignment.Alignment.getRepMap(), entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, plotscripts.SawTeethFunction.name, and helperFunctions.replaceByMap().

158 
159  def createScript(self, path, scriptBaseName = "TkAlOfflineValidation"):
160  # A separate script is created for each parallel jobs.
161  # Since only one cfg is saved a bit ugly solution is needed in the loop.
162  returnValue = []
163  numJobs = int( self.general["parallelJobs"] )
164  for index in range(numJobs):
165  scriptName = "%s.%s.%s_%s.sh"%(scriptBaseName, self.name,
166  self.alignmentToValidate.name,
167  str(index))
168  repMap = self.getRepMap()
169  repMap["nIndex"]=""
170  repMap["nIndex"]=str(index)
171  repMap["CommandLine"]=""
172  repMap.update({
173  "outputFile": replaceByMap("AlignmentValidation_"
174  + self.name +
175  "_.oO[name]Oo._.oO[nIndex]Oo..root",
176  repMap )
177  })
178  for cfg in self.configFiles:
179  # The ugly solution here is to change the name for each parallel job
180  cfgtemp = cfg.replace(str(numJobs-1)+"_cfg.py",
181  str(index)+"_cfg.py")
182  repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":cfgtemp,
183  "postProcess":""
184  }
185  scripts = {scriptName: replaceByMap(configTemplates.parallelScriptTemplate,
186  repMap ) }
187  returnValue.extend(GenericValidationData.createScript(self,
188  scripts,
189  path) )
190  return returnValue
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def offlineValidation.OfflineValidationParallel.getRepMap (   self,
  alignment = None 
)

Definition at line 191 of file offlineValidation.py.

References genericValidation.GenericValidation.general, entry.name, genericValidation.GenericValidation.name, alignment.Alignment.name, MuonGeometrySanityCheckPoint.name, and plotscripts.SawTeethFunction.name.

191 
192  def getRepMap(self, alignment = None):
193  repMap = OfflineValidation.getRepMap(self, alignment)
194  repMap.update({
195  "nJobs": self.general["parallelJobs"],
196  "offlineValidationFileOutput":
197  configTemplates.offlineParallelFileOutputTemplate,
198  "nameValidation": self.name
199  })
200  # In case maxevents==-1, set number of parallel jobs to 1
201  # since we cannot calculate number of events for each
202  # parallel job
203  if str(self.general["maxevents"]) == "-1":
204  repMap.update({ "nJobs": "1" })
205  return repMap

Member Data Documentation

offlineValidation.OfflineValidationParallel.__NJobs
private

Definition at line 108 of file offlineValidation.py.

Referenced by offlineValidation.OfflineValidationParallel.appendToMergeParJobs().

Generated for CMSSW Reference Manual by doxygen 1.8.5