offlineValidation::OfflineValidationParallel Class Reference

Inheritance diagram for offlineValidation::OfflineValidationParallel:

List of all members.

Public Member Functions
def	__init__
def	appendToMergeParJobs
def	createConfiguration
def	createCrabCfg
def	createScript
def	getRepMap
Private Attributes
	__NJobs

---

Detailed Description

Definition at line 98 of file offlineValidation.py.

---

Constructor & Destructor Documentation

def offlineValidation::OfflineValidationParallel::__init__	(		self,
			valName,
			alignment,
			config
	)

Reimplemented from offlineValidation::OfflineValidation.

Definition at line 99 of file offlineValidation.py.

                                                 :
        OfflineValidation.__init__(self, valName, alignment, config)
        defaults = {
            "parallelJobs":"1",
            "jobmode":self.jobmode
            }
        offline = config.getResultingSection( "offline:"+self.name, 
                                              defaultDict = defaults )
        self.general.update( offline )
        self.__NJobs = self.general["parallelJobs"]

---

Member Function Documentation

def offlineValidation::OfflineValidationParallel::appendToMergeParJobs	(		self,
			validationsSoFar = ""
	)

if no argument or "" is passed a string with an instantiation is returned,
else the validation is appended to the list

Definition at line 206 of file offlineValidation.py.

                                                           :
        """
        if no argument or "" is passed a string with an instantiation is returned,
        else the validation is appended to the list
        """
        repMap = self.getRepMap()

        parameters = ""
        fileToAdd = ""
        for index in range(int(self.__NJobs)):
            fileToAdd = '%(outputFile)s'%repMap
            fileToAdd = fileToAdd.replace('.root','_'+str(index)+'.root')
            if index < int( self.general["parallelJobs"] )-1:
                parameters = parameters+fileToAdd+','
            else:
                parameters = parameters+fileToAdd                
                
        mergedoutputfile = ("AlignmentValidation_" + self.name + "_"
                            + '%(name)s'%repMap + ".root")
        validationsSoFar += ('root -x -b -q "TkAlOfflineJobsMerge.C(\\\"'
                             +parameters+'\\\",\\\"'+mergedoutputfile+'\\\")"'
                             +"\n")
        return validationsSoFar

def offlineValidation::OfflineValidationParallel::createConfiguration	(		self,
			path,
			configBaseName = "TkAlOfflineValidation"
	)

Reimplemented from offlineValidation::OfflineValidation.

Definition at line 110 of file offlineValidation.py.

00111                                                                                  :
00112         # if offline validation uses N parallel jobs, we create here N cfg files
00113         numberParallelJobs = int( self.general["parallelJobs"] )
00114         # limit maximum number of parallel jobs to 40
00115         # (each output file is approximately 20MB)
00116         maximumNumberJobs = 40
00117         if numberParallelJobs > maximumNumberJobs:
00118             msg = ("Maximum allowed number of parallel jobs "
00119                    +str(maximumNumberJobs)+" exceeded!!!")
00120             raise AllInOneError(msg)
00121         # if maxevents is not specified, cannot calculate number of events for
00122         # each parallel job, and therefore running only a single job
00123         if int( self.general["maxevents"] ) == -1:
00124             msg = ("Maximum number of events (maxevents) not specified: "
00125                    "cannot use parallel jobs in offline validation")
00126             raise AllInOneError(msg)
00127         if numberParallelJobs > 1:    
00128             if self.general["offlineModuleLevelHistsTransient"] == "True":
00129                 msg = ("To be able to merge results when running parallel jobs,"
00130                        " set offlineModuleLevelHistsTransient to false.")
00131                 raise AllInOneError(msg)
00132         for index in range(numberParallelJobs):
00133             cfgName = "%s.%s.%s_%s_cfg.py"%(configBaseName, self.name,
00134                                             self.alignmentToValidate.name,
00135                                             str(index))
00136             repMap = self.getRepMap()
00137             # in this parallel job, skip index*(maxEvents/nJobs) events from
00138             # the beginning
00139             # (first index is zero, so no skipping for a single job)
00140             # and use _index_ in the name of the output file
00141             repMap.update({"nIndex": str(index)})
00142             # Create the result file directly to datadir since should not use /tmp/
00143             # see https://cern.service-now.com/service-portal/article.do?n=KB0000484
00144             repMap.update({
00145                 "outputFile": replaceByMap("AlignmentValidation_"
00146                                            + self.name +
00147                                            "_.oO[name]Oo._.oO[nIndex]Oo..root",
00148                                            repMap )
00149                 })
00150             repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] )
00151 
00152             cfgs = {cfgName:replaceByMap(configTemplates.offlineParallelTemplate,
00153                                          repMap)}
00154             self.filesToCompare[GenericValidationData.defaultReferenceName] = repMap["resultFile"] 
00155             GenericValidationData.createConfiguration(self, cfgs, path)
00156             # here is a small problem. only the last cfgs is saved
00157             # it requires a bit ugly solution later
        

def offlineValidation::OfflineValidationParallel::createCrabCfg

(

self

)

Definition at line 230 of file offlineValidation.py.

                             :
        msg =  ("jobmode 'crab' not supported for 'offlineParallel' validation."
                " Please choose another 'jobmode'.")
        raise AllInOneError(msg)

def offlineValidation::OfflineValidationParallel::createScript	(		self,
			path,
			scriptBaseName = "TkAlOfflineValidation"
	)

Reimplemented from offlineValidation::OfflineValidation.

Definition at line 158 of file offlineValidation.py.

                                                                          :
        # A separate script is created for each parallel jobs.
        # Since only one cfg is saved a bit ugly solution is needed in the loop.
        returnValue = []
        numJobs = int( self.general["parallelJobs"] )
        for index in range(numJobs):
            scriptName = "%s.%s.%s_%s.sh"%(scriptBaseName, self.name, 
                                           self.alignmentToValidate.name,
                                           str(index))
            repMap = self.getRepMap()
            repMap["nIndex"]=""
            repMap["nIndex"]=str(index)
            repMap["CommandLine"]=""
            repMap.update({
                "outputFile": replaceByMap("AlignmentValidation_"
                                           + self.name +
                                           "_.oO[name]Oo._.oO[nIndex]Oo..root",
                                           repMap )
                })
            for cfg in self.configFiles:
                # The ugly solution here is to change the name for each parallel job 
                cfgtemp = cfg.replace(str(numJobs-1)+"_cfg.py",
                                      str(index)+"_cfg.py")
                repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":cfgtemp,
                                                                       "postProcess":""
                                                                       }
                scripts = {scriptName: replaceByMap(configTemplates.parallelScriptTemplate,
                                                    repMap ) }
                returnValue.extend(GenericValidationData.createScript(self,
                                                                      scripts,
                                                                      path) )
        return returnValue

def offlineValidation::OfflineValidationParallel::getRepMap	(		self,
			alignment = None
	)

Reimplemented from offlineValidation::OfflineValidation.

Definition at line 191 of file offlineValidation.py.

                                         :
        repMap = OfflineValidation.getRepMap(self, alignment) 
        repMap.update({
            "nJobs": self.general["parallelJobs"],
            "offlineValidationFileOutput":
            configTemplates.offlineParallelFileOutputTemplate,
            "nameValidation": self.name
            })
        # In case maxevents==-1, set number of parallel jobs to 1
        # since we cannot calculate number of events for each
        # parallel job
        if str(self.general["maxevents"]) == "-1":
            repMap.update({ "nJobs": "1" })
        return repMap

---

Member Data Documentation

offlineValidation::OfflineValidationParallel::__NJobs [private]

Definition at line 99 of file offlineValidation.py.