|
|
|
Public Member Functions | |
| def | __init__ |
| def | appendToMergeParJobs |
| def | createConfiguration |
| def | createCrabCfg |
| def | createScript |
| def | getRepMap |
Private Attributes | |
| __NJobs | |
Definition at line 595 of file validateAlignments.py.
| def validateAlignments::OfflineValidationParallel::__init__ | ( | self, | |
| valName, | |||
| alignment, | |||
| config | |||
| ) |
Reimplemented from validateAlignments::OfflineValidation.
Definition at line 596 of file validateAlignments.py.
00597 : 00598 OfflineValidation.__init__(self, valName, alignment, config) 00599 defaults = { 00600 "parallelJobs":"1", 00601 "jobmode":self.jobmode 00602 } 00603 if not config.has_section( "offline:"+self.name ): 00604 offline = config.getResultingSection( "general", 00605 defaultDict = defaults ) 00606 else: 00607 offline = config.getResultingSection( "offline:"+self.name, 00608 defaultDict = defaults ) 00609 self.general.update( offline ) 00610 self.__NJobs = self.general["parallelJobs"]
| def validateAlignments::OfflineValidationParallel::appendToMergeParJobs | ( | self, | |
validationsSoFar = "" |
|||
| ) |
if no argument or "" is passed a string with an instantiation is returned, else the validation is appended to the list
Definition at line 685 of file validateAlignments.py.
00686 : 00687 """ 00688 if no argument or "" is passed a string with an instantiation is returned, 00689 else the validation is appended to the list 00690 """ 00691 repMap = self.getRepMap() 00692 00693 parameters = "" 00694 fileToAdd = "" 00695 for index in range(int(self.__NJobs)): 00696 fileToAdd = '%(resultFile)s'%repMap 00697 fileToAdd = fileToAdd.replace('.root','_'+str(index)+'.root') 00698 if index < int( self.general["parallelJobs"] )-1: 00699 parameters = parameters+fileToAdd+',' 00700 else: 00701 parameters = parameters+fileToAdd 00702 00703 mergedoutputfile = "AlignmentValidation_" + self.name + "_" + '%(name)s'%repMap + ".root" 00704 validationsSoFar += 'hadd("'+parameters+'","'+mergedoutputfile+'");' + "\n" 00705 return validationsSoFar
| def validateAlignments::OfflineValidationParallel::createConfiguration | ( | self, | |
| path, | |||
configBaseName = "TkAlOfflineValidation" |
|||
| ) |
Reimplemented from validateAlignments::OfflineValidation.
Definition at line 611 of file validateAlignments.py.
00612 : 00613 # if offline validation uses N parallel jobs, we create here N cfg files 00614 numberParallelJobs = int( self.general["parallelJobs"] ) 00615 # limit maximum number of parallel jobs to 40 00616 # (each output file is approximately 20MB) 00617 maximumNumberJobs = 40 00618 if numberParallelJobs > maximumNumberJobs: 00619 raise StandardError, "Maximum allowed number of parallel jobs "+str(maximumNumberJobs)+" exceeded!!!" 00620 # if maxevents is not specified, cannot calculate number of events for each 00621 # parallel job, and therefore running only a single job 00622 if int( self.general["maxevents"] ) == -1: 00623 raise StandardError, "Maximum number of events (maxevents) not specified: cannot use parallel jobs in offline validation" 00624 if numberParallelJobs > 1: 00625 if self.general["offlineModuleLevelHistsTransient"] == "True": 00626 raise StandardError, "To be able to merge results when running parallel jobs, set offlineModuleLevelHistsTransient to false." 00627 for index in range(numberParallelJobs): 00628 cfgName = "%s.%s.%s_%s_cfg.py"%( configBaseName, self.name, self.alignmentToValidate.name, str(index) ) 00629 repMap = self.getRepMap() 00630 # in this parallel job, skip index*(maxEvents/nJobs) events from the beginning 00631 # (first index is zero, so no skipping for a single job) 00632 # and use _index_ in the name of the output file 00633 repMap.update({"nIndex": str(index)}) 00634 # Create the result file directly to datadir since should not use /tmp/ 00635 # see https://cern.service-now.com/service-portal/article.do?n=KB0000484 00636 repMap.update({ 00637 "outputFile": replaceByMap( ".oO[datadir]Oo./AlignmentValidation_" 00638 + self.name + 00639 "_.oO[name]Oo._.oO[nIndex]Oo..root", repMap ) 00640 }) 00641 repMap["outputFile"] = os.path.expandvars( repMap["outputFile"] ) 00642 repMap["outputFile"] = os.path.abspath( repMap["outputFile"] ) 00643 00644 cfgs = {cfgName:replaceByMap( configTemplates.offlineParallelTemplate, repMap)} 00645 self.filesToCompare[ GenericValidation.defaultReferenceName ] = repMap["resultFile"] 00646 GenericValidation.createConfiguration(self, cfgs, path) 00647 # here is a small problem. only the last cfgs is saved 00648 # it requires a bit ugly solution later
| def validateAlignments::OfflineValidationParallel::createCrabCfg | ( | self | ) |
Definition at line 706 of file validateAlignments.py.
| def validateAlignments::OfflineValidationParallel::createScript | ( | self, | |
| path, | |||
scriptBaseName = "TkAlOfflineValidation" |
|||
| ) |
Reimplemented from validateAlignments::OfflineValidation.
Definition at line 649 of file validateAlignments.py.
00650 : 00651 # A separate script is created for each parallel jobs. 00652 # Since only one cfg is saved a bit ugly solution is needed in the loop. 00653 returnValue = [] 00654 numJobs = int( self.general["parallelJobs"] ) 00655 for index in range(numJobs): 00656 scriptName = "%s.%s.%s_%s.sh"%(scriptBaseName, self.name, self.alignmentToValidate.name, str(index) ) 00657 repMap = GenericValidation.getRepMap(self) 00658 repMap["nIndex"]="" 00659 repMap["nIndex"]=str(index) 00660 repMap["CommandLine"]="" 00661 for cfg in self.configFiles: 00662 # The ugly solution here is to change the name for each parallel job 00663 cfgtemp = cfg.replace( str(numJobs-1)+"_cfg.py" , str(index)+"_cfg.py" ) 00664 repMap["CommandLine"]+= repMap["CommandLineTemplate"]%{"cfgFile":cfgtemp, 00665 "postProcess":"" 00666 } 00667 scripts = {scriptName: replaceByMap( configTemplates.parallelScriptTemplate, repMap ) } 00668 returnValue.extend( GenericValidation.createScript(self, scripts, path) ) 00669 return returnValue
| def validateAlignments::OfflineValidationParallel::getRepMap | ( | self, | |
alignment = None |
|||
| ) |
Reimplemented from validateAlignments::OfflineValidation.
Definition at line 670 of file validateAlignments.py.
00671 : 00672 repMap = OfflineValidation.getRepMap(self, alignment) 00673 repMap.update({ 00674 "nJobs": self.general["parallelJobs"], 00675 "offlineValidationFileOutput": 00676 configTemplates.offlineParallelFileOutputTemplate, 00677 "nameValidation": self.name 00678 }) 00679 # In case maxevents==-1, set number of parallel jobs to 1 00680 # since we cannot calculate number of events for each 00681 # parallel job 00682 if str(self.general["maxevents"]) == "-1": 00683 repMap.update({ "nJobs": "1" }) 00684 return repMap
Definition at line 596 of file validateAlignments.py.
1.7.3