10 import Alignment.OfflineValidation.TkAlAllInOneTool.configTemplates \
12 import Alignment.OfflineValidation.TkAlAllInOneTool.crabWrapper
as crabWrapper
13 from Alignment.OfflineValidation.TkAlAllInOneTool.TkAlExceptions \
15 from Alignment.OfflineValidation.TkAlAllInOneTool.helperFunctions \
16 import replaceByMap, getCommandOutput2
17 from Alignment.OfflineValidation.TkAlAllInOneTool.betterConfigParser \
18 import BetterConfigParser
19 from Alignment.OfflineValidation.TkAlAllInOneTool.alignment
import Alignment
21 from Alignment.OfflineValidation.TkAlAllInOneTool.genericValidation \
22 import GenericValidation
23 from Alignment.OfflineValidation.TkAlAllInOneTool.geometryComparison \
24 import GeometryComparison
25 from Alignment.OfflineValidation.TkAlAllInOneTool.offlineValidation \
26 import OfflineValidation, OfflineValidationDQM, OfflineValidationParallel
27 from Alignment.OfflineValidation.TkAlAllInOneTool.monteCarloValidation \
28 import MonteCarloValidation
29 from Alignment.OfflineValidation.TkAlAllInOneTool.trackSplittingValidation \
30 import TrackSplittingValidation
31 from Alignment.OfflineValidation.TkAlAllInOneTool.zMuMuValidation \
32 import ZMuMuValidation
33 import Alignment.OfflineValidation.TkAlAllInOneTool.globalDictionaries \
39 def __init__( self, validation, config, options ):
40 if validation[1] ==
"":
42 valString = validation[0].
split(
"->" )[0]
43 alignments = validation[0].
split(
"->" )[1]
45 if "->" in validation[0]:
46 msg = (
"Instead of using the intermediate syntax\n'"
47 +valString.strip()+
"-> "+alignments.strip()
48 +
":'\nyou have to use the now fully supported syntax \n'"
49 +valString.strip()+
": "
50 +alignments.strip()+
"'.")
51 raise AllInOneError(msg)
53 valString = validation[0]
54 alignments = validation[1]
55 valString = valString.split()
62 section =
"offline" +
":" + self.
__valName
65 if not self.__config.has_section( section ):
66 raise AllInOneError, (
"Validation '%s' of type '%s' is requested in"
67 " '[validation]' section, but is not defined."
68 "\nYou have to add a '[%s]' section."
75 if valType ==
"compare":
76 alignmentsList = alignments.split(
"," )
77 firstAlignList = alignmentsList[0].
split()
78 firstAlignName = firstAlignList[0].strip()
79 if firstAlignName ==
"IDEAL":
80 raise AllInOneError, (
"'IDEAL' has to be the second (reference)"
81 " alignment in 'compare <val_name>: "
82 "<alignment> <reference>'.")
83 if len( firstAlignList ) > 1:
84 firstRun = firstAlignList[1]
87 firstAlign = Alignment( firstAlignName, self.
__config, firstRun )
88 secondAlignList = alignmentsList[1].
split()
89 secondAlignName = secondAlignList[0].strip()
90 if len( secondAlignList ) > 1:
91 secondRun = secondAlignList[1]
94 if secondAlignName ==
"IDEAL":
95 secondAlign = secondAlignName
97 secondAlign = Alignment( secondAlignName, self.
__config,
101 randomWorkdirPart = \
102 globalDictionaries.alignRandDict[firstAlignName]
104 randomWorkdirPart =
None
106 validation = GeometryComparison( name, firstAlign, secondAlign,
108 self.__commandLineOptions.getImages,
110 globalDictionaries.alignRandDict[firstAlignName] = \
111 validation.randomWorkdirPart
112 if not secondAlignName ==
"IDEAL":
113 globalDictionaries.alignRandDict[secondAlignName] = \
114 validation.randomWorkdirPart
115 elif valType ==
"offline":
116 validation = OfflineValidation( name,
118 elif valType ==
"offlineDQM":
119 validation = OfflineValidationDQM( name,
121 elif valType ==
"offlineParallel":
122 validation = OfflineValidationParallel( name,
124 elif valType ==
"mcValidate":
125 validation = MonteCarloValidation( name,
127 elif valType ==
"split":
128 validation = TrackSplittingValidation( name,
130 elif valType ==
"zmumu":
131 validation = ZMuMuValidation( name,
134 raise AllInOneError,
"Unknown validation mode '%s'"%valType
138 """This private method creates the needed files for the validation job.
140 self.validation.createConfiguration( outpath )
141 self.
__scripts = self.validation.createScript( outpath )
142 if jobMode.split(
',' )[0] ==
"crab":
143 self.validation.createCrabCfg( outpath )
147 """This is the method called to create the job files."""
149 os.path.abspath( self.__commandLineOptions.Name) )
152 general = self.__config.getGeneral()
155 name = os.path.splitext( os.path.basename( script) )[0]
156 if self.__commandLineOptions.dryRun:
157 print "%s would run: %s"%( name, os.path.basename( script) )
159 log =
"> Validating "+name
160 print "> Validating "+name
161 if self.validation.jobmode ==
"interactive":
163 elif self.validation.jobmode.split(
",")[0] ==
"lxBatch":
165 "commands": self.validation.jobmode.split(
",")[1],
166 "logDir": general[
"logdir"],
169 "bsub":
"/afs/cern.ch/cms/caf/scripts/cmsbsub"
172 "-o %(logDir)s/%(jobName)s.stdout -e "
173 "%(logDir)s/%(jobName)s.stderr "
175 elif self.validation.jobmode.split(
"," )[0] ==
"crab":
176 os.chdir( general[
"logdir"] )
177 crabName =
"crab." + os.path.basename( script )[:-3]
179 options = {
"-create":
"",
180 "-cfg": crabName +
".cfg",
183 theCrab.run( options )
184 except AllInOneError, e:
185 print "crab:", str(e).
split(
"\n")[0]
188 raise AllInOneError, (
"Unknown 'jobmode'!\n"
189 "Please change this parameter either in "
190 "the [general] or in the ["
192 +
"] section to one of the following "
194 "\tinteractive\n\tlxBatch, -q <queue>\n"
195 "\tcrab, -q <queue>")
204 repMap = offlineValidationList[0].getRepMap()
205 repMap[
"mergeOfflinParJobsInstantiation" ] =
""
207 theFile = open( outFilePath,
"w" )
208 theFile.write(
replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
212 repMap = offlineValidationList[0].getRepMap()
213 repMap[
"resultPlotFile" ] = resultPlotFile
214 repMap[
"extendedInstantiation" ] =
""
216 for validation
in offlineValidationList:
217 repMap[
"extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[
"extendedInstantiation" ] )
219 theFile = open( outFilePath,
"w" )
221 theFile.write(
replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
225 if(len(validations) == 0):
226 msg =
"Cowardly refusing to merge nothing!"
227 raise AllInOneError(msg)
229 repMap = validations[0].getRepMap()
232 "CompareAlignments":
"",
233 "RunExtendedOfflineValidation":
""
238 for validation
in validations:
239 for referenceName
in validation.filesToCompare:
240 validationName =
"%s.%s"%(validation.__class__.__name__, referenceName)
241 validationName = validationName.split(
".%s"%GenericValidation.defaultReferenceName )[0]
242 if validationName
in comparisonLists:
243 comparisonLists[ validationName ].
append( validation )
245 comparisonLists[ validationName ] = [ validation ]
246 if validationName ==
"OfflineValidation":
247 resultPlotFile = validationName
249 if "OfflineValidation" in comparisonLists:
250 repMap[
"extendeValScriptPath"] = \
251 os.path.join(path,
"TkAlExtendedOfflineValidation.C")
253 repMap[
"extendeValScriptPath"],
255 repMap[
"RunExtendedOfflineValidation"] = \
256 replaceByMap(configTemplates.extendedValidationExecution, repMap)
258 repMap[
"CompareAlignments"] =
"#run comparisons"
259 for validationId
in comparisonLists:
260 compareStrings = [ val.getCompareStrings(validationId)
for val
in comparisonLists[validationId] ]
262 repMap.update({
"validationId": validationId,
263 "compareStrings":
" , ".
join(compareStrings) })
265 repMap[
"CompareAlignments"] += \
266 replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
268 filePath = os.path.join(path,
"TkAlMerge.sh")
269 theFile = open( filePath,
"w" )
270 theFile.write(
replaceByMap( configTemplates.mergeTemplate, repMap ) )
272 os.chmod(filePath,0755)
277 if( len(validations) == 0 ):
278 raise AllInOneError,
"cowardly refusing to merge nothing!"
280 repMap = validations[0].getRepMap()
283 "CompareAlignments":
"",
284 "RunExtendedOfflineValidation":
""
289 for validation
in validations:
290 for referenceName
in validation.filesToCompare:
291 validationName =
"%s.%s"%(validation.__class__.__name__, referenceName)
292 validationName = validationName.split(
".%s"%GenericValidation.defaultReferenceName )[0]
293 if validationName
in comparisonLists:
294 comparisonLists[ validationName ].
append( validation )
296 comparisonLists[ validationName ] = [ validation ]
297 if validationName ==
"OfflineValidationParallel":
298 resultPlotFile = validationName
300 if "OfflineValidationParallel" in comparisonLists:
301 repMap[
"extendeValScriptPath"] = os.path.join(path,
"TkAlExtendedOfflineValidation.C")
303 repMap[
"mergeOfflineParJobsScriptPath"] = os.path.join(path,
"TkAlOfflineJobsMerge.C")
305 repMap[
"mergeOfflineParJobsScriptPath"] )
309 repMap[
"haddLoop"] =
"mergeRetCode=0\n"
310 repMap[
"rmUnmerged"] =
"if [[ mergeRetCode -eq 0 ]]; then\n"
311 for validation
in comparisonLists[
"OfflineValidationParallel"]:
312 repMap[
"haddLoop"] = validation.appendToMergeParJobs(repMap[
"haddLoop"])
313 repMap[
"haddLoop"] +=
"tmpMergeRetCode=${?}\n"
314 repMap[
"haddLoop"] += (
"if [[ mergeRetCode -eq 0 ]]; "
315 "then mergeRetCode=${tmpMergeRetCode}; "
317 repMap[
"haddLoop"] += (
"cmsStage -f "
318 +validation.getRepMap()[
"outputFile"]
320 +validation.getRepMap()[
"resultFile"]
322 for f
in validation.outputFiles:
323 longName = os.path.join(
"/store/caf/user/$USER/",
324 validation.getRepMap()[
"eosdir"], f)
325 repMap[
"rmUnmerged"] +=
" cmsRm "+longName+
"\n"
326 repMap[
"rmUnmerged"] += (
"else\n"
327 " echo \"WARNING: Merging failed, unmerged"
328 " files won't be deleted.\"\n"
331 repMap[
"RunExtendedOfflineValidation"] = \
332 replaceByMap(configTemplates.extendedValidationExecution, repMap)
336 repMap[
"DownloadData"] +=
replaceByMap(
"rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
337 repMap[
"DownloadData"] +=
replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
339 repMap[
"CompareAlignments"] =
"#run comparisons"
340 for validationId
in comparisonLists:
341 compareStrings = [ val.getCompareStrings(validationId)
for val
in comparisonLists[validationId] ]
343 repMap.update({
"validationId": validationId,
344 "compareStrings":
" , ".
join(compareStrings) })
346 repMap[
"CompareAlignments"] += \
347 replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
349 filePath = os.path.join(path,
"TkAlMerge.sh")
350 theFile = open( filePath,
"w" )
351 theFile.write(
replaceByMap( configTemplates.mergeTemplate, repMap ) )
353 os.chmod(filePath,0755)
358 if config.has_section(
"alternateTemplates"):
359 for templateName
in config.options(
"alternateTemplates"):
360 newTemplateName = config.get(
"alternateTemplates", templateName )
369 optParser = optparse.OptionParser()
370 optParser.description =
""" all-in-one alignment Validation
371 This will run various validation procedures either on batch queues or interactviely.
373 If no name is given (-N parameter) a name containing time and date is created automatically
375 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
377 optParser.add_option(
"-n",
"--dryRun", dest=
"dryRun", action=
"store_true", default=
False,
378 help=
"create all scripts and cfg File but do not start jobs (default=False)")
379 optParser.add_option(
"--getImages", dest=
"getImages", action=
"store_true", default=
False,
380 help=
"get all Images created during the process (default= False)")
381 defaultConfig =
"TkAlConfig.ini"
382 optParser.add_option(
"-c",
"--config", dest=
"config", default = defaultConfig,
383 help=
"configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar=
"CONFIG")
384 optParser.add_option(
"-N",
"--Name", dest=
"Name",
385 help=
"Name of this validation (default: alignmentValidation_DATE_TIME)", metavar=
"NAME")
386 optParser.add_option(
"-r",
"--restrictTo", dest=
"restrictTo",
387 help=
"restrict validations to given modes (comma seperated) (default: no restriction)", metavar=
"RESTRICTTO")
388 optParser.add_option(
"-s",
"--status", dest=
"crabStatus", action=
"store_true", default =
False,
389 help=
"get the status of the crab jobs", metavar=
"STATUS")
390 optParser.add_option(
"-d",
"--debug", dest=
"debugMode", action=
"store_true",
392 help=
"Run the tool to get full traceback of errors.",
395 (options, args) = optParser.parse_args(argv)
397 if not options.restrictTo ==
None:
398 options.restrictTo = options.restrictTo.split(
",")
400 options.config = [ os.path.abspath( iniFile )
for iniFile
in \
401 options.config.split(
"," ) ]
402 config = BetterConfigParser()
403 outputIniFileSet = set( config.read( options.config ) )
404 failedIniFiles = [ iniFile
for iniFile
in options.config
if iniFile
not in outputIniFileSet ]
407 if options.config == [ os.path.abspath( defaultConfig ) ]:
408 if (
not options.crabStatus )
and \
409 (
not os.path.exists( defaultConfig ) ):
410 raise AllInOneError, (
"Default 'ini' file '%s' not found!\n"
411 "You can specify another name with the "
412 "command line option '-c'/'--config'."
415 for iniFile
in failedIniFiles:
416 if not os.path.exists( iniFile ):
417 raise AllInOneError, (
"'%s' does not exist. Please check for "
418 "typos in the filename passed to the "
419 "'-c'/'--config' option!"
422 raise AllInOneError, (
"'%s' does exist, but parsing of the "
426 if options.Name ==
None:
427 if not options.crabStatus:
428 options.Name =
"alignmentValidation_%s"%(datetime.datetime.now().strftime(
"%y%m%d_%H%M%S"))
430 existingValDirs = fnmatch.filter( os.walk(
'.' ).
next()[1],
431 "alignmentValidation_*" )
432 if len( existingValDirs ) > 0:
433 options.Name = existingValDirs[-1]
435 print "Cannot guess last working directory!"
436 print (
"Please use the parameter '-N' or '--Name' to specify "
437 "the task for which you want a status report." )
441 outPath = os.path.abspath( options.Name )
444 if options.crabStatus:
446 crabLogDirs = fnmatch.filter( os.walk(
'.').
next()[1],
"crab.*" )
447 if len( crabLogDirs ) == 0:
448 print "Found no crab tasks for job name '%s'"%( options.Name )
451 for crabLogDir
in crabLogDirs:
453 print "*" +
"=" * 78 +
"*"
454 print (
"| Status report and output retrieval for:"
455 +
" " * (77 - len(
"Status report and output retrieval for:" ) )
457 taskName = crabLogDir.replace(
"crab.",
"" )
458 print "| " + taskName +
" " * (77 - len( taskName ) ) +
"|"
459 print "*" +
"=" * 78 +
"*"
461 crabOptions = {
"-getoutput":
"",
464 theCrab.run( crabOptions )
465 except AllInOneError, e:
466 print "crab: No output retrieved for this task."
467 crabOptions = {
"-status":
"",
469 theCrab.run( crabOptions )
472 general = config.getGeneral()
473 config.set(
"internals",
"workdir",os.path.join(general[
"workdir"],options.Name) )
474 config.set(
"general",
"datadir",os.path.join(general[
"datadir"],options.Name) )
475 config.set(
"general",
"logdir",os.path.join(general[
"logdir"],options.Name) )
476 config.set(
"general",
"eosdir",os.path.join(
"AlignmentValidation", general[
"eosdir"], options.Name) )
480 if os.path.isdir( outPath ):
481 shutil.rmtree( outPath )
483 if not os.path.exists( outPath ):
484 os.makedirs( outPath )
485 elif not os.path.isdir( outPath ):
486 raise AllInOneError,
"the file %s is in the way rename the Job or move it away"%outPath
492 backupConfigFile = open( os.path.join( outPath,
"usedConfiguration.ini" ) ,
"w" )
493 config.write( backupConfigFile )
496 for validation
in config.items(
"validation"):
497 alignmentList = validation[1].
split(config.getSep())
498 validationsToAdd = [(validation[0],alignment) \
499 for alignment
in alignmentList]
500 validations.extend(validationsToAdd)
502 for validation
in validations ]
503 map(
lambda job: job.createJob(), jobs )
504 validations = [ job.getValidation()
for job
in jobs ]
506 if "OfflineValidationParallel" not in [val.__class__.__name__
for val
in validations]:
512 map(
lambda job: job.runJob(), jobs )
515 if __name__ ==
"__main__":
517 if "-d" in sys.argv[1:]
or "--debug" in sys.argv[1:]:
522 except AllInOneError, e:
523 print "\nAll-In-One Tool:", str(e)
def createOfflineJobsMergeScript
def createParallelMergeScript
— Classes —############################
def main
— Main —############################
def createExtendedValidationScript
def alternateTemplate
### Alternate Templates ###
static std::string join(char **cmd)
def replaceByMap
— Helpers —############################