|
|
|
Classes | |
| class | ValidationJob |
| --- Classes ---############################ More... | |
Functions | |
| def | createExtendedValidationScript |
| def | createMergeScript |
| def | createOfflineJobsMergeScript |
| def | createParallelMergeScript |
| def | loadTemplates |
| def | main |
| --- Main ---############################ | |
| def validateAlignments::createExtendedValidationScript | ( | offlineValidationList, | |
| outFilePath | |||
| ) |
Definition at line 211 of file validateAlignments.py.
00212 : 00213 repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled 00214 repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back 00215 00216 for validation in offlineValidationList: 00217 repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] ) 00218 00219 theFile = open( outFilePath, "w" ) 00220 # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) 00221 theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) ) 00222 theFile.close()
| def validateAlignments::createMergeScript | ( | path, | |
| validations | |||
| ) |
Definition at line 223 of file validateAlignments.py.
00224 : 00225 if(len(validations) == 0): 00226 msg = "Cowardly refusing to merge nothing!" 00227 raise AllInOneError(msg) 00228 00229 repMap = validations[0].getRepMap() #FIXME - not nice this way 00230 repMap.update({ 00231 "DownloadData":"", 00232 "CompareAlignments":"", 00233 "RunExtendedOfflineValidation":"" 00234 }) 00235 00236 comparisonLists = {} # directory of lists containing the validations that are comparable 00237 for validation in validations: 00238 for referenceName in validation.filesToCompare: 00239 validationName = "%s.%s"%(validation.__class__.__name__, referenceName) 00240 validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] 00241 if validationName in comparisonLists: 00242 comparisonLists[ validationName ].append( validation ) 00243 else: 00244 comparisonLists[ validationName ] = [ validation ] 00245 00246 if "OfflineValidation" in comparisonLists: 00247 repMap["extendeValScriptPath"] = \ 00248 os.path.join(path, "TkAlExtendedOfflineValidation.C") 00249 createExtendedValidationScript(comparisonLists["OfflineValidation"], 00250 repMap["extendeValScriptPath"] ) 00251 repMap["RunExtendedOfflineValidation"] = \ 00252 replaceByMap(configTemplates.extendedValidationExecution, repMap) 00253 00254 repMap["CompareAlignments"] = "#run comparisons" 00255 for validationId in comparisonLists: 00256 compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] 00257 00258 repMap.update({"validationId": validationId, 00259 "compareStrings": " , ".join(compareStrings) }) 00260 00261 repMap["CompareAlignments"] += \ 00262 replaceByMap(configTemplates.compareAlignmentsExecution, repMap) 00263 00264 filePath = os.path.join(path, "TkAlMerge.sh") 00265 theFile = open( filePath, "w" ) 00266 theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) 00267 theFile.close() 00268 os.chmod(filePath,0755) 00269 00270 return filePath
| def validateAlignments::createOfflineJobsMergeScript | ( | offlineValidationList, | |
| outFilePath | |||
| ) |
Definition at line 203 of file validateAlignments.py.
00204 : 00205 repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled 00206 repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back 00207 00208 theFile = open( outFilePath, "w" ) 00209 theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) ) 00210 theFile.close()
| def validateAlignments::createParallelMergeScript | ( | path, | |
| validations | |||
| ) |
Definition at line 271 of file validateAlignments.py.
00272 : 00273 if( len(validations) == 0 ): 00274 raise AllInOneError, "cowardly refusing to merge nothing!" 00275 00276 repMap = validations[0].getRepMap() #FIXME - not nice this way 00277 repMap.update({ 00278 "DownloadData":"", 00279 "CompareAlignments":"", 00280 "RunExtendedOfflineValidation":"" 00281 }) 00282 00283 comparisonLists = {} # directory of lists containing the validations that are comparable 00284 for validation in validations: 00285 for referenceName in validation.filesToCompare: 00286 validationName = "%s.%s"%(validation.__class__.__name__, referenceName) 00287 validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0] 00288 if validationName in comparisonLists: 00289 comparisonLists[ validationName ].append( validation ) 00290 else: 00291 comparisonLists[ validationName ] = [ validation ] 00292 00293 if "OfflineValidationParallel" in comparisonLists: 00294 repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C") 00295 createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] ) 00296 repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C") 00297 createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"], 00298 repMap["mergeOfflineParJobsScriptPath"] ) 00299 00300 # introduced to merge individual validation outputs separately 00301 # -> avoids problems with merge script 00302 repMap["haddLoop"] = "" 00303 for validation in comparisonLists["OfflineValidationParallel"]: 00304 repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"]) 00305 repMap["haddLoop"] += ("cmsStage -f " 00306 +validation.getRepMap()["outputFile"] 00307 +" " 00308 +validation.getRepMap()["resultFile"] 00309 +"\n") 00310 00311 repMap["RunExtendedOfflineValidation"] = \ 00312 replaceByMap(configTemplates.extendedValidationExecution, repMap) 00313 00314 # DownloadData is the section which merges output files from parallel jobs 00315 # it uses the file TkAlOfflineJobsMerge.C 00316 repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap) 00317 repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap ) 00318 00319 repMap["CompareAlignments"] = "#run comparisons" 00320 for validationId in comparisonLists: 00321 compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ] 00322 00323 repMap.update({"validationId": validationId, 00324 "compareStrings": " , ".join(compareStrings) }) 00325 00326 repMap["CompareAlignments"] += \ 00327 replaceByMap(configTemplates.compareAlignmentsExecution, repMap) 00328 00329 filePath = os.path.join(path, "TkAlMerge.sh") 00330 theFile = open( filePath, "w" ) 00331 theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) ) 00332 theFile.close() 00333 os.chmod(filePath,0755) 00334 00335 return filePath
| def validateAlignments::loadTemplates | ( | config | ) |
Definition at line 336 of file validateAlignments.py.
00337 : 00338 if config.has_section("alternateTemplates"): 00339 for templateName in config.options("alternateTemplates"): 00340 newTemplateName = config.get("alternateTemplates", templateName ) 00341 #print "replacing default %s template by %s"%( templateName, newTemplateName) 00342 configTemplates.alternateTemplate(templateName, newTemplateName) 00343
| def validateAlignments::main | ( | argv = None | ) |
--- Main ---############################
Definition at line 345 of file validateAlignments.py.
00346 : 00347 if argv == None: 00348 argv = sys.argv[1:] 00349 optParser = optparse.OptionParser() 00350 optParser.description = """ all-in-one alignment Validation 00351 This will run various validation procedures either on batch queues or interactviely. 00352 00353 If no name is given (-N parameter) a name containing time and date is created automatically 00354 00355 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory. 00356 """ 00357 optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False, 00358 help="create all scripts and cfg File but do not start jobs (default=False)") 00359 optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False, 00360 help="get all Images created during the process (default= False)") 00361 defaultConfig = "TkAlConfig.ini" 00362 optParser.add_option("-c", "--config", dest="config", default = defaultConfig, 00363 help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG") 00364 optParser.add_option("-N", "--Name", dest="Name", 00365 help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME") 00366 optParser.add_option("-r", "--restrictTo", dest="restrictTo", 00367 help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO") 00368 optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False, 00369 help="get the status of the crab jobs", metavar="STATUS") 00370 optParser.add_option("-d", "--debug", dest="debugMode", action="store_true", 00371 default = False, 00372 help="Run the tool to get full traceback of errors.", 00373 metavar="DEBUG") 00374 00375 (options, args) = optParser.parse_args(argv) 00376 00377 if not options.restrictTo == None: 00378 options.restrictTo = options.restrictTo.split(",") 00379 00380 options.config = [ os.path.abspath( iniFile ) for iniFile in \ 00381 options.config.split( "," ) ] 00382 config = BetterConfigParser() 00383 outputIniFileSet = set( config.read( options.config ) ) 00384 failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ] 00385 00386 # Check for missing ini file 00387 if options.config == [ os.path.abspath( defaultConfig ) ]: 00388 if ( not options.crabStatus ) and \ 00389 ( not os.path.exists( defaultConfig ) ): 00390 raise AllInOneError, ( "Default 'ini' file '%s' not found!\n" 00391 "You can specify another name with the " 00392 "command line option '-c'/'--config'." 00393 %( defaultConfig )) 00394 else: 00395 for iniFile in failedIniFiles: 00396 if not os.path.exists( iniFile ): 00397 raise AllInOneError, ( "'%s' does not exist. Please check for " 00398 "typos in the filename passed to the " 00399 "'-c'/'--config' option!" 00400 %( iniFile ) ) 00401 else: 00402 raise AllInOneError, ( "'%s' does exist, but parsing of the " 00403 "content failed!" ) 00404 00405 # get the job name 00406 if options.Name == None: 00407 if not options.crabStatus: 00408 options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S")) 00409 else: 00410 existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1], 00411 "alignmentValidation_*" ) 00412 if len( existingValDirs ) > 0: 00413 options.Name = existingValDirs[-1] 00414 else: 00415 print "Cannot guess last working directory!" 00416 print ( "Please use the parameter '-N' or '--Name' to specify " 00417 "the task for which you want a status report." ) 00418 return 1 00419 00420 # set output path 00421 outPath = os.path.abspath( options.Name ) 00422 00423 # Check status of submitted jobs and return 00424 if options.crabStatus: 00425 os.chdir( outPath ) 00426 crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" ) 00427 if len( crabLogDirs ) == 0: 00428 print "Found no crab tasks for job name '%s'"%( options.Name ) 00429 return 1 00430 theCrab = crabWrapper.CrabWrapper() 00431 for crabLogDir in crabLogDirs: 00432 print 00433 print "*" + "=" * 78 + "*" 00434 print ( "| Status report and output retrieval for:" 00435 + " " * (77 - len( "Status report and output retrieval for:" ) ) 00436 + "|" ) 00437 taskName = crabLogDir.replace( "crab.", "" ) 00438 print "| " + taskName + " " * (77 - len( taskName ) ) + "|" 00439 print "*" + "=" * 78 + "*" 00440 print 00441 crabOptions = { "-getoutput":"", 00442 "-c": crabLogDir } 00443 try: 00444 theCrab.run( crabOptions ) 00445 except AllInOneError, e: 00446 print "crab: No output retrieved for this task." 00447 crabOptions = { "-status": "", 00448 "-c": crabLogDir } 00449 theCrab.run( crabOptions ) 00450 return 00451 00452 general = config.getGeneral() 00453 config.set("internals","workdir",os.path.join(general["workdir"],options.Name) ) 00454 config.set("general","datadir",os.path.join(general["datadir"],options.Name) ) 00455 config.set("general","logdir",os.path.join(general["logdir"],options.Name) ) 00456 config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) ) 00457 00458 # clean up of log directory to avoid cluttering with files with different 00459 # random numbers for geometry comparison 00460 if os.path.isdir( outPath ): 00461 shutil.rmtree( outPath ) 00462 00463 if not os.path.exists( outPath ): 00464 os.makedirs( outPath ) 00465 elif not os.path.isdir( outPath ): 00466 raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath 00467 00468 # replace default templates by the ones specified in the "alternateTemplates" section 00469 loadTemplates( config ) 00470 00471 #save backup configuration file 00472 backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" ) 00473 config.write( backupConfigFile ) 00474 00475 validations = [] 00476 for validation in config.items("validation"): 00477 alignmentList = validation[1].split(config.getSep()) 00478 validationsToAdd = [(validation[0],alignment) \ 00479 for alignment in alignmentList] 00480 validations.extend(validationsToAdd) 00481 jobs = [ ValidationJob( validation, config, options) \ 00482 for validation in validations ] 00483 map( lambda job: job.createJob(), jobs ) 00484 validations = [ job.getValidation() for job in jobs ] 00485 00486 if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]: 00487 createMergeScript(outPath, validations) 00488 else: 00489 createParallelMergeScript( outPath, validations ) 00490 00491 print 00492 map( lambda job: job.runJob(), jobs ) 00493
1.7.3