validateAlignments Namespace Reference

Classes
class	ValidationJob
	— Classes —############################ More...

Functions
def	createExtendedValidationScript (offlineValidationList, outFilePath, resultPlotFile)
def	createMergeScript (path, validations)
def	createMergeZmumuPlotsScript (zMuMuValidationList, outFilePath)
def	createOfflineParJobsMergeScript (offlineValidationList, outFilePath)
def	createPrimaryVertexPlotScript (PrimaryVertexValidationList, outFilePath)
def	createTrackSplitPlotScript (trackSplittingValidationList, outFilePath)
def	loadTemplates (config)
def	main (argv=None)
	— Main —############################ More...

Function Documentation

def validateAlignments.createExtendedValidationScript	(		offlineValidationList,
			outFilePath,
			resultPlotFile
	)

Definition at line 254 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
    config = offlineValidationList[0].config
    repMap = PlottingOptions(config, "offline")
    repMap[ "resultPlotFile" ] = resultPlotFile
    repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back

    for validation in offlineValidationList:
        repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )

    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
    theFile.close()
    

def validateAlignments.createMergeScript	(		path,
			validations
	)

Definition at line 303 of file validateAlignments.py.

References mps_alisetup.append, createExtendedValidationScript(), createMergeZmumuPlotsScript(), createOfflineParJobsMergeScript(), createPrimaryVertexPlotScript(), createTrackSplitPlotScript(), reco.if(), join(), plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by main().

def createMergeScript( path, validations ):
    if(len(validations) == 0):
        raise AllInOneError("Cowardly refusing to merge nothing!")

    config = validations[0].config
    repMap = config.getGeneral()
    repMap.update({
            "DownloadData":"",
            "CompareAlignments":"",
            "RunExtendedOfflineValidation":"",
            "RunTrackSplitPlot":"",
            "MergeZmumuPlots":"",
            "RunPrimaryVertexPlot":"",
            "CMSSW_BASE": os.environ["CMSSW_BASE"],
            "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
            "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
            })

    comparisonLists = {} # directory of lists containing the validations that are comparable
    for validation in validations:
        for referenceName in validation.filesToCompare:
            validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
            validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
            validationName = validationName.split("Preexisting")[-1]
            if validationName in comparisonLists:
                comparisonLists[ validationName ].append( validation )
            else:
                comparisonLists[ validationName ] = [ validation ]
   
    # introduced to merge individual validation outputs separately
    #  -> avoids problems with merge script
    repMap["haddLoop"] = "mergeRetCode=0\n"
    repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
                            "    echo -e \\n\"Merging succeeded, removing original files.\"\n")
    repMap["copyMergeScripts"] = ""
    repMap["mergeParallelFilePrefixes"] = ""

    anythingToMerge = []
    
    for validationType in comparisonLists:
        for validation in comparisonLists[validationType]:
            if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
                continue
            if validationType not in anythingToMerge:
                anythingToMerge += [validationType]
                repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
            repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
            repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
            repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
                                   "xrdcp -f "
                                   +validation.getRepMap()["finalOutputFile"]
                                   +" root://eoscms//eos/cms"
                                   +validation.getRepMap()["finalResultFile"]
                                   +"; fi\n")
            repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
                                   "mergeRetCode=${tmpMergeRetCode}; fi\n")
            for f in validation.getRepMap()["outputFiles"]:
                longName = os.path.join("/store/caf/user/$USER/",
                                        validation.getRepMap()["eosdir"], f)
                repMap["rmUnmerged"] += "    $eos rm "+longName+"\n"
    repMap["rmUnmerged"] += ("else\n"
                             "    echo -e \\n\"WARNING: Merging failed, unmerged"
                             " files won't be deleted.\\n"
                             "(Ignore this warning if merging was done earlier)\"\n"
                             "fi\n")

    if "OfflineValidation" in anythingToMerge:
        repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
        
        createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
                                         repMap["mergeOfflineParJobsScriptPath"] )
        repMap["copyMergeScripts"] += ("cp .oO[Alignment/OfflineValidation]Oo./scripts/merge_TrackerOfflineValidation.C .\n"
                                       "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
        repMap_offline = repMap.copy()
        repMap_offline.update(PlottingOptions(config, "offline"))
        repMap["copyMergeScripts"] = \
            replaceByMap(repMap["copyMergeScripts"], repMap_offline)

    if anythingToMerge:
        # DownloadData is the section which merges output files from parallel jobs
        # it uses the file TkAlOfflineJobsMerge.C
        repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
    else:
        repMap["DownloadData"] = ""

    if "OfflineValidation" in comparisonLists:
        repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
        createExtendedValidationScript(comparisonLists["OfflineValidation"],
                                       repMap["extendedValScriptPath"],
                                       "OfflineValidation")
        repMap_offline = repMap.copy()
        repMap_offline.update(PlottingOptions(config, "offline"))
        repMap["RunExtendedOfflineValidation"] = \
            replaceByMap(configTemplates.extendedValidationExecution, repMap_offline)

    if "TrackSplittingValidation" in comparisonLists:
        repMap["trackSplitPlotScriptPath"] = \
            os.path.join(path, "TkAlTrackSplitPlot.C")
        createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
                                       repMap["trackSplitPlotScriptPath"] )
        repMap_split = repMap.copy()
        repMap_split.update(PlottingOptions(config, "split"))
        repMap["RunTrackSplitPlot"] = \
            replaceByMap(configTemplates.trackSplitPlotExecution, repMap_split)

    if "ZMuMuValidation" in comparisonLists:
        repMap["mergeZmumuPlotsScriptPath"] = \
            os.path.join(path, "TkAlMergeZmumuPlots.C")
        createMergeZmumuPlotsScript(comparisonLists["ZMuMuValidation"],
                                       repMap["mergeZmumuPlotsScriptPath"] )
        repMap_zMuMu = repMap.copy()
        repMap_zMuMu.update(PlottingOptions(config, "zmumu"))
        repMap["MergeZmumuPlots"] = \
            replaceByMap(configTemplates.mergeZmumuPlotsExecution, repMap_zMuMu)

    if "PrimaryVertexValidation" in comparisonLists:
        repMap["PrimaryVertexPlotScriptPath"] = \
            os.path.join(path, "TkAlPrimaryVertexValidationPlot.C")

        createPrimaryVertexPlotScript(comparisonLists["PrimaryVertexValidation"],
                                      repMap["PrimaryVertexPlotScriptPath"] )
        repMap_PVVal = repMap.copy()
        repMap_PVVal.update(PlottingOptions(config,"primaryvertex"))
        repMap["RunPrimaryVertexPlot"] = \
            replaceByMap(configTemplates.PrimaryVertexPlotExecution, repMap_PVVal)

    repMap["CompareAlignments"] = "#run comparisons"
    if "OfflineValidation" in comparisonLists:
        compareStrings = [ val.getCompareStrings("OfflineValidation") for val in comparisonLists["OfflineValidation"] ]
        compareStringsPlain = [ val.getCompareStrings("OfflineValidation", plain=True) for val in comparisonLists["OfflineValidation"] ]
            
        repMap_offline = repMap.copy()
        repMap_offline.update(PlottingOptions(config, "offline"))
        repMap_offline.update({"validationId": "OfflineValidation",
                               "compareStrings": " , ".join(compareStrings),
                               "compareStringsPlain": " ".join(compareStringsPlain) })
                               
        repMap["CompareAlignments"] += \
            replaceByMap(configTemplates.compareAlignmentsExecution, repMap_offline)
                
    filePath = os.path.join(path, "TkAlMerge.sh")
    theFile = open( filePath, "w" )
    theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
    theFile.close()
    os.chmod(filePath,0o755)
    
    return filePath
    

def validateAlignments.createMergeZmumuPlotsScript	(		zMuMuValidationList,
			outFilePath
	)

Definition at line 291 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath):
    config = zMuMuValidationList[0].config
    repMap = PlottingOptions(config, "zmumu")
    repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back

    for validation in zMuMuValidationList:
        repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] )

    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) )
    theFile.close()

def validateAlignments.createOfflineParJobsMergeScript	(		offlineValidationList,
			outFilePath
	)

Definition at line 247 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
    repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
    
    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
    theFile.close()

def validateAlignments.createPrimaryVertexPlotScript	(		PrimaryVertexValidationList,
			outFilePath
	)

Definition at line 279 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

def createPrimaryVertexPlotScript(PrimaryVertexValidationList, outFilePath):
    repMap = PrimaryVertexValidationList[0].getRepMap() # bit ugly since some special features are filled
    repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
    repMap[ "PrimaryVertexPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back

    for validation in PrimaryVertexValidationList:
        repMap[ "PrimaryVertexPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "PrimaryVertexPlotInstantiation" ] )

    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.PrimaryVertexPlotTemplate ,repMap ) )
    theFile.close()

def validateAlignments.createTrackSplitPlotScript	(		trackSplittingValidationList,
			outFilePath
	)

Definition at line 267 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
    config = trackSplittingValidationList[0].config
    repMap = PlottingOptions(config, "split")
    repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back

    for validation in trackSplittingValidationList:
        repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
    
    theFile = open( outFilePath, "w" )
    theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
    theFile.close()

def validateAlignments.loadTemplates

(

config

)

Definition at line 451 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

def loadTemplates( config ):
    if config.has_section("alternateTemplates"):
        for templateName in config.options("alternateTemplates"):
            if templateName == "AutoAlternates":
                continue
            newTemplateName = config.get("alternateTemplates", templateName )
            #print "replacing default %s template by %s"%( templateName, newTemplateName)
            configTemplates.alternateTemplate(templateName, newTemplateName)

    

def validateAlignments.main

(

argv = None

)

— Main —############################

Definition at line 462 of file validateAlignments.py.

References createMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), genParticles_cff.map, GetRecoTauVFromDQM_MC_cff.next, split, and harvestTrackValidationPlots.str.

def main(argv = None):
    if argv == None:
       argv = sys.argv[1:]
    optParser = optparse.OptionParser()
    optParser.description = """All-in-one Alignment Validation.
This will run various validation procedures either on batch queues or interactively.
If no name is given (-N parameter) a name containing time and date is created automatically.
To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
"""
    optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
                         help="create all scripts and cfg File but do not start jobs (default=False)")
    optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
                          help="get all Images created during the process (default= True)")
    defaultConfig = "TkAlConfig.ini"
    optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
                         help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
    optParser.add_option("-N", "--Name", dest="Name",
                         help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
    optParser.add_option("-r", "--restrictTo", dest="restrictTo",
                         help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
    optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
                         help="get the status of the crab jobs", metavar="STATUS")
    optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
                         default = False,
                         help="run the tool to get full traceback of errors",
                         metavar="DEBUG")
    optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
                         help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
                              " Works only for batch jobs")

    (options, args) = optParser.parse_args(argv)

    if not options.restrictTo == None:
        options.restrictTo = options.restrictTo.split(",")
    
    options.config = [ os.path.abspath( iniFile ) for iniFile in \
                       options.config.split( "," ) ]
    config = BetterConfigParser()
    outputIniFileSet = set( config.read( options.config ) )
    failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]

    # Check for missing ini file
    if options.config == [ os.path.abspath( defaultConfig ) ]:
        if ( not options.crabStatus ) and \
               ( not os.path.exists( defaultConfig ) ):
                raise AllInOneError( "Default 'ini' file '%s' not found!\n"
                                       "You can specify another name with the "
                                       "command line option '-c'/'--config'."
                                       %( defaultConfig ))
    else:
        for iniFile in failedIniFiles:
            if not os.path.exists( iniFile ):
                raise AllInOneError( "'%s' does not exist. Please check for "
                                       "typos in the filename passed to the "
                                       "'-c'/'--config' option!"
                                       %( iniFile ))
            else:
                raise AllInOneError(( "'%s' does exist, but parsing of the "
                                       "content failed!" ) % iniFile)

    # get the job name
    if options.Name == None:
        if not options.crabStatus:
            options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
        else:
            existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
                                              "alignmentValidation_*" )
            if len( existingValDirs ) > 0:
                options.Name = existingValDirs[-1]
            else:
                print "Cannot guess last working directory!"
                print ( "Please use the parameter '-N' or '--Name' to specify "
                        "the task for which you want a status report." )
                return 1

    # set output path
    outPath = os.path.abspath( options.Name )

    # Check status of submitted jobs and return
    if options.crabStatus:
        os.chdir( outPath )
        crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
        if len( crabLogDirs ) == 0:
            print "Found no crab tasks for job name '%s'"%( options.Name )
            return 1
        theCrab = crabWrapper.CrabWrapper()
        for crabLogDir in crabLogDirs:
            print
            print "*" + "=" * 78 + "*"
            print ( "| Status report and output retrieval for:"
                    + " " * (77 - len( "Status report and output retrieval for:" ) )
                    + "|" )
            taskName = crabLogDir.replace( "crab.", "" )
            print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
            print "*" + "=" * 78 + "*"
            print
            crabOptions = { "-getoutput":"",
                            "-c": crabLogDir }
            try:
                theCrab.run( crabOptions )
            except AllInOneError as e:
                print "crab:  No output retrieved for this task."
            crabOptions = { "-status": "",
                            "-c": crabLogDir }
            theCrab.run( crabOptions )
        return

    general = config.getGeneral()
    config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
    config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
    config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
    config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )

    if not os.path.exists( outPath ):
        os.makedirs( outPath )
    elif not os.path.isdir( outPath ):
        raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)

    # replace default templates by the ones specified in the "alternateTemplates" section
    loadTemplates( config )

    #save backup configuration file
    backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w"  )
    config.write( backupConfigFile )

    validations = []
    for validation in config.items("validation"):
        alignmentList = [validation[1]]
        validationsToAdd = [(validation[0],alignment) \
                                for alignment in alignmentList]
        validations.extend(validationsToAdd)
    jobs = [ ValidationJob( validation, config, options) \
                 for validation in validations ]
    map( lambda job: job.createJob(), jobs )
    validations = [ job.getValidation() for job in jobs ]

    createMergeScript(outPath, validations)

    print
    map( lambda job: job.runJob(), jobs )

    if options.autoMerge:
        # if everything is done as batch job, also submit TkAlMerge.sh to be run
        # after the jobs have finished
        if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
            print ">             Automatically merging jobs when they have ended"
            repMap = {
                "commands": config.getGeneral()["jobmode"].split(",")[1],
                "jobName": "TkAlMerge",
                "logDir": config.getGeneral()["logdir"],
                "script": "TkAlMerge.sh",
                "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
                "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
                }
            for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
                oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
                if os.path.exists(oldlog):
                    os.remove(oldlog)

            getCommandOutput2("%(bsub)s %(commands)s "
                              "-o %(logDir)s/%(jobName)s.stdout "
                              "-e %(logDir)s/%(jobName)s.stderr "
                              "-w %(conditions)s "
                              "%(logDir)s/%(script)s"%repMap)