Classes
class	ValidationBase
	— Classes —############################ More...

class	ValidationJob

class	ValidationJobMultiIOV

Functions
def	createMergeScript (path, validations, options)

def	flatten (l)

def	loadTemplates (config)

def	main (argv=None)
	— Main —############################ More...

Function Documentation

◆ createMergeScript()

def validateAlignments.createMergeScript	(	path,
		validations,
		options
	)

Definition at line 485 of file validateAlignments.py.

References mps_setup.append, if(), helperFunctions.replaceByMap(), and update.

Referenced by main().

 def createMergeScript( path, validations, options ):
     if(len(validations) == 0):
         raise AllInOneError("Cowardly refusing to merge nothing!")
 
     repMap = {}
 
     comparisonLists = {} # directory of lists containing the validations that are comparable
     for validation in validations:
         if validation.config.has_section("IOV"):
             iov = validation.config.get("IOV", "iov")
             validation.defaultReferenceName = iov
         for referenceName in validation.filesToCompare:
             validationtype = type(validation)
             validationName = validation.name
             if validation.config.has_section("IOV") and (referenceName == "Tracker_defaultRange" or referenceName == "Tracker_autoRange"):
                 referenceName = iov
             if issubclass(validationtype, PreexistingValidation):
                 validationName = validation.originalValName
                 #find the actual validationtype
                 for parentclass in validationtype.mro():
                     if not issubclass(parentclass, PreexistingValidation):
                         validationtype = parentclass
                         break
             key = (validationtype, validationName, referenceName)
 
 
             if key in comparisonLists:
                 comparisonLists[key].append(validation)
             else:
                 comparisonLists[key] = [validation]
                 repMap[key] = validation.config.getGeneral()
                 repMap[key].update({
                         "DownloadData":"",
                         "CompareAlignments":"",
                         "RunValidationPlots":"",
                         "CMSSW_BASE": os.environ["CMSSW_BASE"],
                         "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
                         "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
                         })
 
                 # introduced to merge individual validation outputs separately
                 #  -> avoids problems with merge script
                 repMap[key]["doMerge"] = "mergeRetCode=0\n"
                 repMap[key]["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
                             "    echo -e \\n\"Merging succeeded, removing original files.\"\n")
                 repMap[key]["beforeMerge"] = ""
                 repMap[key]["mergeParallelFilePrefixes"] = ""
                 repMap[key]["createResultsDirectory"]=""
 
     #print("comparisonLists")
     #pprint.pprint(comparisonLists)
     anythingToMerge = []
 
     for (validationtype, validationName, referenceName), validations in comparisonLists.items():
         #pprint.pprint("validations")
         #pprint.pprint(validations)
         globalDictionaries.plottingOptions = {}
         lmap( lambda validation: validation.getRepMap(), validations )
         #plotInfo = "plots:offline"
         #allPlotInfo = dict(validations[0].config.items(plotInfo))
         #repMap[(validationtype, validationName, referenceName)].update(allPlotInfo)
 
         for validation in validations:
             validation.getRepMap()
             #pprint.pprint("validation in validations")
             #pprint.pprint(validation)
             #parallel merging
             if not (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)):
                 if (validationtype, validationName, referenceName) not in anythingToMerge:
                     anythingToMerge.append((validationtype, validationName, referenceName))
                     repMap[(validationtype, validationName, referenceName)]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType
                     repMap[(validationtype, validationName, referenceName)]["beforeMerge"] += validationtype.doInitMerge()
                 repMap[(validationtype, validationName, referenceName)]["doMerge"] += validation.doMerge()
                 for f in validation.getRepMap()["outputFiles"]:
                     longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
                                             validation.getRepMap()["eosdir"], f)
                     repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += "    rm "+longName+"\n"
 
         repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += ("else\n"
                                                                   "    echo -e \\n\"WARNING: Merging failed, unmerged"
                                                                   " files won't be deleted.\\n"
                                                                   "(Ignore this warning if merging was done earlier)\"\n"
                                                                   "fi\n")
 
 
         if anythingToMerge:
             repMap[(validationtype, validationName, referenceName)]["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap[(validationtype, validationName, referenceName)] )
         else:
             repMap[(validationtype, validationName, referenceName)]["DownloadData"] = ""
 
         repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = ""
         repMap[(validationtype, validationName, referenceName)]["plottingscriptpath"] = ""
         if issubclass(validationtype, ValidationWithPlots):
             repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = validationtype.doRunPlots(validations)
 
         repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] = "#run comparisons"
         if issubclass(validationtype, ValidationWithComparison):
             repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] += validationtype.doComparison(validations)
 
         #if not merging parallel, add code to create results directory and set merge script name accordingly
         if validations[0].config.has_section("IOV"):
             repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
             filePath = os.path.join(repMap[(validationtype, validationName, referenceName)]["scriptsdir"], "TkAlMerge.sh")
         else:
             repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
             filePath = os.path.join(path, "TkAlMerge.sh")
 
         theFile = open( filePath, "w" )
         theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap[(validationtype, validationName, referenceName)]) )
         theFile.close()
         os.chmod(filePath,0o755)
 

◆ flatten()

def validateAlignments.flatten ( l )

Definition at line 606 of file validateAlignments.py.

Referenced by python.rootplot.root2matplotlib.Hist2D.box(), python.rootplot.core.divide_axes(), and main().

 def flatten(l):
     flattenList = []
 
     for item in l:
         if type(item) == list:
             flattenList.extend(flatten(item))
 
         else:
             flattenList.append(item)
 
     return flattenList
 
 

◆ loadTemplates()

def validateAlignments.loadTemplates ( config )

Definition at line 597 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

 def loadTemplates( config ):
     if config.has_section("alternateTemplates"):
         for templateName in config.options("alternateTemplates"):
             if templateName == "AutoAlternates":
                 continue
             newTemplateName = config.get("alternateTemplates", templateName )
             #print "replacing default %s template by %s"%( templateName, newTemplateName)
             configTemplates.alternateTemplate(templateName, newTemplateName)
 

◆ main()

def validateAlignments.main ( argv = None )

— Main —############################

Definition at line 620 of file validateAlignments.py.

References createMergeScript(), edm.decode(), beamvalidation.exit(), flatten(), helperFunctions.getCommandOutput2(), createfilelist.int, loadTemplates(), GetRecoTauVFromDQM_MC_cff.next, print(), submitPVValidationJobs.split(), str, and digitizers_cfi.strip.

 def main(argv = None):
     if argv == None:
        argv = sys.argv[1:]
     optParser = optparse.OptionParser()
     optParser.description = """All-in-one Alignment Validation.
 This will run various validation procedures either on batch queues or interactively.
 If no name is given (-N parameter) a name containing time and date is created automatically.
 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
 """
     optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
                          help="create all scripts and cfg File but do not start jobs (default=False)")
     optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
                          help="get all Images created during the process (default= True)")
     defaultConfig = "TkAlConfig.ini"
     optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
                          help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
     optParser.add_option("-N", "--Name", dest="Name",
                          help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
     optParser.add_option("-r", "--restrictTo", dest="restrictTo",
                          help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
     optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
                          default = False,
                          help="run the tool to get full traceback of errors",
                          metavar="DEBUG")
 
 
     (options, args) = optParser.parse_args(argv)
 
     if not options.dryRun:
         schedinfo = subprocess.check_output(["myschedd","show"]).decode()
         if not 'tzero' in schedinfo:
             print("\nAll-In-One Tool: you need to call `module load lxbatch/tzero` before trying to submit jobs. Please do so and try again")
             exit(1)
 
 
     if not options.restrictTo == None:
         options.restrictTo = options.restrictTo.split(",")
 
     options.config = [ os.path.abspath( iniFile ) for iniFile in \
                        options.config.split( "," )]
 
     config = BetterConfigParser()
     outputIniFileSet = set( config.read( options.config ) )
     failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
 
     # Check for missing ini file
     if options.config == [ os.path.abspath( defaultConfig ) ]:
         if ( not os.path.exists( defaultConfig ) ):
                 raise AllInOneError( "Default 'ini' file '%s' not found!\n"
                                        "You can specify another name with the "
                                        "command line option '-c'/'--config'."
                                        %( defaultConfig ))
     else:
         for iniFile in failedIniFiles:
             if not os.path.exists( iniFile ):
                 raise AllInOneError( "'%s' does not exist. Please check for "
                                        "typos in the filename passed to the "
                                        "'-c'/'--config' option!"
                                        %( iniFile ))
             else:
                 raise AllInOneError(( "'%s' does exist, but parsing of the "
                                        "content failed!" ) % iniFile)
 
     # get the job name
     if options.Name == None:
         existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
                                               "alignmentValidation_*" )
         if len( existingValDirs ) > 0:
             options.Name = existingValDirs[-1]
         else:
             print("Cannot guess last working directory!")
             print ( "Please use the parameter '-N' or '--Name' to specify "
                     "the task for which you want a status report." )
             return 1
 
     # set output path
     outPath = os.path.abspath( options.Name )
 
     general = config.getGeneral()
     config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
     config.set("internals","scriptsdir",outPath)
     config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
     config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
     config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
 
     if not os.path.exists( outPath ):
         os.makedirs( outPath )
     elif not os.path.isdir( outPath ):
         raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
 
     # replace default templates by the ones specified in the "alternateTemplates" section
     loadTemplates( config )
 
     #save backup configuration file
     backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w"  )
     config.write( backupConfigFile )
 
     #copy proxy, if there is one
     try:
         proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
     except RuntimeError:
         proxyexists = False
 
     if proxyexists:
         shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
 
     validations = []
     jobs = []
     for validation in config.items("validation"):
         validation = validation[0].split("-")
         alignmentList = [validation[1]]
         validationsToAdd = [(validation[0],alignment) \
                                 for alignment in alignmentList]
         validations.extend(validationsToAdd)
 
     for validation in validations:
 
         job = ValidationJobMultiIOV(validation, config, options, outPath, len(validations))
         if (job.optionMultiIOV == True):
             jobs.extend(job)
         else:
             jobs.extend( ValidationJob(validation, config, options, 1) )
 
     for job in jobs:
         if job.needsproxy and not proxyexists:
             raise AllInOneError("At least one job needs a grid proxy, please init one.")
 
     lmap( lambda job: job.createJob(), jobs )
 
     validations = [ job.getValidation() for job in jobs ]
     validations = flatten(validations)
 
     createMergeScript(outPath, validations, options)
 
     lmap( lambda job: job.runJob(), jobs )
 
     if options.dryRun:
         pass
     else:
         ValidationJobMultiIOV.runCondorJobs(outPath)
 
 

Classes

Functions

Function Documentation

◆ createMergeScript()

◆ flatten()

◆ loadTemplates()

◆ main()