CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createMergeZmumuPlotsScript
 
def createOfflineParJobsMergeScript
 
def createPrimaryVertexPlotScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 254 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

254 
255 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
256  config = offlineValidationList[0].config
257  repMap = PlottingOptions(config, "offline")
258  repMap[ "resultPlotFile" ] = resultPlotFile
259  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
260 
261  for validation in offlineValidationList:
262  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
263 
264  theFile = open( outFilePath, "w" )
265  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
266  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 303 of file validateAlignments.py.

References bitset_utilities.append(), createExtendedValidationScript(), createMergeZmumuPlotsScript(), createOfflineParJobsMergeScript(), createPrimaryVertexPlotScript(), createTrackSplitPlotScript(), reco.if(), join(), plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by main().

303 
304 def createMergeScript( path, validations ):
305  if(len(validations) == 0):
306  raise AllInOneError("Cowardly refusing to merge nothing!")
307 
308  config = validations[0].config
309  repMap = config.getGeneral()
310  repMap.update({
311  "DownloadData":"",
312  "CompareAlignments":"",
313  "RunExtendedOfflineValidation":"",
314  "RunTrackSplitPlot":"",
315  "MergeZmumuPlots":"",
316  "RunPrimaryVertexPlot":"",
317  "CMSSW_BASE": os.environ["CMSSW_BASE"],
318  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
319  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
320  })
321 
322  comparisonLists = {} # directory of lists containing the validations that are comparable
323  for validation in validations:
324  for referenceName in validation.filesToCompare:
325  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
326  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
327  validationName = validationName.split("Preexisting")[-1]
328  if validationName in comparisonLists:
329  comparisonLists[ validationName ].append( validation )
330  else:
331  comparisonLists[ validationName ] = [ validation ]
332 
333  # introduced to merge individual validation outputs separately
334  # -> avoids problems with merge script
335  repMap["haddLoop"] = "mergeRetCode=0\n"
336  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
337  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
338  repMap["copyMergeScripts"] = ""
339  repMap["mergeParallelFilePrefixes"] = ""
340 
341  anythingToMerge = []
342 
343  for validationType in comparisonLists:
344  for validation in comparisonLists[validationType]:
345  if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
346  continue
347  if validationType not in anythingToMerge:
348  anythingToMerge += [validationType]
349  repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
350  repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
351  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
352  repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
353  "xrdcp -f "
354  +validation.getRepMap()["finalOutputFile"]
355  +" root://eoscms//eos/cms"
356  +validation.getRepMap()["finalResultFile"]
357  +"; fi\n")
358  repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
359  "mergeRetCode=${tmpMergeRetCode}; fi\n")
360  for f in validation.getRepMap()["outputFiles"]:
361  longName = os.path.join("/store/caf/user/$USER/",
362  validation.getRepMap()["eosdir"], f)
363  repMap["rmUnmerged"] += " $eos rm "+longName+"\n"
364  repMap["rmUnmerged"] += ("else\n"
365  " echo -e \\n\"WARNING: Merging failed, unmerged"
366  " files won't be deleted.\\n"
367  "(Ignore this warning if merging was done earlier)\"\n"
368  "fi\n")
369 
370  if "OfflineValidation" in anythingToMerge:
371  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
372 
373  createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
374  repMap["mergeOfflineParJobsScriptPath"] )
375  repMap["copyMergeScripts"] += ("cp .oO[Alignment/OfflineValidation]Oo./scripts/merge_TrackerOfflineValidation.C .\n"
376  "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
377  repMap_offline = repMap.copy()
378  repMap_offline.update(PlottingOptions(config, "offline"))
379  repMap["copyMergeScripts"] = \
380  replaceByMap(repMap["copyMergeScripts"], repMap_offline)
381 
382  if anythingToMerge:
383  # DownloadData is the section which merges output files from parallel jobs
384  # it uses the file TkAlOfflineJobsMerge.C
385  repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
386  else:
387  repMap["DownloadData"] = ""
388 
389  if "OfflineValidation" in comparisonLists:
390  repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
391  createExtendedValidationScript(comparisonLists["OfflineValidation"],
392  repMap["extendedValScriptPath"],
393  "OfflineValidation")
394  repMap_offline = repMap.copy()
395  repMap_offline.update(PlottingOptions(config, "offline"))
396  repMap["RunExtendedOfflineValidation"] = \
397  replaceByMap(configTemplates.extendedValidationExecution, repMap_offline)
398 
399  if "TrackSplittingValidation" in comparisonLists:
400  repMap["trackSplitPlotScriptPath"] = \
401  os.path.join(path, "TkAlTrackSplitPlot.C")
402  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
403  repMap["trackSplitPlotScriptPath"] )
404  repMap_split = repMap.copy()
405  repMap_split.update(PlottingOptions(config, "split"))
406  repMap["RunTrackSplitPlot"] = \
407  replaceByMap(configTemplates.trackSplitPlotExecution, repMap_split)
408 
409  if "ZMuMuValidation" in comparisonLists:
410  repMap["mergeZmumuPlotsScriptPath"] = \
411  os.path.join(path, "TkAlMergeZmumuPlots.C")
412  createMergeZmumuPlotsScript(comparisonLists["ZMuMuValidation"],
413  repMap["mergeZmumuPlotsScriptPath"] )
414  repMap_zMuMu = repMap.copy()
415  repMap_zMuMu.update(PlottingOptions(config, "zmumu"))
416  repMap["MergeZmumuPlots"] = \
417  replaceByMap(configTemplates.mergeZmumuPlotsExecution, repMap_zMuMu)
418 
419  if "PrimaryVertexValidation" in comparisonLists:
420  repMap["PrimaryVertexPlotScriptPath"] = \
421  os.path.join(path, "TkAlPrimaryVertexValidationPlot.C")
422 
423  createPrimaryVertexPlotScript(comparisonLists["PrimaryVertexValidation"],
424  repMap["PrimaryVertexPlotScriptPath"] )
425  repMap_PVVal = repMap.copy()
426  repMap_PVVal.update(PlottingOptions(config,"primaryvertex"))
427  repMap["RunPrimaryVertexPlot"] = \
428  replaceByMap(configTemplates.PrimaryVertexPlotExecution, repMap_PVVal)
429 
430  repMap["CompareAlignments"] = "#run comparisons"
431  if "OfflineValidation" in comparisonLists:
432  compareStrings = [ val.getCompareStrings("OfflineValidation") for val in comparisonLists["OfflineValidation"] ]
433  compareStringsPlain = [ val.getCompareStrings("OfflineValidation", plain=True) for val in comparisonLists["OfflineValidation"] ]
434 
435  repMap_offline = repMap.copy()
436  repMap_offline.update(PlottingOptions(config, "offline"))
437  repMap_offline.update({"validationId": "OfflineValidation",
438  "compareStrings": " , ".join(compareStrings),
439  "compareStringsPlain": " ".join(compareStringsPlain) })
440 
441  repMap["CompareAlignments"] += \
442  replaceByMap(configTemplates.compareAlignmentsExecution, repMap_offline)
443 
444  filePath = os.path.join(path, "TkAlMerge.sh")
445  theFile = open( filePath, "w" )
446  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
447  theFile.close()
448  os.chmod(filePath,0o755)
449 
450  return filePath
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createMergeZmumuPlotsScript (   zMuMuValidationList,
  outFilePath 
)

Definition at line 291 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

291 
292 def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath):
293  config = zMuMuValidationList[0].config
294  repMap = PlottingOptions(config, "zmumu")
295  repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
296 
297  for validation in zMuMuValidationList:
298  repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] )
299 
300  theFile = open( outFilePath, "w" )
301  theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) )
302  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createOfflineParJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 247 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

247 
248 def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
249  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
250 
251  theFile = open( outFilePath, "w" )
252  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
253  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createPrimaryVertexPlotScript (   PrimaryVertexValidationList,
  outFilePath 
)

Definition at line 279 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

279 
280 def createPrimaryVertexPlotScript(PrimaryVertexValidationList, outFilePath):
281  repMap = PrimaryVertexValidationList[0].getRepMap() # bit ugly since some special features are filled
282  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
283  repMap[ "PrimaryVertexPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
284 
285  for validation in PrimaryVertexValidationList:
286  repMap[ "PrimaryVertexPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "PrimaryVertexPlotInstantiation" ] )
287 
288  theFile = open( outFilePath, "w" )
289  theFile.write( replaceByMap( configTemplates.PrimaryVertexPlotTemplate ,repMap ) )
290  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 267 of file validateAlignments.py.

References plottingOptions.PlottingOptions(), and helperFunctions.replaceByMap().

Referenced by createMergeScript().

267 
268 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
269  config = trackSplittingValidationList[0].config
270  repMap = PlottingOptions(config, "split")
271  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
272 
273  for validation in trackSplittingValidationList:
274  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
275 
276  theFile = open( outFilePath, "w" )
277  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
278  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.loadTemplates (   config)

Definition at line 451 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

451 
452 def loadTemplates( config ):
453  if config.has_section("alternateTemplates"):
454  for templateName in config.options("alternateTemplates"):
455  if templateName == "AutoAlternates":
456  continue
457  newTemplateName = config.get("alternateTemplates", templateName )
458  #print "replacing default %s template by %s"%( templateName, newTemplateName)
459  configTemplates.alternateTemplate(templateName, newTemplateName)
460 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:289
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 462 of file validateAlignments.py.

References createMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), GetRecoTauVFromDQM_MC_cff.next, and split.

462 
463 def main(argv = None):
464  if argv == None:
465  argv = sys.argv[1:]
466  optParser = optparse.OptionParser()
467  optParser.description = """All-in-one Alignment Validation.
468 This will run various validation procedures either on batch queues or interactively.
469 If no name is given (-N parameter) a name containing time and date is created automatically.
470 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
471 """
472  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
473  help="create all scripts and cfg File but do not start jobs (default=False)")
474  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
475  help="get all Images created during the process (default= True)")
476  defaultConfig = "TkAlConfig.ini"
477  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
478  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
479  optParser.add_option("-N", "--Name", dest="Name",
480  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
481  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
482  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
483  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
484  help="get the status of the crab jobs", metavar="STATUS")
485  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
486  default = False,
487  help="run the tool to get full traceback of errors",
488  metavar="DEBUG")
489  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
490  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
491  " Works only for batch jobs")
492 
493  (options, args) = optParser.parse_args(argv)
494 
495  if not options.restrictTo == None:
496  options.restrictTo = options.restrictTo.split(",")
497 
498  options.config = [ os.path.abspath( iniFile ) for iniFile in \
499  options.config.split( "," ) ]
500  config = BetterConfigParser()
501  outputIniFileSet = set( config.read( options.config ) )
502  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
503 
504  # Check for missing ini file
505  if options.config == [ os.path.abspath( defaultConfig ) ]:
506  if ( not options.crabStatus ) and \
507  ( not os.path.exists( defaultConfig ) ):
508  raise AllInOneError( "Default 'ini' file '%s' not found!\n"
509  "You can specify another name with the "
510  "command line option '-c'/'--config'."
511  %( defaultConfig ))
512  else:
513  for iniFile in failedIniFiles:
514  if not os.path.exists( iniFile ):
515  raise AllInOneError( "'%s' does not exist. Please check for "
516  "typos in the filename passed to the "
517  "'-c'/'--config' option!"
518  %( iniFile ))
519  else:
520  raise AllInOneError(( "'%s' does exist, but parsing of the "
521  "content failed!" ) % iniFile)
522 
523  # get the job name
524  if options.Name == None:
525  if not options.crabStatus:
526  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
527  else:
528  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
529  "alignmentValidation_*" )
530  if len( existingValDirs ) > 0:
531  options.Name = existingValDirs[-1]
532  else:
533  print "Cannot guess last working directory!"
534  print ( "Please use the parameter '-N' or '--Name' to specify "
535  "the task for which you want a status report." )
536  return 1
537 
538  # set output path
539  outPath = os.path.abspath( options.Name )
540 
541  # Check status of submitted jobs and return
542  if options.crabStatus:
543  os.chdir( outPath )
544  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
545  if len( crabLogDirs ) == 0:
546  print "Found no crab tasks for job name '%s'"%( options.Name )
547  return 1
548  theCrab = crabWrapper.CrabWrapper()
549  for crabLogDir in crabLogDirs:
550  print
551  print "*" + "=" * 78 + "*"
552  print ( "| Status report and output retrieval for:"
553  + " " * (77 - len( "Status report and output retrieval for:" ) )
554  + "|" )
555  taskName = crabLogDir.replace( "crab.", "" )
556  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
557  print "*" + "=" * 78 + "*"
558  print
559  crabOptions = { "-getoutput":"",
560  "-c": crabLogDir }
561  try:
562  theCrab.run( crabOptions )
563  except AllInOneError as e:
564  print "crab: No output retrieved for this task."
565  crabOptions = { "-status": "",
566  "-c": crabLogDir }
567  theCrab.run( crabOptions )
568  return
569 
570  general = config.getGeneral()
571  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
572  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
573  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
574  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
575 
576  if not os.path.exists( outPath ):
577  os.makedirs( outPath )
578  elif not os.path.isdir( outPath ):
579  raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
580 
581  # replace default templates by the ones specified in the "alternateTemplates" section
582  loadTemplates( config )
583 
584  #save backup configuration file
585  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
586  config.write( backupConfigFile )
587 
588  validations = []
589  for validation in config.items("validation"):
590  alignmentList = [validation[1]]
591  validationsToAdd = [(validation[0],alignment) \
592  for alignment in alignmentList]
593  validations.extend(validationsToAdd)
594  jobs = [ ValidationJob( validation, config, options) \
595  for validation in validations ]
596  map( lambda job: job.createJob(), jobs )
597  validations = [ job.getValidation() for job in jobs ]
598 
599  createMergeScript(outPath, validations)
600 
601  print
602  map( lambda job: job.runJob(), jobs )
603 
604  if options.autoMerge:
605  # if everything is done as batch job, also submit TkAlMerge.sh to be run
606  # after the jobs have finished
607  if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
608  print "> Automatically merging jobs when they have ended"
609  repMap = {
610  "commands": config.getGeneral()["jobmode"].split(",")[1],
611  "jobName": "TkAlMerge",
612  "logDir": config.getGeneral()["logdir"],
613  "script": "TkAlMerge.sh",
614  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
615  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
616  }
617  for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
618  oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
619  if os.path.exists(oldlog):
620  os.remove(oldlog)
621 
622  getCommandOutput2("%(bsub)s %(commands)s "
623  "-o %(logDir)s/%(jobName)s.stdout "
624  "-e %(logDir)s/%(jobName)s.stderr "
625  "-w %(conditions)s "
626  "%(logDir)s/%(script)s"%repMap)
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:44
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:462
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5