CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationBase
 — Classes —############################ More...
 
class  ValidationJob
 
class  ValidationJobMultiIOV
 

Functions

def createMergeScript (path, validations, options)
 
def flatten (l)
 
def loadTemplates (config)
 
def main (argv=None)
 — Main —############################ More...
 

Function Documentation

◆ createMergeScript()

def validateAlignments.createMergeScript (   path,
  validations,
  options 
)

Definition at line 486 of file validateAlignments.py.

486 def createMergeScript( path, validations, options ):
487  if(len(validations) == 0):
488  raise AllInOneError("Cowardly refusing to merge nothing!")
489 
490  repMap = {}
491 
492  comparisonLists = {} # directory of lists containing the validations that are comparable
493  for validation in validations:
494  if validation.config.has_section("IOV"):
495  iov = validation.config.get("IOV", "iov")
496  validation.defaultReferenceName = iov
497  for referenceName in validation.filesToCompare:
498  validationtype = type(validation)
499  validationName = validation.name
500  if validation.config.has_section("IOV") and (referenceName == "Tracker_defaultRange" or referenceName == "Tracker_autoRange"):
501  referenceName = iov
502  if issubclass(validationtype, PreexistingValidation):
503  validationName = validation.originalValName
504  #find the actual validationtype
505  for parentclass in validationtype.mro():
506  if not issubclass(parentclass, PreexistingValidation):
507  validationtype = parentclass
508  break
509  key = (validationtype, validationName, referenceName)
510 
511 
512  if key in comparisonLists:
513  comparisonLists[key].append(validation)
514  else:
515  comparisonLists[key] = [validation]
516  repMap[key] = validation.config.getGeneral()
517  repMap[key].update({
518  "DownloadData":"",
519  "CompareAlignments":"",
520  "RunValidationPlots":"",
521  "CMSSW_BASE": os.environ["CMSSW_BASE"],
522  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
523  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
524  })
525 
526  # introduced to merge individual validation outputs separately
527  # -> avoids problems with merge script
528  repMap[key]["doMerge"] = "mergeRetCode=0\n"
529  repMap[key]["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
530  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
531  repMap[key]["beforeMerge"] = ""
532  repMap[key]["mergeParallelFilePrefixes"] = ""
533  repMap[key]["createResultsDirectory"]=""
534 
535  #print("comparisonLists")
536  #pprint.pprint(comparisonLists)
537  anythingToMerge = []
538 
539  for (validationtype, validationName, referenceName), validations in six.iteritems(comparisonLists):
540  #pprint.pprint("validations")
541  #pprint.pprint(validations)
542  globalDictionaries.plottingOptions = {}
543  lmap( lambda validation: validation.getRepMap(), validations )
544  #plotInfo = "plots:offline"
545  #allPlotInfo = dict(validations[0].config.items(plotInfo))
546  #repMap[(validationtype, validationName, referenceName)].update(allPlotInfo)
547 
548  for validation in validations:
549  validation.getRepMap()
550  #pprint.pprint("validation in validations")
551  #pprint.pprint(validation)
552  #parallel merging
553  if not (isinstance(validation, PreexistingValidation) or validation.NJobs == 1 or not isinstance(validation, ParallelValidation)):
554  if (validationtype, validationName, referenceName) not in anythingToMerge:
555  anythingToMerge.append((validationtype, validationName, referenceName))
556  repMap[(validationtype, validationName, referenceName)]["doMerge"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationtype.valType
557  repMap[(validationtype, validationName, referenceName)]["beforeMerge"] += validationtype.doInitMerge()
558  repMap[(validationtype, validationName, referenceName)]["doMerge"] += validation.doMerge()
559  for f in validation.getRepMap()["outputFiles"]:
560  longName = os.path.join("/eos/cms/store/group/alca_trackeralign/AlignmentValidation/",
561  validation.getRepMap()["eosdir"], f)
562  repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += " rm "+longName+"\n"
563 
564  repMap[(validationtype, validationName, referenceName)]["rmUnmerged"] += ("else\n"
565  " echo -e \\n\"WARNING: Merging failed, unmerged"
566  " files won't be deleted.\\n"
567  "(Ignore this warning if merging was done earlier)\"\n"
568  "fi\n")
569 
570 
571  if anythingToMerge:
572  repMap[(validationtype, validationName, referenceName)]["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap[(validationtype, validationName, referenceName)] )
573  else:
574  repMap[(validationtype, validationName, referenceName)]["DownloadData"] = ""
575 
576  repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = ""
577  repMap[(validationtype, validationName, referenceName)]["plottingscriptpath"] = ""
578  if issubclass(validationtype, ValidationWithPlots):
579  repMap[(validationtype, validationName, referenceName)]["RunValidationPlots"] = validationtype.doRunPlots(validations)
580 
581  repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] = "#run comparisons"
582  if issubclass(validationtype, ValidationWithComparison):
583  repMap[(validationtype, validationName, referenceName)]["CompareAlignments"] += validationtype.doComparison(validations)
584 
585  #if not merging parallel, add code to create results directory and set merge script name accordingly
586  if validations[0].config.has_section("IOV"):
587  repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
588  filePath = os.path.join(repMap[(validationtype, validationName, referenceName)]["scriptsdir"], "TkAlMerge.sh")
589  else:
590  repMap[(validationtype, validationName, referenceName)]["createResultsDirectory"]=replaceByMap(configTemplates.createResultsDirectoryTemplate, repMap[(validationtype, validationName, referenceName)])
591  filePath = os.path.join(path, "TkAlMerge.sh")
592 
593  theFile = open( filePath, "w" )
594  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap[(validationtype, validationName, referenceName)]) )
595  theFile.close()
596  os.chmod(filePath,0o755)
597 

References mps_setup.append, helperFunctions.replaceByMap(), and update.

Referenced by main().

◆ flatten()

def validateAlignments.flatten (   l)

Definition at line 607 of file validateAlignments.py.

607 def flatten(l):
608  flattenList = []
609 
610  for item in l:
611  if type(item) == list:
612  flattenList.extend(flatten(item))
613 
614  else:
615  flattenList.append(item)
616 
617  return flattenList
618 
619 

Referenced by python.rootplot.root2matplotlib.Hist2D.box(), python.rootplot.core.divide_axes(), and main().

◆ loadTemplates()

def validateAlignments.loadTemplates (   config)

Definition at line 598 of file validateAlignments.py.

598 def loadTemplates( config ):
599  if config.has_section("alternateTemplates"):
600  for templateName in config.options("alternateTemplates"):
601  if templateName == "AutoAlternates":
602  continue
603  newTemplateName = config.get("alternateTemplates", templateName )
604  #print "replacing default %s template by %s"%( templateName, newTemplateName)
605  configTemplates.alternateTemplate(templateName, newTemplateName)
606 

References configTemplates.alternateTemplate().

Referenced by main().

◆ main()

def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 621 of file validateAlignments.py.

621 def main(argv = None):
622  if argv == None:
623  argv = sys.argv[1:]
624  optParser = optparse.OptionParser()
625  optParser.description = """All-in-one Alignment Validation.
626 This will run various validation procedures either on batch queues or interactively.
627 If no name is given (-N parameter) a name containing time and date is created automatically.
628 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
629 """
630  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
631  help="create all scripts and cfg File but do not start jobs (default=False)")
632  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
633  help="get all Images created during the process (default= True)")
634  defaultConfig = "TkAlConfig.ini"
635  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
636  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
637  optParser.add_option("-N", "--Name", dest="Name",
638  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
639  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
640  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
641  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
642  default = False,
643  help="run the tool to get full traceback of errors",
644  metavar="DEBUG")
645 
646 
647  (options, args) = optParser.parse_args(argv)
648 
649  if not options.dryRun:
650  schedinfo = subprocess.check_output(["myschedd","show"])
651  if not 'tzero' in schedinfo:
652  print("\nAll-In-One Tool: you need to call `module load lxbatch/tzero` before trying to submit jobs. Please do so and try again")
653  exit(1)
654 
655 
656  if not options.restrictTo == None:
657  options.restrictTo = options.restrictTo.split(",")
658 
659  options.config = [ os.path.abspath( iniFile ) for iniFile in \
660  options.config.split( "," )]
661 
662  config = BetterConfigParser()
663  outputIniFileSet = set( config.read( options.config ) )
664  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
665 
666  # Check for missing ini file
667  if options.config == [ os.path.abspath( defaultConfig ) ]:
668  if ( not os.path.exists( defaultConfig ) ):
669  raise AllInOneError( "Default 'ini' file '%s' not found!\n"
670  "You can specify another name with the "
671  "command line option '-c'/'--config'."
672  %( defaultConfig ))
673  else:
674  for iniFile in failedIniFiles:
675  if not os.path.exists( iniFile ):
676  raise AllInOneError( "'%s' does not exist. Please check for "
677  "typos in the filename passed to the "
678  "'-c'/'--config' option!"
679  %( iniFile ))
680  else:
681  raise AllInOneError(( "'%s' does exist, but parsing of the "
682  "content failed!" ) % iniFile)
683 
684  # get the job name
685  if options.Name == None:
686  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
687  "alignmentValidation_*" )
688  if len( existingValDirs ) > 0:
689  options.Name = existingValDirs[-1]
690  else:
691  print("Cannot guess last working directory!")
692  print ( "Please use the parameter '-N' or '--Name' to specify "
693  "the task for which you want a status report." )
694  return 1
695 
696  # set output path
697  outPath = os.path.abspath( options.Name )
698 
699  general = config.getGeneral()
700  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
701  config.set("internals","scriptsdir",outPath)
702  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
703  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
704  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
705 
706  if not os.path.exists( outPath ):
707  os.makedirs( outPath )
708  elif not os.path.isdir( outPath ):
709  raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
710 
711  # replace default templates by the ones specified in the "alternateTemplates" section
712  loadTemplates( config )
713 
714  #save backup configuration file
715  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
716  config.write( backupConfigFile )
717 
718  #copy proxy, if there is one
719  try:
720  proxyexists = int(getCommandOutput2("voms-proxy-info --timeleft")) > 10
721  except RuntimeError:
722  proxyexists = False
723 
724  if proxyexists:
725  shutil.copyfile(getCommandOutput2("voms-proxy-info --path").strip(), os.path.join(outPath, ".user_proxy"))
726 
727  validations = []
728  jobs = []
729  for validation in config.items("validation"):
730  validation = validation[0].split("-")
731  alignmentList = [validation[1]]
732  validationsToAdd = [(validation[0],alignment) \
733  for alignment in alignmentList]
734  validations.extend(validationsToAdd)
735 
736  for validation in validations:
737 
738  job = ValidationJobMultiIOV(validation, config, options, outPath, len(validations))
739  if (job.optionMultiIOV == True):
740  jobs.extend(job)
741  else:
742  jobs.extend( ValidationJob(validation, config, options, 1) )
743 
744  for job in jobs:
745  if job.needsproxy and not proxyexists:
746  raise AllInOneError("At least one job needs a grid proxy, please init one.")
747 
748  lmap( lambda job: job.createJob(), jobs )
749 
750  validations = [ job.getValidation() for job in jobs ]
751  validations = flatten(validations)
752 
753  createMergeScript(outPath, validations, options)
754 
755  lmap( lambda job: job.runJob(), jobs )
756 
757  if options.dryRun:
758  pass
759  else:
760  ValidationJobMultiIOV.runCondorJobs(outPath)
761 
762 

References createMergeScript(), beamvalidation.exit(), flatten(), helperFunctions.getCommandOutput2(), createfilelist.int, loadTemplates(), GetRecoTauVFromDQM_MC_cff.next, edm.print(), cms::dd.split(), str, and digitizers_cfi.strip.

digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
cms::dd::split
std::vector< std::string_view > split(std::string_view, const char *)
helperFunctions.replaceByMap
def replaceByMap(target, the_map)
— Helpers —############################
Definition: helperFunctions.py:13
configTemplates.alternateTemplate
def alternateTemplate(templateName, alternateTemplateName)
Definition: configTemplates.py:392
validateAlignments.main
def main(argv=None)
— Main —############################
Definition: validateAlignments.py:621
helperFunctions.getCommandOutput2
def getCommandOutput2(command)
Definition: helperFunctions.py:52
mps_setup.append
append
Definition: mps_setup.py:85
createfilelist.int
int
Definition: createfilelist.py:10
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
validateAlignments.flatten
def flatten(l)
Definition: validateAlignments.py:607
validateAlignments.createMergeScript
def createMergeScript(path, validations, options)
Definition: validateAlignments.py:486
beamvalidation.exit
def exit(msg="")
Definition: beamvalidation.py:53
validateAlignments.loadTemplates
def loadTemplates(config)
Definition: validateAlignments.py:598
update
#define update(a, b)
Definition: TrackClassifier.cc:10
GetRecoTauVFromDQM_MC_cff.next
next
Definition: GetRecoTauVFromDQM_MC_cff.py:31