CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createMergeZmumuPlotsScript
 
def createOfflineParJobsMergeScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 244 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

244 
245 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
246  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
247  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
248  repMap[ "resultPlotFile" ] = resultPlotFile
249  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
250 
251  for validation in offlineValidationList:
252  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
253 
254  theFile = open( outFilePath, "w" )
255  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
256  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 281 of file validateAlignments.py.

References bitset_utilities.append(), createExtendedValidationScript(), createMergeZmumuPlotsScript(), createOfflineParJobsMergeScript(), createTrackSplitPlotScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

281 
282 def createMergeScript( path, validations ):
283  if(len(validations) == 0):
284  raise AllInOneError("Cowardly refusing to merge nothing!")
285 
286  repMap = validations[0].getRepMap() #FIXME - not nice this way
287  repMap.update({
288  "DownloadData":"",
289  "CompareAlignments":"",
290  "RunExtendedOfflineValidation":"",
291  "RunTrackSplitPlot":"",
292  "MergeZmumuPlots":"",
293  "CMSSW_BASE": os.environ["CMSSW_BASE"],
294  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
295  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
296  })
297 
298  comparisonLists = {} # directory of lists containing the validations that are comparable
299  for validation in validations:
300  for referenceName in validation.filesToCompare:
301  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
302  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
303  validationName = validationName.split("Preexisting")[-1]
304  if validationName in comparisonLists:
305  comparisonLists[ validationName ].append( validation )
306  else:
307  comparisonLists[ validationName ] = [ validation ]
308 
309  # introduced to merge individual validation outputs separately
310  # -> avoids problems with merge script
311  repMap["haddLoop"] = "mergeRetCode=0\n"
312  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
313  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
314  repMap["copyMergeScripts"] = ""
315  repMap["mergeParallelFilePrefixes"] = ""
316 
317  anythingToMerge = []
318  for validationType in comparisonLists:
319  for validation in comparisonLists[validationType]:
320  if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
321  continue
322  if validationType not in anythingToMerge:
323  anythingToMerge += [validationType]
324  repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
325  repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
326  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
327  repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
328  "xrdcp -f "
329  +validation.getRepMap()["finalOutputFile"]
330  +" root://eoscms//eos/cms"
331  +validation.getRepMap()["finalResultFile"]
332  +"; fi\n")
333  repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
334  "mergeRetCode=${tmpMergeRetCode}; fi\n")
335  for f in validation.getRepMap()["outputFiles"]:
336  longName = os.path.join("/store/caf/user/$USER/",
337  validation.getRepMap()["eosdir"], f)
338  repMap["rmUnmerged"] += " $eos rm "+longName+"\n"
339  repMap["rmUnmerged"] += ("else\n"
340  " echo -e \\n\"WARNING: Merging failed, unmerged"
341  " files won't be deleted.\\n"
342  "(Ignore this warning if merging was done earlier)\"\n"
343  "fi\n")
344 
345  if "OfflineValidation" in anythingToMerge:
346  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
347  createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
348  repMap["mergeOfflineParJobsScriptPath"] )
349  repMap["copyMergeScripts"] += ("cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n"
350  "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
351 
352  if anythingToMerge:
353  # DownloadData is the section which merges output files from parallel jobs
354  # it uses the file TkAlOfflineJobsMerge.C
355  repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
356  else:
357  repMap["DownloadData"] = ""
358 
359 
360  if "OfflineValidation" in comparisonLists:
361  repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
362  createExtendedValidationScript(comparisonLists["OfflineValidation"],
363  repMap["extendedValScriptPath"],
364  "OfflineValidation")
365  repMap["RunExtendedOfflineValidation"] = \
366  replaceByMap(configTemplates.extendedValidationExecution, repMap)
367 
368  if "TrackSplittingValidation" in comparisonLists:
369  repMap["trackSplitPlotScriptPath"] = \
370  os.path.join(path, "TkAlTrackSplitPlot.C")
371  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
372  repMap["trackSplitPlotScriptPath"] )
373  repMap["RunTrackSplitPlot"] = \
374  replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
375 
376  if "ZMuMuValidation" in comparisonLists:
377  repMap["mergeZmumuPlotsScriptPath"] = \
378  os.path.join(path, "TkAlMergeZmumuPlots.C")
379  createMergeZmumuPlotsScript(comparisonLists["ZMuMuValidation"],
380  repMap["mergeZmumuPlotsScriptPath"] )
381  repMap["MergeZmumuPlots"] = \
382  replaceByMap(configTemplates.mergeZmumuPlotsExecution, repMap)
383 
384  repMap["CompareAlignments"] = "#run comparisons"
385  if "OfflineValidation" in comparisonLists:
386  compareStrings = [ val.getCompareStrings("OfflineValidation") for val in comparisonLists["OfflineValidation"] ]
387  compareStringsPlain = [ val.getCompareStrings("OfflineValidation", plain=True) for val in comparisonLists["OfflineValidation"] ]
388 
389  repMap.update({"validationId": "OfflineValidation",
390  "compareStrings": " , ".join(compareStrings),
391  "compareStringsPlain": " ".join(compareStringsPlain) })
392 
393  repMap["CompareAlignments"] += \
394  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
395 
396  filePath = os.path.join(path, "TkAlMerge.sh")
397  theFile = open( filePath, "w" )
398  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
399  theFile.close()
400  os.chmod(filePath,0o755)
401 
402  return filePath
bitset_utilities::append
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
Definition: bitset_append.cc:10
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createMergeZmumuPlotsScript (   zMuMuValidationList,
  outFilePath 
)

Definition at line 269 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

269 
270 def createMergeZmumuPlotsScript(zMuMuValidationList, outFilePath):
271  repMap = zMuMuValidationList[0].getRepMap() # bit ugly since some special features are filled
272  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
273  repMap[ "mergeZmumuPlotsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
274 
275  for validation in zMuMuValidationList:
276  repMap[ "mergeZmumuPlotsInstantiation" ] = validation.appendToExtendedValidation( repMap[ "mergeZmumuPlotsInstantiation" ] )
277 
278  theFile = open( outFilePath, "w" )
279  theFile.write( replaceByMap( configTemplates.mergeZmumuPlotsTemplate ,repMap ) )
280  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createOfflineParJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 237 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

237 
238 def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
239  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
240 
241  theFile = open( outFilePath, "w" )
242  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
243  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 257 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

257 
258 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
259  repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
260  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
261  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
262 
263  for validation in trackSplittingValidationList:
264  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
265 
266  theFile = open( outFilePath, "w" )
267  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
268  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:6
def validateAlignments.loadTemplates (   config)

Definition at line 403 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

403 
404 def loadTemplates( config ):
405  if config.has_section("alternateTemplates"):
406  for templateName in config.options("alternateTemplates"):
407  if templateName == "AutoAlternates":
408  continue
409  newTemplateName = config.get("alternateTemplates", templateName )
410  #print "replacing default %s template by %s"%( templateName, newTemplateName)
411  configTemplates.alternateTemplate(templateName, newTemplateName)
412 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:280
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 414 of file validateAlignments.py.

References createMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), GetRecoTauVFromDQM_MC_cff.next, and split.

414 
415 def main(argv = None):
416  if argv == None:
417  argv = sys.argv[1:]
418  optParser = optparse.OptionParser()
419  optParser.description = """All-in-one Alignment Validation.
420 This will run various validation procedures either on batch queues or interactively.
421 If no name is given (-N parameter) a name containing time and date is created automatically.
422 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
423 """
424  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
425  help="create all scripts and cfg File but do not start jobs (default=False)")
426  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
427  help="get all Images created during the process (default= True)")
428  defaultConfig = "TkAlConfig.ini"
429  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
430  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
431  optParser.add_option("-N", "--Name", dest="Name",
432  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
433  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
434  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
435  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
436  help="get the status of the crab jobs", metavar="STATUS")
437  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
438  default = False,
439  help="run the tool to get full traceback of errors",
440  metavar="DEBUG")
441  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
442  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
443  " Works only for batch jobs")
444 
445  (options, args) = optParser.parse_args(argv)
446 
447  if not options.restrictTo == None:
448  options.restrictTo = options.restrictTo.split(",")
449 
450  options.config = [ os.path.abspath( iniFile ) for iniFile in \
451  options.config.split( "," ) ]
452  config = BetterConfigParser()
453  outputIniFileSet = set( config.read( options.config ) )
454  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
455 
456  # Check for missing ini file
457  if options.config == [ os.path.abspath( defaultConfig ) ]:
458  if ( not options.crabStatus ) and \
459  ( not os.path.exists( defaultConfig ) ):
460  raise AllInOneError( "Default 'ini' file '%s' not found!\n"
461  "You can specify another name with the "
462  "command line option '-c'/'--config'."
463  %( defaultConfig ))
464  else:
465  for iniFile in failedIniFiles:
466  if not os.path.exists( iniFile ):
467  raise AllInOneError( "'%s' does not exist. Please check for "
468  "typos in the filename passed to the "
469  "'-c'/'--config' option!"
470  %( iniFile ))
471  else:
472  raise AllInOneError(( "'%s' does exist, but parsing of the "
473  "content failed!" ) % iniFile)
474 
475  # get the job name
476  if options.Name == None:
477  if not options.crabStatus:
478  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
479  else:
480  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
481  "alignmentValidation_*" )
482  if len( existingValDirs ) > 0:
483  options.Name = existingValDirs[-1]
484  else:
485  print "Cannot guess last working directory!"
486  print ( "Please use the parameter '-N' or '--Name' to specify "
487  "the task for which you want a status report." )
488  return 1
489 
490  # set output path
491  outPath = os.path.abspath( options.Name )
492 
493  # Check status of submitted jobs and return
494  if options.crabStatus:
495  os.chdir( outPath )
496  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
497  if len( crabLogDirs ) == 0:
498  print "Found no crab tasks for job name '%s'"%( options.Name )
499  return 1
500  theCrab = crabWrapper.CrabWrapper()
501  for crabLogDir in crabLogDirs:
502  print
503  print "*" + "=" * 78 + "*"
504  print ( "| Status report and output retrieval for:"
505  + " " * (77 - len( "Status report and output retrieval for:" ) )
506  + "|" )
507  taskName = crabLogDir.replace( "crab.", "" )
508  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
509  print "*" + "=" * 78 + "*"
510  print
511  crabOptions = { "-getoutput":"",
512  "-c": crabLogDir }
513  try:
514  theCrab.run( crabOptions )
515  except AllInOneError as e:
516  print "crab: No output retrieved for this task."
517  crabOptions = { "-status": "",
518  "-c": crabLogDir }
519  theCrab.run( crabOptions )
520  return
521 
522  general = config.getGeneral()
523  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
524  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
525  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
526  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
527 
528  if not os.path.exists( outPath ):
529  os.makedirs( outPath )
530  elif not os.path.isdir( outPath ):
531  raise AllInOneError("the file %s is in the way rename the Job or move it away"%outPath)
532 
533  # replace default templates by the ones specified in the "alternateTemplates" section
534  loadTemplates( config )
535 
536  #save backup configuration file
537  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
538  config.write( backupConfigFile )
539 
540  validations = []
541  for validation in config.items("validation"):
542  alignmentList = [validation[1]]
543  validationsToAdd = [(validation[0],alignment) \
544  for alignment in alignmentList]
545  validations.extend(validationsToAdd)
546  jobs = [ ValidationJob( validation, config, options) \
547  for validation in validations ]
548  map( lambda job: job.createJob(), jobs )
549  validations = [ job.getValidation() for job in jobs ]
550 
551  createMergeScript(outPath, validations)
552 
553  print
554  map( lambda job: job.runJob(), jobs )
555 
556  if options.autoMerge:
557  # if everything is done as batch job, also submit TkAlMerge.sh to be run
558  # after the jobs have finished
559  if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
560  print "> Automatically merging jobs when they have ended"
561  repMap = {
562  "commands": config.getGeneral()["jobmode"].split(",")[1],
563  "jobName": "TkAlMerge",
564  "logDir": config.getGeneral()["logdir"],
565  "script": "TkAlMerge.sh",
566  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
567  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
568  }
569  for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
570  oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
571  if os.path.exists(oldlog):
572  os.remove(oldlog)
573 
574  getCommandOutput2("%(bsub)s %(commands)s "
575  "-o %(logDir)s/%(jobName)s.stdout "
576  "-e %(logDir)s/%(jobName)s.stderr "
577  "-w %(conditions)s "
578  "%(logDir)s/%(script)s"%repMap)
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:40
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:414
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5