test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineParJobsMergeScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 241 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

241 
242 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
243  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
244  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
245  repMap[ "resultPlotFile" ] = resultPlotFile
246  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
247 
248  for validation in offlineValidationList:
249  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
250 
251  theFile = open( outFilePath, "w" )
252  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
253  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
254  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 268 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineParJobsMergeScript(), createTrackSplitPlotScript(), citk.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

268 
269 def createMergeScript( path, validations ):
270  if(len(validations) == 0):
271  raise AllInOneError("Cowardly refusing to merge nothing!")
272 
273  repMap = validations[0].getRepMap() #FIXME - not nice this way
274  repMap.update({
275  "DownloadData":"",
276  "CompareAlignments":"",
277  "RunExtendedOfflineValidation":"",
278  "RunTrackSplitPlot":"",
279  "CMSSW_BASE": os.environ["CMSSW_BASE"],
280  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
281  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
282  })
283 
284  comparisonLists = {} # directory of lists containing the validations that are comparable
285  for validation in validations:
286  for referenceName in validation.filesToCompare:
287  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
288  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
289  validationName = validationName.split("Preexisting")[-1]
290  if validationName in comparisonLists:
291  comparisonLists[ validationName ].append( validation )
292  else:
293  comparisonLists[ validationName ] = [ validation ]
294 
295  # introduced to merge individual validation outputs separately
296  # -> avoids problems with merge script
297  repMap["haddLoop"] = "mergeRetCode=0\n"
298  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
299  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
300  repMap["copyMergeScripts"] = ""
301  repMap["mergeParallelFilePrefixes"] = ""
302 
303  anythingToMerge = []
304  for validationType in comparisonLists:
305  for validation in comparisonLists[validationType]:
306  if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
307  continue
308  if validationType not in anythingToMerge:
309  anythingToMerge += [validationType]
310  repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
311  repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
312  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
313  repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
314  "xrdcp -f "
315  +validation.getRepMap()["finalOutputFile"]
316  +" root://eoscms//eos/cms"
317  +validation.getRepMap()["finalResultFile"]
318  +"; fi\n")
319  repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
320  "mergeRetCode=${tmpMergeRetCode}; fi\n")
321  for f in validation.getRepMap()["outputFiles"]:
322  longName = os.path.join("/store/caf/user/$USER/",
323  validation.getRepMap()["eosdir"], f)
324  repMap["rmUnmerged"] += " $eos rm "+longName+"\n"
325  repMap["rmUnmerged"] += ("else\n"
326  " echo -e \\n\"WARNING: Merging failed, unmerged"
327  " files won't be deleted.\\n"
328  "(Ignore this warning if merging was done earlier)\"\n"
329  "fi\n")
330 
331  if "OfflineValidation" in anythingToMerge:
332  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
333  createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
334  repMap["mergeOfflineParJobsScriptPath"] )
335  repMap["copyMergeScripts"] += ("cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n"
336  "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
337 
338  if anythingToMerge:
339  # DownloadData is the section which merges output files from parallel jobs
340  # it uses the file TkAlOfflineJobsMerge.C
341  repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
342  else:
343  repMap["DownloadData"] = ""
344 
345 
346  if "OfflineValidation" in comparisonLists:
347  repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
348  createExtendedValidationScript(comparisonLists["OfflineValidation"],
349  repMap["extendedValScriptPath"],
350  "OfflineValidation")
351  repMap["RunExtendedOfflineValidation"] = \
352  replaceByMap(configTemplates.extendedValidationExecution, repMap)
353 
354  if "TrackSplittingValidation" in comparisonLists:
355  repMap["trackSplitPlotScriptPath"] = \
356  os.path.join(path, "TkAlTrackSplitPlot.C")
357  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
358  repMap["trackSplitPlotScriptPath"] )
359  repMap["RunTrackSplitPlot"] = \
360  replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
361 
362  repMap["CompareAlignments"] = "#run comparisons"
363  for validationId in comparisonLists:
364  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
365  compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
366 
367  repMap.update({"validationId": validationId,
368  "compareStrings": " , ".join(compareStrings),
369  "compareStringsPlain": " ".join(compareStringsPlain) })
370 
371  repMap["CompareAlignments"] += \
372  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
373 
374  filePath = os.path.join(path, "TkAlMerge.sh")
375  theFile = open( filePath, "w" )
376  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
377  theFile.close()
378  os.chmod(filePath,0755)
379 
380  return filePath
citk::if
if(c.getParameter< edm::InputTag >("puppiValueMap").label().size()!=0)
Definition: CITKPFIsolationSumProducerForPUPPI.cc:72
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createOfflineParJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 234 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

234 
235 def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
236  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
237 
238  theFile = open( outFilePath, "w" )
239  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
240  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 255 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

255 
256 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
257  repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
258  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
259  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
260 
261  for validation in trackSplittingValidationList:
262  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
263 
264  theFile = open( outFilePath, "w" )
265  # theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
266  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
267  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.loadTemplates (   config)

Definition at line 381 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

381 
382 def loadTemplates( config ):
383  if config.has_section("alternateTemplates"):
384  for templateName in config.options("alternateTemplates"):
385  if templateName == "AutoAlternates":
386  continue
387  newTemplateName = config.get("alternateTemplates", templateName )
388  #print "replacing default %s template by %s"%( templateName, newTemplateName)
389  configTemplates.alternateTemplate(templateName, newTemplateName)
390 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:307
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 392 of file validateAlignments.py.

References createMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), python.multivaluedict.map(), GetRecoTauVFromDQM_MC_cff.next, and split.

392 
393 def main(argv = None):
394  if argv == None:
395  argv = sys.argv[1:]
396  optParser = optparse.OptionParser()
397  optParser.description = """All-in-one Alignment Validation.
398 This will run various validation procedures either on batch queues or interactively.
399 If no name is given (-N parameter) a name containing time and date is created automatically.
400 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
401 """
402  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
403  help="create all scripts and cfg File but do not start jobs (default=False)")
404  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=True,
405  help="get all Images created during the process (default= True)")
406  defaultConfig = "TkAlConfig.ini"
407  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
408  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
409  optParser.add_option("-N", "--Name", dest="Name",
410  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
411  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
412  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
413  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
414  help="get the status of the crab jobs", metavar="STATUS")
415  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
416  default = False,
417  help="run the tool to get full traceback of errors",
418  metavar="DEBUG")
419  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
420  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
421  " Works only for batch jobs")
422 
423  (options, args) = optParser.parse_args(argv)
424 
425  if not options.restrictTo == None:
426  options.restrictTo = options.restrictTo.split(",")
427 
428  options.config = [ os.path.abspath( iniFile ) for iniFile in \
429  options.config.split( "," ) ]
430  config = BetterConfigParser()
431  outputIniFileSet = set( config.read( options.config ) )
432  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
433 
434  # Check for missing ini file
435  if options.config == [ os.path.abspath( defaultConfig ) ]:
436  if ( not options.crabStatus ) and \
437  ( not os.path.exists( defaultConfig ) ):
438  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
439  "You can specify another name with the "
440  "command line option '-c'/'--config'."
441  %( defaultConfig ))
442  else:
443  for iniFile in failedIniFiles:
444  if not os.path.exists( iniFile ):
445  raise AllInOneError, ( "'%s' does not exist. Please check for "
446  "typos in the filename passed to the "
447  "'-c'/'--config' option!"
448  %( iniFile ) )
449  else:
450  raise AllInOneError, ( "'%s' does exist, but parsing of the "
451  "content failed!" ) % iniFile
452 
453  # get the job name
454  if options.Name == None:
455  if not options.crabStatus:
456  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
457  else:
458  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
459  "alignmentValidation_*" )
460  if len( existingValDirs ) > 0:
461  options.Name = existingValDirs[-1]
462  else:
463  print "Cannot guess last working directory!"
464  print ( "Please use the parameter '-N' or '--Name' to specify "
465  "the task for which you want a status report." )
466  return 1
467 
468  # set output path
469  outPath = os.path.abspath( options.Name )
470 
471  # Check status of submitted jobs and return
472  if options.crabStatus:
473  os.chdir( outPath )
474  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
475  if len( crabLogDirs ) == 0:
476  print "Found no crab tasks for job name '%s'"%( options.Name )
477  return 1
478  theCrab = crabWrapper.CrabWrapper()
479  for crabLogDir in crabLogDirs:
480  print
481  print "*" + "=" * 78 + "*"
482  print ( "| Status report and output retrieval for:"
483  + " " * (77 - len( "Status report and output retrieval for:" ) )
484  + "|" )
485  taskName = crabLogDir.replace( "crab.", "" )
486  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
487  print "*" + "=" * 78 + "*"
488  print
489  crabOptions = { "-getoutput":"",
490  "-c": crabLogDir }
491  try:
492  theCrab.run( crabOptions )
493  except AllInOneError, e:
494  print "crab: No output retrieved for this task."
495  crabOptions = { "-status": "",
496  "-c": crabLogDir }
497  theCrab.run( crabOptions )
498  return
499 
500  general = config.getGeneral()
501  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
502  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
503  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
504  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
505 
506  if not os.path.exists( outPath ):
507  os.makedirs( outPath )
508  elif not os.path.isdir( outPath ):
509  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
510 
511  # replace default templates by the ones specified in the "alternateTemplates" section
512  loadTemplates( config )
513 
514  #save backup configuration file
515  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
516  config.write( backupConfigFile )
517 
518  validations = []
519  for validation in config.items("validation"):
520  alignmentList = [validation[1]]
521  validationsToAdd = [(validation[0],alignment) \
522  for alignment in alignmentList]
523  validations.extend(validationsToAdd)
524  jobs = [ ValidationJob( validation, config, options) \
525  for validation in validations ]
526  map( lambda job: job.createJob(), jobs )
527  validations = [ job.getValidation() for job in jobs ]
528 
529  createMergeScript(outPath, validations)
530 
531  print
532  map( lambda job: job.runJob(), jobs )
533 
534  if options.autoMerge:
535  # if everything is done as batch job, also submit TkAlMerge.sh to be run
536  # after the jobs have finished
537  if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
538  print "> Automatically merging jobs when they have ended"
539  repMap = {
540  "commands": config.getGeneral()["jobmode"].split(",")[1],
541  "jobName": "TkAlMerge",
542  "logDir": config.getGeneral()["logdir"],
543  "script": "TkAlMerge.sh",
544  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
545  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
546  }
547  for ext in ("stdout", "stderr", "stdout.gz", "stderr.gz"):
548  oldlog = "%(logDir)s/%(jobName)s."%repMap + ext
549  if os.path.exists(oldlog):
550  os.remove(oldlog)
551 
552  getCommandOutput2("%(bsub)s %(commands)s "
553  "-o %(logDir)s/%(jobName)s.stdout "
554  "-e %(logDir)s/%(jobName)s.stderr "
555  "-w %(conditions)s "
556  "%(logDir)s/%(script)s"%repMap)
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:40
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:392
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5