CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineParJobsMergeScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 245 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

245 
246 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
247  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
248  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
249  repMap[ "resultPlotFile" ] = resultPlotFile
250  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
251 
252  for validation in offlineValidationList:
253  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
254 
255  theFile = open( outFilePath, "w" )
256  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
257  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
258  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 272 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineParJobsMergeScript(), createTrackSplitPlotScript(), if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

272 
273 def createMergeScript( path, validations ):
274  if(len(validations) == 0):
275  raise AllInOneError("Cowardly refusing to merge nothing!")
276 
277  repMap = validations[0].getRepMap() #FIXME - not nice this way
278  repMap.update({
279  "DownloadData":"",
280  "CompareAlignments":"",
281  "RunExtendedOfflineValidation":"",
282  "RunTrackSplitPlot":"",
283  "CMSSW_BASE": os.environ["CMSSW_BASE"],
284  "SCRAM_ARCH": os.environ["SCRAM_ARCH"],
285  "CMSSW_RELEASE_BASE": os.environ["CMSSW_RELEASE_BASE"],
286  })
287 
288  comparisonLists = {} # directory of lists containing the validations that are comparable
289  for validation in validations:
290  for referenceName in validation.filesToCompare:
291  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
292  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
293  validationName = validationName.split("Preexisting")[-1]
294  if validationName in comparisonLists:
295  comparisonLists[ validationName ].append( validation )
296  else:
297  comparisonLists[ validationName ] = [ validation ]
298 
299  # introduced to merge individual validation outputs separately
300  # -> avoids problems with merge script
301  repMap["haddLoop"] = "mergeRetCode=0\n"
302  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
303  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
304  repMap["copyMergeScripts"] = ""
305  repMap["mergeParallelFilePrefixes"] = ""
306 
307  anythingToMerge = []
308  for validationType in comparisonLists:
309  for validation in comparisonLists[validationType]:
310  if isinstance(validation, PreexistingValidation) or validation.NJobs == 1:
311  continue
312  if validationType not in anythingToMerge:
313  anythingToMerge += [validationType]
314  repMap["haddLoop"] += '\n\n\n\necho -e "\n\nMerging results from %s jobs"\n\n' % validationType
315  repMap["haddLoop"] = validation.appendToMerge(repMap["haddLoop"])
316  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
317  repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
318  "cmsStage -f "
319  +validation.getRepMap()["finalOutputFile"]
320  +" "
321  +validation.getRepMap()["finalResultFile"]
322  +"; fi\n")
323  repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
324  "mergeRetCode=${tmpMergeRetCode}; fi\n")
325  for f in validation.getRepMap()["outputFiles"]:
326  longName = os.path.join("/store/caf/user/$USER/",
327  validation.getRepMap()["eosdir"], f)
328  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
329  repMap["rmUnmerged"] += ("else\n"
330  " echo -e \\n\"WARNING: Merging failed, unmerged"
331  " files won't be deleted.\\n"
332  "(Ignore this warning if merging was done earlier)\"\n"
333  "fi\n")
334 
335  if "OfflineValidation" in anythingToMerge:
336  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
337  createOfflineParJobsMergeScript( comparisonLists["OfflineValidation"],
338  repMap["mergeOfflineParJobsScriptPath"] )
339  repMap["copyMergeScripts"] += ("cp .oO[CMSSW_BASE]Oo./src/Alignment/OfflineValidation/scripts/merge_TrackerOfflineValidation.C .\n"
340  "rfcp %s .\n" % repMap["mergeOfflineParJobsScriptPath"])
341 
342  if anythingToMerge:
343  # DownloadData is the section which merges output files from parallel jobs
344  # it uses the file TkAlOfflineJobsMerge.C
345  repMap["DownloadData"] += replaceByMap( configTemplates.mergeParallelResults, repMap )
346  else:
347  repMap["DownloadData"] = ""
348 
349 
350  if "OfflineValidation" in comparisonLists:
351  repMap["extendedValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
352  createExtendedValidationScript(comparisonLists["OfflineValidation"],
353  repMap["extendedValScriptPath"],
354  "OfflineValidation")
355  repMap["RunExtendedOfflineValidation"] = \
356  replaceByMap(configTemplates.extendedValidationExecution, repMap)
357 
358  if "TrackSplittingValidation" in comparisonLists:
359  repMap["trackSplitPlotScriptPath"] = \
360  os.path.join(path, "TkAlTrackSplitPlot.C")
361  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
362  repMap["trackSplitPlotScriptPath"] )
363  repMap["RunTrackSplitPlot"] = \
364  replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
365 
366  repMap["CompareAlignments"] = "#run comparisons"
367  for validationId in comparisonLists:
368  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
369  compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
370 
371  repMap.update({"validationId": validationId,
372  "compareStrings": " , ".join(compareStrings),
373  "compareStringsPlain": " ".join(compareStringsPlain) })
374 
375  repMap["CompareAlignments"] += \
376  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
377 
378  filePath = os.path.join(path, "TkAlMerge.sh")
379  theFile = open( filePath, "w" )
380  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
381  theFile.close()
382  os.chmod(filePath,0755)
383 
384  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
if
if(conf.exists("allCellsPositionCalc"))
Definition: Basic2DGenericPFlowClusterizer.cc:58
def validateAlignments.createOfflineParJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 238 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

238 
239 def createOfflineParJobsMergeScript(offlineValidationList, outFilePath):
240  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
241 
242  theFile = open( outFilePath, "w" )
243  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
244  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 259 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

259 
260 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
261  repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
262  repMap[ "CMSSW_BASE" ] = os.environ['CMSSW_BASE']
263  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
264 
265  for validation in trackSplittingValidationList:
266  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
267 
268  theFile = open( outFilePath, "w" )
269  # theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
270  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
271  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.loadTemplates (   config)

Definition at line 385 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

385 
386 def loadTemplates( config ):
387  if config.has_section("alternateTemplates"):
388  for templateName in config.options("alternateTemplates"):
389  if templateName == "AutoAlternates":
390  continue
391  newTemplateName = config.get("alternateTemplates", templateName )
392  #print "replacing default %s template by %s"%( templateName, newTemplateName)
393  configTemplates.alternateTemplate(templateName, newTemplateName)
394 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:304
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 396 of file validateAlignments.py.

References createMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), python.multivaluedict.map(), GetRecoTauVFromDQM_MC_cff.next, and split.

396 
397 def main(argv = None):
398  if argv == None:
399  argv = sys.argv[1:]
400  optParser = optparse.OptionParser()
401  optParser.description = """All-in-one Alignment Validation.
402 This will run various validation procedures either on batch queues or interactviely.
403 If no name is given (-N parameter) a name containing time and date is created automatically.
404 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
405 """
406  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
407  help="create all scripts and cfg File but do not start jobs (default=False)")
408  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
409  help="get all Images created during the process (default= False)")
410  defaultConfig = "TkAlConfig.ini"
411  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
412  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
413  optParser.add_option("-N", "--Name", dest="Name",
414  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
415  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
416  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
417  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
418  help="get the status of the crab jobs", metavar="STATUS")
419  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
420  default = False,
421  help="run the tool to get full traceback of errors",
422  metavar="DEBUG")
423  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
424  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
425  " Works only for batch jobs")
426 
427  (options, args) = optParser.parse_args(argv)
428 
429  if not options.restrictTo == None:
430  options.restrictTo = options.restrictTo.split(",")
431 
432  options.config = [ os.path.abspath( iniFile ) for iniFile in \
433  options.config.split( "," ) ]
434  config = BetterConfigParser()
435  outputIniFileSet = set( config.read( options.config ) )
436  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
437 
438  # Check for missing ini file
439  if options.config == [ os.path.abspath( defaultConfig ) ]:
440  if ( not options.crabStatus ) and \
441  ( not os.path.exists( defaultConfig ) ):
442  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
443  "You can specify another name with the "
444  "command line option '-c'/'--config'."
445  %( defaultConfig ))
446  else:
447  for iniFile in failedIniFiles:
448  if not os.path.exists( iniFile ):
449  raise AllInOneError, ( "'%s' does not exist. Please check for "
450  "typos in the filename passed to the "
451  "'-c'/'--config' option!"
452  %( iniFile ) )
453  else:
454  raise AllInOneError, ( "'%s' does exist, but parsing of the "
455  "content failed!" ) % iniFile
456 
457  # get the job name
458  if options.Name == None:
459  if not options.crabStatus:
460  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
461  else:
462  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
463  "alignmentValidation_*" )
464  if len( existingValDirs ) > 0:
465  options.Name = existingValDirs[-1]
466  else:
467  print "Cannot guess last working directory!"
468  print ( "Please use the parameter '-N' or '--Name' to specify "
469  "the task for which you want a status report." )
470  return 1
471 
472  # set output path
473  outPath = os.path.abspath( options.Name )
474 
475  # Check status of submitted jobs and return
476  if options.crabStatus:
477  os.chdir( outPath )
478  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
479  if len( crabLogDirs ) == 0:
480  print "Found no crab tasks for job name '%s'"%( options.Name )
481  return 1
482  theCrab = crabWrapper.CrabWrapper()
483  for crabLogDir in crabLogDirs:
484  print
485  print "*" + "=" * 78 + "*"
486  print ( "| Status report and output retrieval for:"
487  + " " * (77 - len( "Status report and output retrieval for:" ) )
488  + "|" )
489  taskName = crabLogDir.replace( "crab.", "" )
490  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
491  print "*" + "=" * 78 + "*"
492  print
493  crabOptions = { "-getoutput":"",
494  "-c": crabLogDir }
495  try:
496  theCrab.run( crabOptions )
497  except AllInOneError, e:
498  print "crab: No output retrieved for this task."
499  crabOptions = { "-status": "",
500  "-c": crabLogDir }
501  theCrab.run( crabOptions )
502  return
503 
504  general = config.getGeneral()
505  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
506  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
507  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
508  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
509 
510  # clean up of log directory to avoid cluttering with files with different
511  # random numbers for geometry comparison
512  if os.path.isdir( outPath ):
513  shutil.rmtree( outPath )
514 
515  if not os.path.exists( outPath ):
516  os.makedirs( outPath )
517  elif not os.path.isdir( outPath ):
518  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
519 
520  # replace default templates by the ones specified in the "alternateTemplates" section
521  loadTemplates( config )
522 
523  #save backup configuration file
524  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
525  config.write( backupConfigFile )
526 
527  validations = []
528  for validation in config.items("validation"):
529  alignmentList = [validation[1]]
530  validationsToAdd = [(validation[0],alignment) \
531  for alignment in alignmentList]
532  validations.extend(validationsToAdd)
533  jobs = [ ValidationJob( validation, config, options) \
534  for validation in validations ]
535  map( lambda job: job.createJob(), jobs )
536  validations = [ job.getValidation() for job in jobs ]
537 
538  createMergeScript(outPath, validations)
539 
540  print
541  map( lambda job: job.runJob(), jobs )
542 
543  if options.autoMerge:
544  # if everything is done as batch job, also submit TkAlMerge.sh to be run
545  # after the jobs have finished
546  if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
547  print "> Automatically merging jobs when they have ended"
548  repMap = {
549  "commands": config.getGeneral()["jobmode"].split(",")[1],
550  "jobName": "TkAlMerge",
551  "logDir": config.getGeneral()["logdir"],
552  "script": "TkAlMerge.sh",
553  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
554  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
555  }
556  getCommandOutput2("%(bsub)s %(commands)s "
557  "-o %(logDir)s/%(jobName)s.stdout "
558  "-e %(logDir)s/%(jobName)s.stderr "
559  "-w %(conditions)s "
560  "%(logDir)s/%(script)s"%repMap)
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:40
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:396
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5