CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineJobsMergeScript
 
def createParallelMergeScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 230 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript(), and createParallelMergeScript().

230 
231 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
232  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
233  repMap[ "resultPlotFile" ] = resultPlotFile
234  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
235 
236  for validation in offlineValidationList:
237  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
238 
239  theFile = open( outFilePath, "w" )
240  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
241  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
242  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 255 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createTrackSplitPlotScript(), if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

255 
256 def createMergeScript( path, validations ):
257  if(len(validations) == 0):
258  msg = "Cowardly refusing to merge nothing!"
259  raise AllInOneError(msg)
260 
261  repMap = validations[0].getRepMap() #FIXME - not nice this way
262  repMap.update({
263  "DownloadData":"",
264  "CompareAlignments":"",
265  "RunExtendedOfflineValidation":"",
266  "RunTrackSplitPlot":""
267  })
268 
269  comparisonLists = {} # directory of lists containing the validations that are comparable
270  for validation in validations:
271  for referenceName in validation.filesToCompare:
272  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
273  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
274  if validationName in comparisonLists:
275  comparisonLists[ validationName ].append( validation )
276  else:
277  comparisonLists[ validationName ] = [ validation ]
278 
279  if "OfflineValidation" in comparisonLists:
280  repMap["extendeValScriptPath"] = \
281  os.path.join(path, "TkAlExtendedOfflineValidation.C")
282  createExtendedValidationScript(comparisonLists["OfflineValidation"],
283  repMap["extendeValScriptPath"],
284  "OfflineValidation")
285  repMap["RunExtendedOfflineValidation"] = \
286  replaceByMap(configTemplates.extendedValidationExecution, repMap)
287 
288  if "TrackSplittingValidation" in comparisonLists:
289  repMap["trackSplitPlotScriptPath"] = \
290  os.path.join(path, "TkAlTrackSplitPlot.C")
291  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
292  repMap["trackSplitPlotScriptPath"] )
293  repMap["RunTrackSplitPlot"] = \
294  replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
295 
296  repMap["CompareAlignments"] = "#run comparisons"
297  for validationId in comparisonLists:
298  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
299  compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
300 
301  repMap.update({"validationId": validationId,
302  "compareStrings": " , ".join(compareStrings),
303  "compareStringsPlain": " ".join(compareStringsPlain) })
304 
305  repMap["CompareAlignments"] += \
306  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
307 
308  filePath = os.path.join(path, "TkAlMerge.sh")
309  theFile = open( filePath, "w" )
310  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
311  theFile.close()
312  os.chmod(filePath,0755)
313 
314  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
if
perl if(1 lt scalar(@::datatypes))
Definition: edlooper.cc:31
def validateAlignments.createOfflineJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 222 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createParallelMergeScript().

222 
223 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
224  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
225  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
226 
227  theFile = open( outFilePath, "w" )
228  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
229  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createParallelMergeScript (   path,
  validations 
)

Definition at line 315 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineJobsMergeScript(), if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

315 
316 def createParallelMergeScript( path, validations ):
317  if( len(validations) == 0 ):
318  raise AllInOneError, "cowardly refusing to merge nothing!"
319 
320  repMap = validations[0].getRepMap() #FIXME - not nice this way
321  repMap.update({
322  "DownloadData":"",
323  "CompareAlignments":"",
324  "RunExtendedOfflineValidation":"",
325  "RunTrackSplitPlot":""
326  })
327 
328  comparisonLists = {} # directory of lists containing the validations that are comparable
329  for validation in validations:
330  for referenceName in validation.filesToCompare:
331  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
332  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
333  if validationName in comparisonLists:
334  comparisonLists[ validationName ].append( validation )
335  else:
336  comparisonLists[ validationName ] = [ validation ]
337 
338  if "OfflineValidationParallel" in comparisonLists:
339  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
340  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
341  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
342  repMap["mergeOfflineParJobsScriptPath"] )
343 
344  # introduced to merge individual validation outputs separately
345  # -> avoids problems with merge script
346  repMap["haddLoop"] = "mergeRetCode=0\n"
347  repMap["rmUnmerged"] = ("if [[ mergeRetCode -eq 0 ]]; then\n"
348  " echo -e \\n\"Merging succeeded, removing original files.\"\n")
349  for validation in comparisonLists["OfflineValidationParallel"]:
350  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
351  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
352  repMap["haddLoop"] += ("if [[ tmpMergeRetCode -eq 0 ]]; then "
353  "cmsStage -f "
354  +validation.getRepMap()["outputFile"]
355  +" "
356  +validation.getRepMap()["resultFile"]
357  +"; fi\n")
358  repMap["haddLoop"] += ("if [[ ${tmpMergeRetCode} -gt ${mergeRetCode} ]]; then "
359  "mergeRetCode=${tmpMergeRetCode}; fi\n")
360  for f in validation.outputFiles:
361  longName = os.path.join("/store/caf/user/$USER/",
362  validation.getRepMap()["eosdir"], f)
363  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
364  repMap["rmUnmerged"] += ("else\n"
365  " echo -e \\n\"WARNING: Merging failed, unmerged"
366  " files won't be deleted.\\n"
367  "(Ignore this warning if merging was done earlier)\"\n"
368  "fi\n")
369 
370  repMap["RunExtendedOfflineValidation"] = \
371  replaceByMap(configTemplates.extendedValidationExecution, repMap)
372 
373  # DownloadData is the section which merges output files from parallel jobs
374  # it uses the file TkAlOfflineJobsMerge.C
375  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
376  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
377 
378  # For the rest of the script, OfflineValidations and
379  # OfflineParallelValidations are comparable:
380  if "OfflineValidation" in comparisonLists:
381  comparisonLists["OfflineValidationParallel"].extend(comparisonLists["OfflineValidation"])
382  del comparisonLists["OfflineValidation"]
383  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"],
384  repMap["extendeValScriptPath"],
385  "OfflineValidationParallel")
386 
387  repMap["CompareAlignments"] = "#run comparisons"
388  for validationId in comparisonLists:
389  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
390  compareStringsPlain = [ val.getCompareStrings(validationId, plain=True) for val in comparisonLists[validationId] ]
391 
392  repMap.update({"validationId": validationId,
393  "compareStrings": " , ".join(compareStrings),
394  "compareStringsPlain": " ".join(compareStringsPlain) })
395 
396  repMap["CompareAlignments"] += \
397  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
398 
399  filePath = os.path.join(path, "TkAlMerge.sh")
400  theFile = open( filePath, "w" )
401  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
402  theFile.close()
403  os.chmod(filePath,0755)
404 
405  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
if
perl if(1 lt scalar(@::datatypes))
Definition: edlooper.cc:31
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 243 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

243 
244 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
245  repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
246  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
247 
248  for validation in trackSplittingValidationList:
249  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
250 
251  theFile = open( outFilePath, "w" )
252  # theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
253  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
254  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.loadTemplates (   config)

Definition at line 406 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

406 
407 def loadTemplates( config ):
408  if config.has_section("alternateTemplates"):
409  for templateName in config.options("alternateTemplates"):
410  newTemplateName = config.get("alternateTemplates", templateName )
411  #print "replacing default %s template by %s"%( templateName, newTemplateName)
412  configTemplates.alternateTemplate(templateName, newTemplateName)
413 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:341
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 415 of file validateAlignments.py.

References createMergeScript(), createParallelMergeScript(), cmsRelvalreport.exit, helperFunctions.getCommandOutput2(), join(), loadTemplates(), python.multivaluedict.map(), GetRecoTauVFromDQM_MC_cff.next, runtimedef.set(), and split.

415 
416 def main(argv = None):
417  if argv == None:
418  argv = sys.argv[1:]
419  optParser = optparse.OptionParser()
420  optParser.description = """All-in-one Alignment Validation.
421 This will run various validation procedures either on batch queues or interactviely.
422 If no name is given (-N parameter) a name containing time and date is created automatically.
423 To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
424 """
425  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
426  help="create all scripts and cfg File but do not start jobs (default=False)")
427  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
428  help="get all Images created during the process (default= False)")
429  defaultConfig = "TkAlConfig.ini"
430  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
431  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
432  optParser.add_option("-N", "--Name", dest="Name",
433  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
434  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
435  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
436  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
437  help="get the status of the crab jobs", metavar="STATUS")
438  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
439  default = False,
440  help="run the tool to get full traceback of errors",
441  metavar="DEBUG")
442  optParser.add_option("-m", "--autoMerge", dest="autoMerge", action="store_true", default = False,
443  help="submit TkAlMerge.sh to run automatically when all jobs have finished (default=False)."
444  " Works only for batch jobs")
445 
446  (options, args) = optParser.parse_args(argv)
447 
448  if not options.restrictTo == None:
449  options.restrictTo = options.restrictTo.split(",")
450 
451  options.config = [ os.path.abspath( iniFile ) for iniFile in \
452  options.config.split( "," ) ]
453  config = BetterConfigParser()
454  outputIniFileSet = set( config.read( options.config ) )
455  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
456 
457  # Check for missing ini file
458  if options.config == [ os.path.abspath( defaultConfig ) ]:
459  if ( not options.crabStatus ) and \
460  ( not os.path.exists( defaultConfig ) ):
461  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
462  "You can specify another name with the "
463  "command line option '-c'/'--config'."
464  %( defaultConfig ))
465  else:
466  for iniFile in failedIniFiles:
467  if not os.path.exists( iniFile ):
468  raise AllInOneError, ( "'%s' does not exist. Please check for "
469  "typos in the filename passed to the "
470  "'-c'/'--config' option!"
471  %( iniFile ) )
472  else:
473  raise AllInOneError, ( "'%s' does exist, but parsing of the "
474  "content failed!" )
475 
476  # get the job name
477  if options.Name == None:
478  if not options.crabStatus:
479  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
480  else:
481  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
482  "alignmentValidation_*" )
483  if len( existingValDirs ) > 0:
484  options.Name = existingValDirs[-1]
485  else:
486  print "Cannot guess last working directory!"
487  print ( "Please use the parameter '-N' or '--Name' to specify "
488  "the task for which you want a status report." )
489  return 1
490 
491  # set output path
492  outPath = os.path.abspath( options.Name )
493 
494  # Check status of submitted jobs and return
495  if options.crabStatus:
496  os.chdir( outPath )
497  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
498  if len( crabLogDirs ) == 0:
499  print "Found no crab tasks for job name '%s'"%( options.Name )
500  return 1
501  theCrab = crabWrapper.CrabWrapper()
502  for crabLogDir in crabLogDirs:
503  print
504  print "*" + "=" * 78 + "*"
505  print ( "| Status report and output retrieval for:"
506  + " " * (77 - len( "Status report and output retrieval for:" ) )
507  + "|" )
508  taskName = crabLogDir.replace( "crab.", "" )
509  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
510  print "*" + "=" * 78 + "*"
511  print
512  crabOptions = { "-getoutput":"",
513  "-c": crabLogDir }
514  try:
515  theCrab.run( crabOptions )
516  except AllInOneError, e:
517  print "crab: No output retrieved for this task."
518  crabOptions = { "-status": "",
519  "-c": crabLogDir }
520  theCrab.run( crabOptions )
521  return
522 
523  general = config.getGeneral()
524  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
525  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
526  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
527  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
528 
529  # clean up of log directory to avoid cluttering with files with different
530  # random numbers for geometry comparison
531  if os.path.isdir( outPath ):
532  shutil.rmtree( outPath )
533 
534  if not os.path.exists( outPath ):
535  os.makedirs( outPath )
536  elif not os.path.isdir( outPath ):
537  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
538 
539  # replace default templates by the ones specified in the "alternateTemplates" section
540  loadTemplates( config )
541 
542  #save backup configuration file
543  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
544  config.write( backupConfigFile )
545 
546  validations = []
547  for validation in config.items("validation"):
548  alignmentList = validation[1].split(config.getSep())
549  validationsToAdd = [(validation[0],alignment) \
550  for alignment in alignmentList]
551  validations.extend(validationsToAdd)
552  jobs = [ ValidationJob( validation, config, options) \
553  for validation in validations ]
554  map( lambda job: job.createJob(), jobs )
555  validations = [ job.getValidation() for job in jobs ]
556 
557  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
558  createMergeScript(outPath, validations)
559  else:
560  createParallelMergeScript( outPath, validations )
561 
562  print
563  map( lambda job: job.runJob(), jobs )
564 
565  if options.autoMerge:
566  # if everything is done as batch job, also submit TkAlMerge.sh to be run
567  # after the jobs have finished
568  if ValidationJob.jobCount == ValidationJob.batchCount and config.getGeneral()["jobmode"].split(",")[0] == "lxBatch":
569  print "> Automatically merging jobs when they have ended"
570  repMap = {
571  "commands": config.getGeneral()["jobmode"].split(",")[1],
572  "jobName": "TkAlMerge",
573  "logDir": config.getGeneral()["logdir"],
574  "script": "TkAlMerge.sh",
575  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub",
576  "conditions": '"' + " && ".join(["ended(" + jobId + ")" for jobId in ValidationJob.batchJobIds]) + '"'
577  }
578  getCommandOutput2("%(bsub)s %(commands)s "
579  "-o %(logDir)s/%(jobName)s.stdout "
580  "-e %(logDir)s/%(jobName)s.stderr "
581  "-w %(conditions)s "
582  "%(logDir)s/%(script)s"%repMap)
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:38
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:415
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
runtimedef::set
void set(const std::string &name, int value)
set the flag, with a run-time name
Definition: ProfilingTools.cc:67
Generated for CMSSW Reference Manual by doxygen 1.8.5