CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineJobsMergeScript
 
def createParallelMergeScript
 
def createTrackSplitPlotScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 211 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript(), and createParallelMergeScript().

211 
212 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
213  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
214  repMap[ "resultPlotFile" ] = resultPlotFile
215  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
216 
217  for validation in offlineValidationList:
218  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
219 
220  theFile = open( outFilePath, "w" )
221  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
222  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
223  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 236 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createTrackSplitPlotScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

236 
237 def createMergeScript( path, validations ):
238  if(len(validations) == 0):
239  msg = "Cowardly refusing to merge nothing!"
240  raise AllInOneError(msg)
241 
242  repMap = validations[0].getRepMap() #FIXME - not nice this way
243  repMap.update({
244  "DownloadData":"",
245  "CompareAlignments":"",
246  "RunExtendedOfflineValidation":"",
247  "RunTrackSplitPlot":""
248  })
249 
250  comparisonLists = {} # directory of lists containing the validations that are comparable
251  resultPlotFile = "" # string of a file name for createExtendedValidationScript
252  for validation in validations:
253  for referenceName in validation.filesToCompare:
254  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
255  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
256  if validationName in comparisonLists:
257  comparisonLists[ validationName ].append( validation )
258  else:
259  comparisonLists[ validationName ] = [ validation ]
260  if validationName == "OfflineValidation":
261  resultPlotFile = validationName
262 
263  if "OfflineValidation" in comparisonLists:
264  repMap["extendeValScriptPath"] = \
265  os.path.join(path, "TkAlExtendedOfflineValidation.C")
266  createExtendedValidationScript(comparisonLists["OfflineValidation"],
267  repMap["extendeValScriptPath"],
268  resultPlotFile)
269  repMap["RunExtendedOfflineValidation"] = \
270  replaceByMap(configTemplates.extendedValidationExecution, repMap)
271 
272  if "TrackSplittingValidation" in comparisonLists:
273  repMap["trackSplitPlotScriptPath"] = \
274  os.path.join(path, "TkAlTrackSplitPlot.C")
275  createTrackSplitPlotScript(comparisonLists["TrackSplittingValidation"],
276  repMap["trackSplitPlotScriptPath"] )
277  repMap["RunTrackSplitPlot"] = \
278  replaceByMap(configTemplates.trackSplitPlotExecution, repMap)
279 
280  repMap["CompareAlignments"] = "#run comparisons"
281  for validationId in comparisonLists:
282  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
283 
284  repMap.update({"validationId": validationId,
285  "compareStrings": " , ".join(compareStrings) })
286 
287  repMap["CompareAlignments"] += \
288  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
289 
290  filePath = os.path.join(path, "TkAlMerge.sh")
291  theFile = open( filePath, "w" )
292  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
293  theFile.close()
294  os.chmod(filePath,0755)
295 
296  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createOfflineJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 203 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createParallelMergeScript().

203 
204 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
205  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
206  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
207 
208  theFile = open( outFilePath, "w" )
209  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
210  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createParallelMergeScript (   path,
  validations 
)

Definition at line 297 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineJobsMergeScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

297 
298 def createParallelMergeScript( path, validations ):
299  if( len(validations) == 0 ):
300  raise AllInOneError, "cowardly refusing to merge nothing!"
301 
302  repMap = validations[0].getRepMap() #FIXME - not nice this way
303  repMap.update({
304  "DownloadData":"",
305  "CompareAlignments":"",
306  "RunExtendedOfflineValidation":""
307  })
308 
309  comparisonLists = {} # directory of lists containing the validations that are comparable
310  resultPlotFile = "" # string of a file name for createExtendedValidationScript
311  for validation in validations:
312  for referenceName in validation.filesToCompare:
313  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
314  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
315  if validationName in comparisonLists:
316  comparisonLists[ validationName ].append( validation )
317  else:
318  comparisonLists[ validationName ] = [ validation ]
319  if validationName == "OfflineValidationParallel":
320  resultPlotFile = validationName
321 
322  if "OfflineValidationParallel" in comparisonLists:
323  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
324  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"], resultPlotFile )
325  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
326  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
327  repMap["mergeOfflineParJobsScriptPath"] )
328 
329  # introduced to merge individual validation outputs separately
330  # -> avoids problems with merge script
331  repMap["haddLoop"] = "mergeRetCode=0\n"
332  repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n"
333  for validation in comparisonLists["OfflineValidationParallel"]:
334  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
335  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
336  repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; "
337  "then mergeRetCode=${tmpMergeRetCode}; "
338  "fi\n")
339  repMap["haddLoop"] += ("cmsStage -f "
340  +validation.getRepMap()["outputFile"]
341  +" "
342  +validation.getRepMap()["resultFile"]
343  +"\n")
344  for f in validation.outputFiles:
345  longName = os.path.join("/store/caf/user/$USER/",
346  validation.getRepMap()["eosdir"], f)
347  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
348  repMap["rmUnmerged"] += ("else\n"
349  " echo \"WARNING: Merging failed, unmerged"
350  " files won't be deleted.\"\n"
351  "fi\n")
352 
353  repMap["RunExtendedOfflineValidation"] = \
354  replaceByMap(configTemplates.extendedValidationExecution, repMap)
355 
356  # DownloadData is the section which merges output files from parallel jobs
357  # it uses the file TkAlOfflineJobsMerge.C
358  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
359  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
360 
361  repMap["CompareAlignments"] = "#run comparisons"
362  for validationId in comparisonLists:
363  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
364 
365  repMap.update({"validationId": validationId,
366  "compareStrings": " , ".join(compareStrings) })
367 
368  repMap["CompareAlignments"] += \
369  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
370 
371  filePath = os.path.join(path, "TkAlMerge.sh")
372  theFile = open( filePath, "w" )
373  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
374  theFile.close()
375  os.chmod(filePath,0755)
376 
377  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createTrackSplitPlotScript (   trackSplittingValidationList,
  outFilePath 
)

Definition at line 224 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript().

224 
225 def createTrackSplitPlotScript(trackSplittingValidationList, outFilePath):
226  repMap = trackSplittingValidationList[0].getRepMap() # bit ugly since some special features are filled
227  repMap[ "trackSplitPlotInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
228 
229  for validation in trackSplittingValidationList:
230  repMap[ "trackSplitPlotInstantiation" ] = validation.appendToExtendedValidation( repMap[ "trackSplitPlotInstantiation" ] )
231 
232  theFile = open( outFilePath, "w" )
233  # theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
234  theFile.write( replaceByMap( configTemplates.trackSplitPlotTemplate ,repMap ) )
235  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.loadTemplates (   config)

Definition at line 378 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

378 
379 def loadTemplates( config ):
380  if config.has_section("alternateTemplates"):
381  for templateName in config.options("alternateTemplates"):
382  newTemplateName = config.get("alternateTemplates", templateName )
383  #print "replacing default %s template by %s"%( templateName, newTemplateName)
384  configTemplates.alternateTemplate(templateName, newTemplateName)
385 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:327
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 387 of file validateAlignments.py.

References createMergeScript(), createParallelMergeScript(), cmsRelvalreport.exit, loadTemplates(), python.multivaluedict.map(), GetRecoTauVFromDQM_MC_cff.next, and split.

387 
388 def main(argv = None):
389  if argv == None:
390  argv = sys.argv[1:]
391  optParser = optparse.OptionParser()
392  optParser.description = """ all-in-one alignment Validation
393  This will run various validation procedures either on batch queues or interactviely.
394 
395  If no name is given (-N parameter) a name containing time and date is created automatically
396 
397  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
398  """
399  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
400  help="create all scripts and cfg File but do not start jobs (default=False)")
401  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
402  help="get all Images created during the process (default= False)")
403  defaultConfig = "TkAlConfig.ini"
404  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
405  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
406  optParser.add_option("-N", "--Name", dest="Name",
407  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
408  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
409  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
410  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
411  help="get the status of the crab jobs", metavar="STATUS")
412  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
413  default = False,
414  help="Run the tool to get full traceback of errors.",
415  metavar="DEBUG")
416 
417  (options, args) = optParser.parse_args(argv)
418 
419  if not options.restrictTo == None:
420  options.restrictTo = options.restrictTo.split(",")
421 
422  options.config = [ os.path.abspath( iniFile ) for iniFile in \
423  options.config.split( "," ) ]
424  config = BetterConfigParser()
425  outputIniFileSet = set( config.read( options.config ) )
426  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
427 
428  # Check for missing ini file
429  if options.config == [ os.path.abspath( defaultConfig ) ]:
430  if ( not options.crabStatus ) and \
431  ( not os.path.exists( defaultConfig ) ):
432  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
433  "You can specify another name with the "
434  "command line option '-c'/'--config'."
435  %( defaultConfig ))
436  else:
437  for iniFile in failedIniFiles:
438  if not os.path.exists( iniFile ):
439  raise AllInOneError, ( "'%s' does not exist. Please check for "
440  "typos in the filename passed to the "
441  "'-c'/'--config' option!"
442  %( iniFile ) )
443  else:
444  raise AllInOneError, ( "'%s' does exist, but parsing of the "
445  "content failed!" )
446 
447  # get the job name
448  if options.Name == None:
449  if not options.crabStatus:
450  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
451  else:
452  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
453  "alignmentValidation_*" )
454  if len( existingValDirs ) > 0:
455  options.Name = existingValDirs[-1]
456  else:
457  print "Cannot guess last working directory!"
458  print ( "Please use the parameter '-N' or '--Name' to specify "
459  "the task for which you want a status report." )
460  return 1
461 
462  # set output path
463  outPath = os.path.abspath( options.Name )
464 
465  # Check status of submitted jobs and return
466  if options.crabStatus:
467  os.chdir( outPath )
468  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
469  if len( crabLogDirs ) == 0:
470  print "Found no crab tasks for job name '%s'"%( options.Name )
471  return 1
472  theCrab = crabWrapper.CrabWrapper()
473  for crabLogDir in crabLogDirs:
474  print
475  print "*" + "=" * 78 + "*"
476  print ( "| Status report and output retrieval for:"
477  + " " * (77 - len( "Status report and output retrieval for:" ) )
478  + "|" )
479  taskName = crabLogDir.replace( "crab.", "" )
480  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
481  print "*" + "=" * 78 + "*"
482  print
483  crabOptions = { "-getoutput":"",
484  "-c": crabLogDir }
485  try:
486  theCrab.run( crabOptions )
487  except AllInOneError, e:
488  print "crab: No output retrieved for this task."
489  crabOptions = { "-status": "",
490  "-c": crabLogDir }
491  theCrab.run( crabOptions )
492  return
493 
494  general = config.getGeneral()
495  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
496  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
497  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
498  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
499 
500  # clean up of log directory to avoid cluttering with files with different
501  # random numbers for geometry comparison
502  if os.path.isdir( outPath ):
503  shutil.rmtree( outPath )
504 
505  if not os.path.exists( outPath ):
506  os.makedirs( outPath )
507  elif not os.path.isdir( outPath ):
508  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
509 
510  # replace default templates by the ones specified in the "alternateTemplates" section
511  loadTemplates( config )
512 
513  #save backup configuration file
514  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
515  config.write( backupConfigFile )
516 
517  validations = []
518  for validation in config.items("validation"):
519  alignmentList = validation[1].split(config.getSep())
520  validationsToAdd = [(validation[0],alignment) \
521  for alignment in alignmentList]
522  validations.extend(validationsToAdd)
523  jobs = [ ValidationJob( validation, config, options) \
524  for validation in validations ]
525  map( lambda job: job.createJob(), jobs )
526  validations = [ job.getValidation() for job in jobs ]
527 
528  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
529  createMergeScript(outPath, validations)
530  else:
531  createParallelMergeScript( outPath, validations )
532 
533  print
534  map( lambda job: job.runJob(), jobs )
535 
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:38
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:387
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5