CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineJobsMergeScript
 
def createParallelMergeScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath,
  resultPlotFile 
)

Definition at line 211 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript(), and createParallelMergeScript().

211 
212 def createExtendedValidationScript(offlineValidationList, outFilePath, resultPlotFile):
213  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
214  repMap[ "resultPlotFile" ] = resultPlotFile
215  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
216 
217  for validation in offlineValidationList:
218  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
219 
220  theFile = open( outFilePath, "w" )
221  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
222  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
223  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 224 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

224 
225 def createMergeScript( path, validations ):
226  if(len(validations) == 0):
227  msg = "Cowardly refusing to merge nothing!"
228  raise AllInOneError(msg)
229 
230  repMap = validations[0].getRepMap() #FIXME - not nice this way
231  repMap.update({
232  "DownloadData":"",
233  "CompareAlignments":"",
234  "RunExtendedOfflineValidation":""
235  })
236 
237  comparisonLists = {} # directory of lists containing the validations that are comparable
238  resultPlotFile = "" # string of a file name for createExtendedValidationScript
239  for validation in validations:
240  for referenceName in validation.filesToCompare:
241  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
242  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
243  if validationName in comparisonLists:
244  comparisonLists[ validationName ].append( validation )
245  else:
246  comparisonLists[ validationName ] = [ validation ]
247  if validationName == "OfflineValidation":
248  resultPlotFile = validationName
249 
250  if "OfflineValidation" in comparisonLists:
251  repMap["extendeValScriptPath"] = \
252  os.path.join(path, "TkAlExtendedOfflineValidation.C")
253  createExtendedValidationScript(comparisonLists["OfflineValidation"],
254  repMap["extendeValScriptPath"],
255  resultPlotFile)
256  repMap["RunExtendedOfflineValidation"] = \
257  replaceByMap(configTemplates.extendedValidationExecution, repMap)
258 
259  repMap["CompareAlignments"] = "#run comparisons"
260  for validationId in comparisonLists:
261  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
262 
263  repMap.update({"validationId": validationId,
264  "compareStrings": " , ".join(compareStrings) })
265 
266  repMap["CompareAlignments"] += \
267  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
268 
269  filePath = os.path.join(path, "TkAlMerge.sh")
270  theFile = open( filePath, "w" )
271  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
272  theFile.close()
273  os.chmod(filePath,0755)
274 
275  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createOfflineJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 203 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createParallelMergeScript().

203 
204 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
205  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
206  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
207 
208  theFile = open( outFilePath, "w" )
209  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
210  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createParallelMergeScript (   path,
  validations 
)

Definition at line 276 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineJobsMergeScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

276 
277 def createParallelMergeScript( path, validations ):
278  if( len(validations) == 0 ):
279  raise AllInOneError, "cowardly refusing to merge nothing!"
280 
281  repMap = validations[0].getRepMap() #FIXME - not nice this way
282  repMap.update({
283  "DownloadData":"",
284  "CompareAlignments":"",
285  "RunExtendedOfflineValidation":""
286  })
287 
288  comparisonLists = {} # directory of lists containing the validations that are comparable
289  resultPlotFile = "" # string of a file name for createExtendedValidationScript
290  for validation in validations:
291  for referenceName in validation.filesToCompare:
292  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
293  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
294  if validationName in comparisonLists:
295  comparisonLists[ validationName ].append( validation )
296  else:
297  comparisonLists[ validationName ] = [ validation ]
298  if validationName == "OfflineValidationParallel":
299  resultPlotFile = validationName
300 
301  if "OfflineValidationParallel" in comparisonLists:
302  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
303  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"], resultPlotFile )
304  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
305  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
306  repMap["mergeOfflineParJobsScriptPath"] )
307 
308  # introduced to merge individual validation outputs separately
309  # -> avoids problems with merge script
310  repMap["haddLoop"] = "mergeRetCode=0\n"
311  repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n"
312  for validation in comparisonLists["OfflineValidationParallel"]:
313  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
314  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
315  repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; "
316  "then mergeRetCode=${tmpMergeRetCode}; "
317  "fi\n")
318  repMap["haddLoop"] += ("cmsStage -f "
319  +validation.getRepMap()["outputFile"]
320  +" "
321  +validation.getRepMap()["resultFile"]
322  +"\n")
323  for f in validation.outputFiles:
324  longName = os.path.join("/store/caf/user/$USER/",
325  validation.getRepMap()["eosdir"], f)
326  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
327  repMap["rmUnmerged"] += ("else\n"
328  " echo \"WARNING: Merging failed, unmerged"
329  " files won't be deleted.\"\n"
330  "fi\n")
331 
332  repMap["RunExtendedOfflineValidation"] = \
333  replaceByMap(configTemplates.extendedValidationExecution, repMap)
334 
335  # DownloadData is the section which merges output files from parallel jobs
336  # it uses the file TkAlOfflineJobsMerge.C
337  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
338  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
339 
340  repMap["CompareAlignments"] = "#run comparisons"
341  for validationId in comparisonLists:
342  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
343 
344  repMap.update({"validationId": validationId,
345  "compareStrings": " , ".join(compareStrings) })
346 
347  repMap["CompareAlignments"] += \
348  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
349 
350  filePath = os.path.join(path, "TkAlMerge.sh")
351  theFile = open( filePath, "w" )
352  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
353  theFile.close()
354  os.chmod(filePath,0755)
355 
356  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.loadTemplates (   config)

Definition at line 357 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

357 
358 def loadTemplates( config ):
359  if config.has_section("alternateTemplates"):
360  for templateName in config.options("alternateTemplates"):
361  newTemplateName = config.get("alternateTemplates", templateName )
362  #print "replacing default %s template by %s"%( templateName, newTemplateName)
363  configTemplates.alternateTemplate(templateName, newTemplateName)
364 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:326
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 366 of file validateAlignments.py.

References createMergeScript(), createParallelMergeScript(), cmsRelvalreport.exit, loadTemplates(), python.multivaluedict.map(), GetRecoTauVFromDQM_MC_cff.next, and split.

366 
367 def main(argv = None):
368  if argv == None:
369  argv = sys.argv[1:]
370  optParser = optparse.OptionParser()
371  optParser.description = """ all-in-one alignment Validation
372  This will run various validation procedures either on batch queues or interactviely.
373 
374  If no name is given (-N parameter) a name containing time and date is created automatically
375 
376  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
377  """
378  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
379  help="create all scripts and cfg File but do not start jobs (default=False)")
380  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
381  help="get all Images created during the process (default= False)")
382  defaultConfig = "TkAlConfig.ini"
383  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
384  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
385  optParser.add_option("-N", "--Name", dest="Name",
386  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
387  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
388  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
389  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
390  help="get the status of the crab jobs", metavar="STATUS")
391  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
392  default = False,
393  help="Run the tool to get full traceback of errors.",
394  metavar="DEBUG")
395 
396  (options, args) = optParser.parse_args(argv)
397 
398  if not options.restrictTo == None:
399  options.restrictTo = options.restrictTo.split(",")
400 
401  options.config = [ os.path.abspath( iniFile ) for iniFile in \
402  options.config.split( "," ) ]
403  config = BetterConfigParser()
404  outputIniFileSet = set( config.read( options.config ) )
405  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
406 
407  # Check for missing ini file
408  if options.config == [ os.path.abspath( defaultConfig ) ]:
409  if ( not options.crabStatus ) and \
410  ( not os.path.exists( defaultConfig ) ):
411  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
412  "You can specify another name with the "
413  "command line option '-c'/'--config'."
414  %( defaultConfig ))
415  else:
416  for iniFile in failedIniFiles:
417  if not os.path.exists( iniFile ):
418  raise AllInOneError, ( "'%s' does not exist. Please check for "
419  "typos in the filename passed to the "
420  "'-c'/'--config' option!"
421  %( iniFile ) )
422  else:
423  raise AllInOneError, ( "'%s' does exist, but parsing of the "
424  "content failed!" )
425 
426  # get the job name
427  if options.Name == None:
428  if not options.crabStatus:
429  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
430  else:
431  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
432  "alignmentValidation_*" )
433  if len( existingValDirs ) > 0:
434  options.Name = existingValDirs[-1]
435  else:
436  print "Cannot guess last working directory!"
437  print ( "Please use the parameter '-N' or '--Name' to specify "
438  "the task for which you want a status report." )
439  return 1
440 
441  # set output path
442  outPath = os.path.abspath( options.Name )
443 
444  # Check status of submitted jobs and return
445  if options.crabStatus:
446  os.chdir( outPath )
447  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
448  if len( crabLogDirs ) == 0:
449  print "Found no crab tasks for job name '%s'"%( options.Name )
450  return 1
451  theCrab = crabWrapper.CrabWrapper()
452  for crabLogDir in crabLogDirs:
453  print
454  print "*" + "=" * 78 + "*"
455  print ( "| Status report and output retrieval for:"
456  + " " * (77 - len( "Status report and output retrieval for:" ) )
457  + "|" )
458  taskName = crabLogDir.replace( "crab.", "" )
459  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
460  print "*" + "=" * 78 + "*"
461  print
462  crabOptions = { "-getoutput":"",
463  "-c": crabLogDir }
464  try:
465  theCrab.run( crabOptions )
466  except AllInOneError, e:
467  print "crab: No output retrieved for this task."
468  crabOptions = { "-status": "",
469  "-c": crabLogDir }
470  theCrab.run( crabOptions )
471  return
472 
473  general = config.getGeneral()
474  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
475  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
476  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
477  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
478 
479  # clean up of log directory to avoid cluttering with files with different
480  # random numbers for geometry comparison
481  if os.path.isdir( outPath ):
482  shutil.rmtree( outPath )
483 
484  if not os.path.exists( outPath ):
485  os.makedirs( outPath )
486  elif not os.path.isdir( outPath ):
487  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
488 
489  # replace default templates by the ones specified in the "alternateTemplates" section
490  loadTemplates( config )
491 
492  #save backup configuration file
493  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
494  config.write( backupConfigFile )
495 
496  validations = []
497  for validation in config.items("validation"):
498  alignmentList = validation[1].split(config.getSep())
499  validationsToAdd = [(validation[0],alignment) \
500  for alignment in alignmentList]
501  validations.extend(validationsToAdd)
502  jobs = [ ValidationJob( validation, config, options) \
503  for validation in validations ]
504  map( lambda job: job.createJob(), jobs )
505  validations = [ job.getValidation() for job in jobs ]
506 
507  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
508  createMergeScript(outPath, validations)
509  else:
510  createParallelMergeScript( outPath, validations )
511 
512  print
513  map( lambda job: job.runJob(), jobs )
514 
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:38
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:366
GetRecoTauVFromDQM_MC_cff.next
tuple next
Definition: GetRecoTauVFromDQM_MC_cff.py:30
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5