CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineJobsMergeScript
 
def createParallelMergeScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath 
)

Definition at line 211 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript(), and createParallelMergeScript().

211 
212 def createExtendedValidationScript(offlineValidationList, outFilePath):
213  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
214  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
215 
216  for validation in offlineValidationList:
217  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
218 
219  theFile = open( outFilePath, "w" )
220  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
221  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
222  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 223 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

223 
224 def createMergeScript( path, validations ):
225  if(len(validations) == 0):
226  msg = "Cowardly refusing to merge nothing!"
227  raise AllInOneError(msg)
228 
229  repMap = validations[0].getRepMap() #FIXME - not nice this way
230  repMap.update({
231  "DownloadData":"",
232  "CompareAlignments":"",
233  "RunExtendedOfflineValidation":""
234  })
235 
236  comparisonLists = {} # directory of lists containing the validations that are comparable
237  for validation in validations:
238  for referenceName in validation.filesToCompare:
239  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
240  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
241  if validationName in comparisonLists:
242  comparisonLists[ validationName ].append( validation )
243  else:
244  comparisonLists[ validationName ] = [ validation ]
245 
246  if "OfflineValidation" in comparisonLists:
247  repMap["extendeValScriptPath"] = \
248  os.path.join(path, "TkAlExtendedOfflineValidation.C")
249  createExtendedValidationScript(comparisonLists["OfflineValidation"],
250  repMap["extendeValScriptPath"] )
251  repMap["RunExtendedOfflineValidation"] = \
252  replaceByMap(configTemplates.extendedValidationExecution, repMap)
253 
254  repMap["CompareAlignments"] = "#run comparisons"
255  for validationId in comparisonLists:
256  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
257 
258  repMap.update({"validationId": validationId,
259  "compareStrings": " , ".join(compareStrings) })
260 
261  repMap["CompareAlignments"] += \
262  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
263 
264  filePath = os.path.join(path, "TkAlMerge.sh")
265  theFile = open( filePath, "w" )
266  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
267  theFile.close()
268  os.chmod(filePath,0755)
269 
270  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createOfflineJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 203 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createParallelMergeScript().

203 
204 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
205  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
206  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
207 
208  theFile = open( outFilePath, "w" )
209  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
210  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createParallelMergeScript (   path,
  validations 
)

Definition at line 271 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineJobsMergeScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

271 
272 def createParallelMergeScript( path, validations ):
273  if( len(validations) == 0 ):
274  raise AllInOneError, "cowardly refusing to merge nothing!"
275 
276  repMap = validations[0].getRepMap() #FIXME - not nice this way
277  repMap.update({
278  "DownloadData":"",
279  "CompareAlignments":"",
280  "RunExtendedOfflineValidation":""
281  })
282 
283  comparisonLists = {} # directory of lists containing the validations that are comparable
284  for validation in validations:
285  for referenceName in validation.filesToCompare:
286  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
287  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
288  if validationName in comparisonLists:
289  comparisonLists[ validationName ].append( validation )
290  else:
291  comparisonLists[ validationName ] = [ validation ]
292 
293  if "OfflineValidationParallel" in comparisonLists:
294  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
295  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] )
296  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
297  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
298  repMap["mergeOfflineParJobsScriptPath"] )
299 
300  # introduced to merge individual validation outputs separately
301  # -> avoids problems with merge script
302  repMap["haddLoop"] = "mergeRetCode=0\n"
303  repMap["rmUnmerged"] = "if [[ mergeRetCode -eq 0 ]]; then\n"
304  for validation in comparisonLists["OfflineValidationParallel"]:
305  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
306  repMap["haddLoop"] += "tmpMergeRetCode=${?}\n"
307  repMap["haddLoop"] += ("if [[ mergeRetCode -eq 0 ]]; "
308  "then mergeRetCode=${tmpMergeRetCode}; "
309  "fi\n")
310  repMap["haddLoop"] += ("cmsStage -f "
311  +validation.getRepMap()["outputFile"]
312  +" "
313  +validation.getRepMap()["resultFile"]
314  +"\n")
315  for f in validation.outputFiles:
316  longName = os.path.join("/store/caf/user/$USER/",
317  validation.getRepMap()["eosdir"], f)
318  repMap["rmUnmerged"] += " cmsRm "+longName+"\n"
319  repMap["rmUnmerged"] += ("else\n"
320  " echo \"WARNING: Merging failed, unmerged"
321  " files won't be deleted.\"\n"
322  "fi\n")
323 
324  repMap["RunExtendedOfflineValidation"] = \
325  replaceByMap(configTemplates.extendedValidationExecution, repMap)
326 
327  # DownloadData is the section which merges output files from parallel jobs
328  # it uses the file TkAlOfflineJobsMerge.C
329  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
330  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
331 
332  repMap["CompareAlignments"] = "#run comparisons"
333  for validationId in comparisonLists:
334  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
335 
336  repMap.update({"validationId": validationId,
337  "compareStrings": " , ".join(compareStrings) })
338 
339  repMap["CompareAlignments"] += \
340  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
341 
342  filePath = os.path.join(path, "TkAlMerge.sh")
343  theFile = open( filePath, "w" )
344  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
345  theFile.close()
346  os.chmod(filePath,0755)
347 
348  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.loadTemplates (   config)

Definition at line 349 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

349 
350 def loadTemplates( config ):
351  if config.has_section("alternateTemplates"):
352  for templateName in config.options("alternateTemplates"):
353  newTemplateName = config.get("alternateTemplates", templateName )
354  #print "replacing default %s template by %s"%( templateName, newTemplateName)
355  configTemplates.alternateTemplate(templateName, newTemplateName)
356 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:325
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 358 of file validateAlignments.py.

References createMergeScript(), createParallelMergeScript(), cmsRelvalreport.exit, loadTemplates(), Association.map, and split.

358 
359 def main(argv = None):
360  if argv == None:
361  argv = sys.argv[1:]
362  optParser = optparse.OptionParser()
363  optParser.description = """ all-in-one alignment Validation
364  This will run various validation procedures either on batch queues or interactviely.
365 
366  If no name is given (-N parameter) a name containing time and date is created automatically
367 
368  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
369  """
370  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
371  help="create all scripts and cfg File but do not start jobs (default=False)")
372  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
373  help="get all Images created during the process (default= False)")
374  defaultConfig = "TkAlConfig.ini"
375  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
376  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
377  optParser.add_option("-N", "--Name", dest="Name",
378  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
379  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
380  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
381  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
382  help="get the status of the crab jobs", metavar="STATUS")
383  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
384  default = False,
385  help="Run the tool to get full traceback of errors.",
386  metavar="DEBUG")
387 
388  (options, args) = optParser.parse_args(argv)
389 
390  if not options.restrictTo == None:
391  options.restrictTo = options.restrictTo.split(",")
392 
393  options.config = [ os.path.abspath( iniFile ) for iniFile in \
394  options.config.split( "," ) ]
395  config = BetterConfigParser()
396  outputIniFileSet = set( config.read( options.config ) )
397  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
398 
399  # Check for missing ini file
400  if options.config == [ os.path.abspath( defaultConfig ) ]:
401  if ( not options.crabStatus ) and \
402  ( not os.path.exists( defaultConfig ) ):
403  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
404  "You can specify another name with the "
405  "command line option '-c'/'--config'."
406  %( defaultConfig ))
407  else:
408  for iniFile in failedIniFiles:
409  if not os.path.exists( iniFile ):
410  raise AllInOneError, ( "'%s' does not exist. Please check for "
411  "typos in the filename passed to the "
412  "'-c'/'--config' option!"
413  %( iniFile ) )
414  else:
415  raise AllInOneError, ( "'%s' does exist, but parsing of the "
416  "content failed!" )
417 
418  # get the job name
419  if options.Name == None:
420  if not options.crabStatus:
421  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
422  else:
423  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
424  "alignmentValidation_*" )
425  if len( existingValDirs ) > 0:
426  options.Name = existingValDirs[-1]
427  else:
428  print "Cannot guess last working directory!"
429  print ( "Please use the parameter '-N' or '--Name' to specify "
430  "the task for which you want a status report." )
431  return 1
432 
433  # set output path
434  outPath = os.path.abspath( options.Name )
435 
436  # Check status of submitted jobs and return
437  if options.crabStatus:
438  os.chdir( outPath )
439  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
440  if len( crabLogDirs ) == 0:
441  print "Found no crab tasks for job name '%s'"%( options.Name )
442  return 1
443  theCrab = crabWrapper.CrabWrapper()
444  for crabLogDir in crabLogDirs:
445  print
446  print "*" + "=" * 78 + "*"
447  print ( "| Status report and output retrieval for:"
448  + " " * (77 - len( "Status report and output retrieval for:" ) )
449  + "|" )
450  taskName = crabLogDir.replace( "crab.", "" )
451  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
452  print "*" + "=" * 78 + "*"
453  print
454  crabOptions = { "-getoutput":"",
455  "-c": crabLogDir }
456  try:
457  theCrab.run( crabOptions )
458  except AllInOneError, e:
459  print "crab: No output retrieved for this task."
460  crabOptions = { "-status": "",
461  "-c": crabLogDir }
462  theCrab.run( crabOptions )
463  return
464 
465  general = config.getGeneral()
466  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
467  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
468  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
469  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
470 
471  # clean up of log directory to avoid cluttering with files with different
472  # random numbers for geometry comparison
473  if os.path.isdir( outPath ):
474  shutil.rmtree( outPath )
475 
476  if not os.path.exists( outPath ):
477  os.makedirs( outPath )
478  elif not os.path.isdir( outPath ):
479  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
480 
481  # replace default templates by the ones specified in the "alternateTemplates" section
482  loadTemplates( config )
483 
484  #save backup configuration file
485  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
486  config.write( backupConfigFile )
487 
488  validations = []
489  for validation in config.items("validation"):
490  alignmentList = validation[1].split(config.getSep())
491  validationsToAdd = [(validation[0],alignment) \
492  for alignment in alignmentList]
493  validations.extend(validationsToAdd)
494  jobs = [ ValidationJob( validation, config, options) \
495  for validation in validations ]
496  map( lambda job: job.createJob(), jobs )
497  validations = [ job.getValidation() for job in jobs ]
498 
499  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
500  createMergeScript(outPath, validations)
501  else:
502  createParallelMergeScript( outPath, validations )
503 
504  print
505  map( lambda job: job.runJob(), jobs )
506 
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:38
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:358
Association.map
dictionary map
Definition: Association.py:205
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5