CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  ValidationJob
 — Classes —############################ More...
 

Functions

def createExtendedValidationScript
 
def createMergeScript
 
def createOfflineJobsMergeScript
 
def createParallelMergeScript
 
def loadTemplates
 
def main
 — Main —############################ More...
 

Function Documentation

def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath 
)

Definition at line 211 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createMergeScript(), and createParallelMergeScript().

211 
212 def createExtendedValidationScript(offlineValidationList, outFilePath):
213  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
214  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
215 
216  for validation in offlineValidationList:
217  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
218 
219  theFile = open( outFilePath, "w" )
220  # theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
221  theFile.write( replaceByMap( configTemplates.extendedValidationTemplate ,repMap ) )
222  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 223 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

223 
224 def createMergeScript( path, validations ):
225  if(len(validations) == 0):
226  msg = "Cowardly refusing to merge nothing!"
227  raise AllInOneError(msg)
228 
229  repMap = validations[0].getRepMap() #FIXME - not nice this way
230  repMap.update({
231  "DownloadData":"",
232  "CompareAlignments":"",
233  "RunExtendedOfflineValidation":""
234  })
235 
236  comparisonLists = {} # directory of lists containing the validations that are comparable
237  for validation in validations:
238  for referenceName in validation.filesToCompare:
239  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
240  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
241  if validationName in comparisonLists:
242  comparisonLists[ validationName ].append( validation )
243  else:
244  comparisonLists[ validationName ] = [ validation ]
245 
246  if "OfflineValidation" in comparisonLists:
247  repMap["extendeValScriptPath"] = \
248  os.path.join(path, "TkAlExtendedOfflineValidation.C")
249  createExtendedValidationScript(comparisonLists["OfflineValidation"],
250  repMap["extendeValScriptPath"] )
251  repMap["RunExtendedOfflineValidation"] = \
252  replaceByMap(configTemplates.extendedValidationExecution, repMap)
253 
254  repMap["CompareAlignments"] = "#run comparisons"
255  for validationId in comparisonLists:
256  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
257 
258  repMap.update({"validationId": validationId,
259  "compareStrings": " , ".join(compareStrings) })
260 
261  repMap["CompareAlignments"] += \
262  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
263 
264  filePath = os.path.join(path, "TkAlMerge.sh")
265  theFile = open( filePath, "w" )
266  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
267  theFile.close()
268  os.chmod(filePath,0755)
269 
270  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.createOfflineJobsMergeScript (   offlineValidationList,
  outFilePath 
)

Definition at line 203 of file validateAlignments.py.

References helperFunctions.replaceByMap().

Referenced by createParallelMergeScript().

203 
204 def createOfflineJobsMergeScript(offlineValidationList, outFilePath):
205  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
206  repMap[ "mergeOfflinParJobsInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
207 
208  theFile = open( outFilePath, "w" )
209  theFile.write( replaceByMap( configTemplates.mergeOfflineParJobsTemplate ,repMap ) )
210  theFile.close()
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
def validateAlignments.createParallelMergeScript (   path,
  validations 
)

Definition at line 271 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), createOfflineJobsMergeScript(), reco.if(), join(), and helperFunctions.replaceByMap().

Referenced by main().

271 
272 def createParallelMergeScript( path, validations ):
273  if( len(validations) == 0 ):
274  raise AllInOneError, "cowardly refusing to merge nothing!"
275 
276  repMap = validations[0].getRepMap() #FIXME - not nice this way
277  repMap.update({
278  "DownloadData":"",
279  "CompareAlignments":"",
280  "RunExtendedOfflineValidation":""
281  })
282 
283  comparisonLists = {} # directory of lists containing the validations that are comparable
284  for validation in validations:
285  for referenceName in validation.filesToCompare:
286  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
287  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
288  if validationName in comparisonLists:
289  comparisonLists[ validationName ].append( validation )
290  else:
291  comparisonLists[ validationName ] = [ validation ]
292 
293  if "OfflineValidationParallel" in comparisonLists:
294  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
295  createExtendedValidationScript( comparisonLists["OfflineValidationParallel"], repMap["extendeValScriptPath"] )
296  repMap["mergeOfflineParJobsScriptPath"] = os.path.join(path, "TkAlOfflineJobsMerge.C")
297  createOfflineJobsMergeScript( comparisonLists["OfflineValidationParallel"],
298  repMap["mergeOfflineParJobsScriptPath"] )
299 
300  # introduced to merge individual validation outputs separately
301  # -> avoids problems with merge script
302  repMap["haddLoop"] = ""
303  for validation in comparisonLists["OfflineValidationParallel"]:
304  repMap["haddLoop"] = validation.appendToMergeParJobs(repMap["haddLoop"])
305  repMap["haddLoop"] += ("cmsStage -f "
306  +validation.getRepMap()["outputFile"]
307  +" "
308  +validation.getRepMap()["resultFile"]
309  +"\n")
310 
311  repMap["RunExtendedOfflineValidation"] = \
312  replaceByMap(configTemplates.extendedValidationExecution, repMap)
313 
314  # DownloadData is the section which merges output files from parallel jobs
315  # it uses the file TkAlOfflineJobsMerge.C
316  repMap["DownloadData"] += replaceByMap("rfcp .oO[mergeOfflineParJobsScriptPath]Oo. .", repMap)
317  repMap["DownloadData"] += replaceByMap( configTemplates.mergeOfflineParallelResults, repMap )
318 
319  repMap["CompareAlignments"] = "#run comparisons"
320  for validationId in comparisonLists:
321  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
322 
323  repMap.update({"validationId": validationId,
324  "compareStrings": " , ".join(compareStrings) })
325 
326  repMap["CompareAlignments"] += \
327  replaceByMap(configTemplates.compareAlignmentsExecution, repMap)
328 
329  filePath = os.path.join(path, "TkAlMerge.sh")
330  theFile = open( filePath, "w" )
331  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
332  theFile.close()
333  os.chmod(filePath,0755)
334 
335  return filePath
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
helperFunctions.replaceByMap
def replaceByMap
— Helpers —############################
Definition: helperFunctions.py:5
reco::if
if(dp >Float(M_PI)) dp-
def validateAlignments.loadTemplates (   config)

Definition at line 336 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

336 
337 def loadTemplates( config ):
338  if config.has_section("alternateTemplates"):
339  for templateName in config.options("alternateTemplates"):
340  newTemplateName = config.get("alternateTemplates", templateName )
341  #print "replacing default %s template by %s"%( templateName, newTemplateName)
342  configTemplates.alternateTemplate(templateName, newTemplateName)
343 
configTemplates.alternateTemplate
def alternateTemplate
### Alternate Templates ###
Definition: configTemplates.py:325
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 345 of file validateAlignments.py.

References createMergeScript(), createParallelMergeScript(), cmsRelvalreport.exit, loadTemplates(), Association.map, and split.

345 
346 def main(argv = None):
347  if argv == None:
348  argv = sys.argv[1:]
349  optParser = optparse.OptionParser()
350  optParser.description = """ all-in-one alignment Validation
351  This will run various validation procedures either on batch queues or interactviely.
352 
353  If no name is given (-N parameter) a name containing time and date is created automatically
354 
355  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
356  """
357  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
358  help="create all scripts and cfg File but do not start jobs (default=False)")
359  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
360  help="get all Images created during the process (default= False)")
361  defaultConfig = "TkAlConfig.ini"
362  optParser.add_option("-c", "--config", dest="config", default = defaultConfig,
363  help="configuration to use (default TkAlConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
364  optParser.add_option("-N", "--Name", dest="Name",
365  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
366  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
367  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
368  optParser.add_option("-s", "--status", dest="crabStatus", action="store_true", default = False,
369  help="get the status of the crab jobs", metavar="STATUS")
370  optParser.add_option("-d", "--debug", dest="debugMode", action="store_true",
371  default = False,
372  help="Run the tool to get full traceback of errors.",
373  metavar="DEBUG")
374 
375  (options, args) = optParser.parse_args(argv)
376 
377  if not options.restrictTo == None:
378  options.restrictTo = options.restrictTo.split(",")
379 
380  options.config = [ os.path.abspath( iniFile ) for iniFile in \
381  options.config.split( "," ) ]
382  config = BetterConfigParser()
383  outputIniFileSet = set( config.read( options.config ) )
384  failedIniFiles = [ iniFile for iniFile in options.config if iniFile not in outputIniFileSet ]
385 
386  # Check for missing ini file
387  if options.config == [ os.path.abspath( defaultConfig ) ]:
388  if ( not options.crabStatus ) and \
389  ( not os.path.exists( defaultConfig ) ):
390  raise AllInOneError, ( "Default 'ini' file '%s' not found!\n"
391  "You can specify another name with the "
392  "command line option '-c'/'--config'."
393  %( defaultConfig ))
394  else:
395  for iniFile in failedIniFiles:
396  if not os.path.exists( iniFile ):
397  raise AllInOneError, ( "'%s' does not exist. Please check for "
398  "typos in the filename passed to the "
399  "'-c'/'--config' option!"
400  %( iniFile ) )
401  else:
402  raise AllInOneError, ( "'%s' does exist, but parsing of the "
403  "content failed!" )
404 
405  # get the job name
406  if options.Name == None:
407  if not options.crabStatus:
408  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
409  else:
410  existingValDirs = fnmatch.filter( os.walk( '.' ).next()[1],
411  "alignmentValidation_*" )
412  if len( existingValDirs ) > 0:
413  options.Name = existingValDirs[-1]
414  else:
415  print "Cannot guess last working directory!"
416  print ( "Please use the parameter '-N' or '--Name' to specify "
417  "the task for which you want a status report." )
418  return 1
419 
420  # set output path
421  outPath = os.path.abspath( options.Name )
422 
423  # Check status of submitted jobs and return
424  if options.crabStatus:
425  os.chdir( outPath )
426  crabLogDirs = fnmatch.filter( os.walk('.').next()[1], "crab.*" )
427  if len( crabLogDirs ) == 0:
428  print "Found no crab tasks for job name '%s'"%( options.Name )
429  return 1
430  theCrab = crabWrapper.CrabWrapper()
431  for crabLogDir in crabLogDirs:
432  print
433  print "*" + "=" * 78 + "*"
434  print ( "| Status report and output retrieval for:"
435  + " " * (77 - len( "Status report and output retrieval for:" ) )
436  + "|" )
437  taskName = crabLogDir.replace( "crab.", "" )
438  print "| " + taskName + " " * (77 - len( taskName ) ) + "|"
439  print "*" + "=" * 78 + "*"
440  print
441  crabOptions = { "-getoutput":"",
442  "-c": crabLogDir }
443  try:
444  theCrab.run( crabOptions )
445  except AllInOneError, e:
446  print "crab: No output retrieved for this task."
447  crabOptions = { "-status": "",
448  "-c": crabLogDir }
449  theCrab.run( crabOptions )
450  return
451 
452  general = config.getGeneral()
453  config.set("internals","workdir",os.path.join(general["workdir"],options.Name) )
454  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
455  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
456  config.set("general","eosdir",os.path.join("AlignmentValidation", general["eosdir"], options.Name) )
457 
458  # clean up of log directory to avoid cluttering with files with different
459  # random numbers for geometry comparison
460  if os.path.isdir( outPath ):
461  shutil.rmtree( outPath )
462 
463  if not os.path.exists( outPath ):
464  os.makedirs( outPath )
465  elif not os.path.isdir( outPath ):
466  raise AllInOneError,"the file %s is in the way rename the Job or move it away"%outPath
467 
468  # replace default templates by the ones specified in the "alternateTemplates" section
469  loadTemplates( config )
470 
471  #save backup configuration file
472  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
473  config.write( backupConfigFile )
474 
475  validations = []
476  for validation in config.items("validation"):
477  alignmentList = validation[1].split(config.getSep())
478  validationsToAdd = [(validation[0],alignment) \
479  for alignment in alignmentList]
480  validations.extend(validationsToAdd)
481  jobs = [ ValidationJob( validation, config, options) \
482  for validation in validations ]
483  map( lambda job: job.createJob(), jobs )
484  validations = [ job.getValidation() for job in jobs ]
485 
486  if "OfflineValidationParallel" not in [val.__class__.__name__ for val in validations]:
487  createMergeScript(outPath, validations)
488  else:
489  createParallelMergeScript( outPath, validations )
490 
491  print
492  map( lambda job: job.runJob(), jobs )
493 
validateAlignments.ValidationJob
— Classes —############################
Definition: validateAlignments.py:38
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:345
Association.map
dictionary map
Definition: Association.py:205
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5