CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions
validateAlignments Namespace Reference

Classes

class  Alignment
 
class  BetterConfigParser
 — Classes —############################ More...
 
class  GenericValidation
 
class  GeometryComparision
 
class  MonteCarloValidation
 
class  OfflineValidation
 
class  OfflineValidationDQM
 
class  TrackSplittingValidation
 

Functions

def castorDirExists
 
def createExtendedValidationScript
 
def createMergeScript
 
def getCommandOutput2
 
def loadTemplates
 
def main
 — Main —############################ More...
 
def readAlignments
 
def readCompare
 
def readGeneral
 
def replaceByMap
 — Helpers —############################ replaces .oO[id]Oo. More...
 
def runJob
 

Function Documentation

def validateAlignments.castorDirExists (   path)

Definition at line 43 of file validateAlignments.py.

References getCommandOutput2().

43 
44 def castorDirExists(path):
45  if path[-1] == "/":
46  path = path[:-1]
47  containingPath = os.path.join( *path.split("/")[:-1] )
48  dirInQuestion = path.split("/")[-1]
49  try:
50  rawLines =getCommandOutput2("rfdir /"+containingPath).splitlines()
51  except RuntimeError:
52  return False
53  for line in rawLines:
54  if line.split()[0][0] == "d":
55  if line.split()[8] == dirInQuestion:
56  return True
57  return False
def validateAlignments.createExtendedValidationScript (   offlineValidationList,
  outFilePath 
)

Definition at line 582 of file validateAlignments.py.

References replaceByMap().

Referenced by createMergeScript().

582 
583 def createExtendedValidationScript(offlineValidationList, outFilePath):
584  repMap = offlineValidationList[0].getRepMap() # bit ugly since some special features are filled
585  repMap[ "extendedInstantiation" ] = "" #give it a "" at first in order to get the initialisation back
586 
587  for validation in offlineValidationList:
588  repMap[ "extendedInstantiation" ] = validation.appendToExtendedValidation( repMap[ "extendedInstantiation" ] )
589 
590  theFile = open( outFilePath, "w" )
591  theFile.write( replaceByMap( configTemplates.extendedVaidationTemplate ,repMap ) )
592  theFile.close()
validateAlignments.replaceByMap
def replaceByMap
— Helpers —############################ replaces .oO[id]Oo.
Definition: validateAlignments.py:14
def validateAlignments.createMergeScript (   path,
  validations 
)

Definition at line 593 of file validateAlignments.py.

References python.multivaluedict.append(), createExtendedValidationScript(), if(), join(), and replaceByMap().

Referenced by main().

593 
594 def createMergeScript( path, validations ):
595  if( len(validations) == 0 ):
596  raise StandardError, "cowardly refusing to merge nothing!"
597 
598  repMap = validations[0].getRepMap() #FIXME - not nice this way
599  repMap.update({
600  "DownloadData":"",
601  "CompareAllignments":"",
602  "RunExtendedOfflineValidation":""
603  })
604 
605  comparisonLists = {} # directory of lists containing the validations that are comparable
606  for validation in validations:
607  for referenceName in validation.filesToCompare:
608  validationName = "%s.%s"%(validation.__class__.__name__, referenceName)
609  validationName = validationName.split(".%s"%GenericValidation.defaultReferenceName )[0]
610  if validationName in comparisonLists:
611  comparisonLists[ validationName ].append( validation )
612  else:
613  comparisonLists[ validationName ] = [ validation ]
614 
615  if "OfflineValidation" in comparisonLists:
616  repMap["extendeValScriptPath"] = os.path.join(path, "TkAlExtendedOfflineValidation.C")
617  createExtendedValidationScript( comparisonLists["OfflineValidation"], repMap["extendeValScriptPath"] )
618  repMap["RunExtendedOfflineValidation"] = replaceByMap(configTemplates.extendedVaidationExecution, repMap)
619 
620  repMap["CompareAllignments"] = "#run comparisons"
621  for validationId in comparisonLists:
622  compareStrings = [ val.getCompareStrings(validationId) for val in comparisonLists[validationId] ]
623 
624  repMap.update({"validationId": validationId,
625  "compareStrings": " , ".join(compareStrings) })
626 
627  repMap["CompareAllignments"] += replaceByMap( configTemplates.compareAlignmentsExecution, repMap )
628 
629  filePath = os.path.join(path, "TkAlMerge.sh")
630  theFile = open( filePath, "w" )
631  theFile.write( replaceByMap( configTemplates.mergeTemplate, repMap ) )
632  theFile.close()
633  os.chmod(filePath,0755)
634 
635  return filePath
validateAlignments.replaceByMap
def replaceByMap
— Helpers —############################ replaces .oO[id]Oo.
Definition: validateAlignments.py:14
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
if
perl if(1 lt scalar(@::datatypes))
Definition: edlooper.cc:31
def validateAlignments.getCommandOutput2 (   command)

Definition at line 34 of file validateAlignments.py.

Referenced by castorDirExists(), and runJob().

34 
35 def getCommandOutput2(command):
36  child = os.popen(command)
37  data = child.read()
38  err = child.close()
39  if err:
40  raise RuntimeError, '%s failed w/ exit code %d' % (command, err)
41  return data
42 
#check if a directory exsits on castor
def validateAlignments.loadTemplates (   config)

Definition at line 636 of file validateAlignments.py.

References configTemplates.alternateTemplate().

Referenced by main().

636 
637 def loadTemplates( config ):
638  if config.has_section("alternateTemplates"):
639  for templateName in config.options("alternateTemplates"):
640  newTemplateName = config.get("alternateTemplates", templateName )
641  #print "replacing default %s template by %s"%( templateName, newTemplateName)
642  configTemplates.alternateTemplate(templateName, newTemplateName)
configTemplates.alternateTemplate
def alternateTemplate
Alternate Templates.
Definition: configTemplates.py:763
def validateAlignments.main (   argv = None)

— Main —############################

Definition at line 644 of file validateAlignments.py.

References createMergeScript(), loadTemplates(), readAlignments(), readGeneral(), and runJob().

Referenced by FWTableView.addTo(), FWTriggerTableView.setFrom(), and FWTableView.setFrom().

644 
645 def main(argv = None):
646  if argv == None:
647  argv = sys.argv[1:]
648  optParser = optparse.OptionParser()
649  optParser.description = """ all-in-one alignment Validation
650  This will run various validation procedures either on batch queues or interactviely.
651 
652  If no name is given (-N parameter) a name containing time and date is created automatically
653 
654  To merge the outcome of all validation procedures run TkAlMerge.sh in your validation's directory.
655  """
656  optParser.add_option("-n", "--dryRun", dest="dryRun", action="store_true", default=False,
657  help="create all scripts and cfg File but do not start jobs (default=False)")
658  optParser.add_option( "--getImages", dest="getImages", action="store_true", default=False,
659  help="get all Images created during the process (default= False)")
660  optParser.add_option("-c", "--config", dest="config",
661  help="configuration to use (default compareConfig.ini) this can be a comma-seperated list of all .ini file you want to merge", metavar="CONFIG")
662  optParser.add_option("-N", "--Name", dest="Name",
663  help="Name of this validation (default: alignmentValidation_DATE_TIME)", metavar="NAME")
664  optParser.add_option("-r", "--restrictTo", dest="restrictTo",
665  help="restrict validations to given modes (comma seperated) (default: no restriction)", metavar="RESTRICTTO")
666 
667  (options, args) = optParser.parse_args(argv)
668 
669  if not options.restrictTo == None:
670  options.restrictTo = options.restrictTo.split(",")
671  if options.config == None:
672  options.config = "compareConfig.ini"
673  else:
674  options.config = options.config.split(",")
675  result = []
676  for iniFile in options.config:
677  result.append( os.path.abspath(iniFile) )
678  options.config = result
679  config = BetterConfigParser()
680  config.read( options.config )
681 
682  if options.Name == None:
683  options.Name = "alignmentValidation_%s"%(datetime.datetime.now().strftime("%y%m%d_%H%M%S"))
684 
685  outPath = os.path.abspath( options.Name )
686  general = readGeneral( config )
687  config.set("general","workdir",os.path.join(general["workdir"],options.Name) )
688  config.set("general","datadir",os.path.join(general["datadir"],options.Name) )
689  config.set("general","logdir",os.path.join(general["logdir"],options.Name) )
690  if not os.path.exists( outPath ):
691  os.makedirs( outPath )
692  elif not os.path.isdir( outPath ):
693  raise StandardError,"the file %s is in the way rename the Job or move it away"%outPath
694 
695  #replace default templates by the once specified in the "alternateTemplates" section
696  loadTemplates( config )
697 
698  log = ""
699  alignments = readAlignments( config )
700  validations = []
701  for alignment in alignments:
702  alignment.restrictTo( options.restrictTo )
703  validations.extend( alignment.createValidations( config, options, alignments ) )
704 
705  scripts = []
706  for validation in validations:
707  validation.createConfiguration( outPath )
708  scripts.extend( validation.createScript( outPath ) )
709 
710  createMergeScript( outPath, validations )
711 
712  #save backup configuration file
713  backupConfigFile = open( os.path.join( outPath, "usedConfiguration.ini" ) , "w" )
714  config.write( backupConfigFile )
715 
716  for script in scripts:
717  name = os.path.splitext( os.path.basename( script ) )[0]
718  if options.dryRun:
719  print "%s would run: %s"%( name, script)
720  else:
721  runJob( name, script, config)
validateAlignments.main
def main
— Main —############################
Definition: validateAlignments.py:644
validateAlignments.BetterConfigParser
— Classes —############################
Definition: validateAlignments.py:59
def validateAlignments.readAlignments (   config)

Definition at line 524 of file validateAlignments.py.

Referenced by main().

524 
525 def readAlignments( config ):
526  result = []
527  for section in config.sections():
528  if "alignment:" in section:
529  result.append( Alignment( section.split("alignment:")[1], config ) )
530  return result
def validateAlignments.readCompare (   config)

Definition at line 531 of file validateAlignments.py.

531 
532 def readCompare( config ):
533  result = {}
534  for section in config.sections():
535  if "compare:" in section:
536  levels = config.get(section, "levels")
537  dbOutput = config.get(section, "dbOutput")
538  result[section.split(":")[1]] =(levels,dbOutput)
539  return result
def validateAlignments.readGeneral (   config)

Definition at line 540 of file validateAlignments.py.

Referenced by validateAlignments.OfflineValidation.__init__(), main(), and runJob().

540 
541 def readGeneral( config ):
542  result = {
543  "jobmode":"interactive",
544  "superPointingDataset":"",# see remark about rethroughung missing entries in general. this is needed to be backward compatible
545  "workdir":os.getcwd(),
546  "datadir":os.getcwd(),
547  "logdir":os.getcwd(),
548  "offlineModuleLevelHistsTransient":"False",
549  "OfflineTreeBaseDir":"TrackHitFilter",
550  "DMRMethod":"medianX",
551  "DMRMinimum":"30"
552  }
553  try:
554  for option in config.options("general"):
555  result[option] = config.get("general",option)
556 
557  if "localGeneral" in config.sections():
558  for option in result:
559  if option in [item[0] for item in config.items("localGeneral")]:
560  result[ option ] = config.get("localGeneral", option)
561  except ConfigParser.NoSectionError, section:
562  raise StandardError, "missing section '%s' in configuration files. This section is mandatory."%section
563  return result
def validateAlignments.replaceByMap (   target,
  map 
)

— Helpers —############################ replaces .oO[id]Oo.

by map[id] in target

Definition at line 14 of file validateAlignments.py.

Referenced by validateAlignments.GeometryComparision.createConfiguration(), validateAlignments.OfflineValidation.createConfiguration(), validateAlignments.MonteCarloValidation.createConfiguration(), validateAlignments.TrackSplittingValidation.createConfiguration(), createExtendedValidationScript(), createMergeScript(), validateAlignments.GeometryComparision.createScript(), validateAlignments.OfflineValidation.createScript(), validateAlignments.MonteCarloValidation.createScript(), validateAlignments.TrackSplittingValidation.createScript(), validateAlignments.Alignment.getLoadTemplate(), and validateAlignments.OfflineValidation.getRepMap().

14 
15 def replaceByMap(target, map):
16  result = target
17  for id in map:
18  #print " "+id+": "+map[id]
19  lifeSaver = 10e3
20  iteration = 0
21  while ".oO[" in result and "]Oo." in result:
22  for id in map:
23  result = result.replace(".oO["+id+"]Oo.",map[id])
24  iteration += 1
25  if iteration > lifeSaver:
26  problematicLines = ""
27  print map.keys()
28  for line in result.splitlines():
29  if ".oO[" in result and "]Oo." in line:
30  problematicLines += "%s\n"%line
31  raise StandardError, "Oh Dear, there seems to be an endless loop in replaceByMap!!\n%s\nrepMap"%problematicLines
32  return result
33 
#excute [command] and return output
validateAlignments.replaceByMap
def replaceByMap
— Helpers —############################ replaces .oO[id]Oo.
Definition: validateAlignments.py:14
def validateAlignments.runJob (   jobName,
  script,
  config 
)

Definition at line 564 of file validateAlignments.py.

References getCommandOutput2(), readGeneral(), and split.

Referenced by main().

564 
565 def runJob(jobName, script, config):
566  general = readGeneral( config )
567  log = "> Validating "+jobName
568  print "> Validating "+jobName
569  if general["jobmode"] == "interactive":
570  log += getCommandOutput2( script )
571  if general["jobmode"].split(",")[0] == "lxBatch":
572  repMap = {
573  "commands": general["jobmode"].split(",")[1],
574  "logDir": general["logdir"],
575  "jobName": jobName,
576  "script": script,
577  "bsub": "/afs/cern.ch/cms/caf/scripts/cmsbsub"
578  }
579 
580  log+=getCommandOutput2("%(bsub)s %(commands)s -J %(jobName)s -o %(logDir)s/%(jobName)s.stdout -e %(logDir)s/%(jobName)s.stderr %(script)s"%repMap)
581  return log
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5