Utilities Namespace Reference

Namespaces
	ReleaseScripts

Classes
class	TestModuleCommand

Functions
def	_build_options (args)
def	cleanUnscheduled (proc)
def	convertToSingleModuleEndPaths (process)
def	convertToUnscheduled (proc)
def	createTaskWithAllProducersAndFilters (process)
def	getHltConfiguration (menu, args)
def	ignoreAllFiltersOnPath (path)
def	loadHltConfiguration (process, menu, args)
def	moduleLabelsInSequences (sequences)
def	modulesInSequences (sequences)

Variables
	__lastCallUnary
	_levelInTasks
	onTask
	operator

Function Documentation

def Utilities._build_options (   args )

private

Definition at line 7 of file Utilities.py.

Referenced by getHltConfiguration(), and loadHltConfiguration().

def _build_options(**args):
  options = _options.HLTProcessOptions()
  for key, val in six.iteritems(args):
    setattr(options, key, val)
  return options

def Utilities.cleanUnscheduled

(

proc

)

Definition at line 78 of file Utilities.py.

References cmsPerfStripChart.dict, and list().

Referenced by convertToUnscheduled().

def cleanUnscheduled(proc):
  import FWCore.ParameterSet.Config as cms

  pathsAndEndPaths = dict(proc.paths)
  pathsAndEndPaths.update( dict(proc.endpaths) )

  #have to get them now since switching them after the
  # paths have been changed gives null labels
  if proc.schedule:
    pathNamesInScheduled = [p.label_() for p in proc.schedule]
  else:
    pathNamesInScheduled = False

  def getUnqualifiedName(name):
    if name[0] in set(['!','-']):
        return name[1:]
    return name

  def getQualifiedModule(name,proc):
    unqual_name = getUnqualifiedName(name)
    p=getattr(proc,unqual_name)
    if unqual_name != name:
      if name[0] == '!':
        p = ~p
      elif name[0] == '-':
        p = cms.ignore(p)
    return p

  # Loop over paths
  # On each path we move EDProducers and EDFilters that
  # are ignored to Tasks
  producerList = list()
  import six
  for pName, originalPath in six.iteritems(pathsAndEndPaths):
    producerList[:] = []
    qualified_names = []
    v = cms.DecoratedNodeNamePlusVisitor(qualified_names)
    originalPath.visit(v)
    remaining =[]

    for n in qualified_names:
      unqual_name = getUnqualifiedName(n)
      mod = getattr(proc,unqual_name)

      #remove EDProducer's and EDFilter's which are set to ignore
      if not (isinstance(mod, cms.EDProducer) or
              (n[0] =='-' and isinstance(mod, cms.EDFilter)) ):
        remaining.append(n)
      else:
        producerList.append(mod)

    taskList = []
    if v.leavesOnTasks():
      taskList.append(cms.Task(*(v.leavesOnTasks())))
    if (producerList):
      taskList.append(cms.Task(*producerList))

    if remaining:
      p = getQualifiedModule(remaining[0],proc)
      for m in remaining[1:]:
        p+=getQualifiedModule(m,proc)
      setattr(proc,pName,type(getattr(proc,pName))(p))
    else:
      setattr(proc,pName,type(getattr(proc,pName))())

    newPath = getattr(proc,pName)
    if (taskList):
      newPath.associate(*taskList)

  # If there is a schedule then it needs to point at
  # the new Path objects
  if proc.schedule:
      listOfTasks = list(proc.schedule._tasks)
      proc.schedule = cms.Schedule([getattr(proc,p) for p in pathNamesInScheduled])
      proc.schedule.associate(*listOfTasks)
  return proc

def Utilities.convertToSingleModuleEndPaths

(

process

)

Remove the EndPaths in the Process with more than one module
and replace with new EndPaths each with only one module.

Definition at line 175 of file Utilities.py.

Referenced by Utilities.TestModuleCommand.testConvertToSingleModuleEndPaths().

def convertToSingleModuleEndPaths(process):
    """Remove the EndPaths in the Process with more than one module
    and replace with new EndPaths each with only one module.
    """
    import FWCore.ParameterSet.Config as cms
    import six
    toRemove =[]
    added = []
    for n,ep in six.iteritems(process.endpaths_()):
        tsks = []
        ep.visit(cms.TaskVisitor(tsks))

        names = ep.moduleNames()
        if 1 == len(names):
            continue
        toRemove.append(n)
        for m in names:
            epName = m+"_endpath"
            setattr(process,epName,cms.EndPath(getattr(process,m),*tsks))
            added.append(epName)

    s = process.schedule_()
    if s:
        pathNames = [p.label_() for p in s]
        for rName in toRemove:
            pathNames.remove(rName)
        for n in added:
            pathNames.append(n)
        newS = cms.Schedule(*[getattr(process,n) for n in pathNames])
        if s._tasks:
          newS.associate(*s._tasks)
        process.setSchedule_(newS)

    for r in toRemove:
        delattr(process,r)

def Utilities.convertToUnscheduled

(

proc

)

Definition at line 46 of file Utilities.py.

References cleanUnscheduled().

Referenced by Utilities.TestModuleCommand.testNoSchedule(), and Utilities.TestModuleCommand.testWithSchedule().

def convertToUnscheduled(proc):
  import FWCore.ParameterSet.Config as cms
  """Given a 'Process', convert the python configuration from scheduled execution to unscheduled. This is done by
    1. Pulling EDProducers off Path and EndPath Sequences and putting them on an associated Task.
    2. Pulling EDFilters whose filter results are ignored off Path and EndPath Sequences and putting
    them on an associated Task.
    3. Fixing up the Schedule if needed
  """
  # Warning: It is not always possible to convert a configuration
  # where EDProducers are all run on Paths to an unscheduled
  # configuration by modifying only the python configuration.
  # There is more than one kind of pathological condition
  # that can cause this conversion to produce a configuration
  # that gives different results than the original configuration
  # when run under cmsRun. One should view the converted configuration
  # as a thing which needs to be validated. It is possible for there
  # to be pathologies that cannot be resolved by modifying only the
  # python configuration and may require redesign inside the C++ code
  # of the modules and redesign of the logic. For example,
  # an EDAnalyzer might try to get a product and check if the
  # returned handle isValid. Then it could behave differently
  # depending on whether or not the product was present.
  # The behavior when the product is not present could
  # be meaningful and important. In the unscheduled case,
  # the EDProducer will always run and the product could
  # always be there.

  proc.resolve()

  proc=cleanUnscheduled(proc)
  return proc

def Utilities.createTaskWithAllProducersAndFilters

(

process

)

Definition at line 167 of file Utilities.py.

Referenced by Utilities.TestModuleCommand.testCreateTaskWithAllProducersAndFilters().

def createTaskWithAllProducersAndFilters(process):
  from FWCore.ParameterSet.Config import Task
  import six

  l = [ p for p in six.itervalues(process.producers)]
  l.extend( (f for f in six.itervalues(process.filters)) )
  return Task(*l)

def Utilities.getHltConfiguration	(		menu,
			args
	)

Definition at line 14 of file Utilities.py.

References _build_options(), and FrontierConditions_GlobalTag_cff.dump.

def getHltConfiguration(menu, **args):
  args['menu']     = menu
  args['fragment'] = False
  options = _build_options(**args)

  hlt = _imp.new_module('hlt')
  exec(_confdb.HLTProcess(options).dump(), globals(), hlt.__dict__)
  return hlt.process

def Utilities.ignoreAllFiltersOnPath

(

path

)

Given a 'Path', find all EDFilters and wrap them in 'cms.ignore'

Definition at line 2 of file Utilities.py.

References resolutioncreator_cfi.object.

def ignoreAllFiltersOnPath(path):
  """Given a 'Path', find all EDFilters and wrap them in 'cms.ignore'
  """
  import FWCore.ParameterSet.Config as cms
  from FWCore.ParameterSet.SequenceTypes import _MutatingSequenceVisitor, _UnarySequenceOperator

  class IgnoreFilters(object):
    def __init__(self):
      self.__lastCallUnary = False
      self.onTask = False
    def __call__(self, obj):
      if self.onTask:
        return obj
      elif isinstance(obj,_UnarySequenceOperator):
        self.__lastCallUnary = True
      elif obj.isLeaf() and isinstance(obj, cms.EDFilter) and not self.__lastCallUnary:
        return cms.ignore(obj)
      else:
        self.__lastCallUnary = False
      return obj
  class IgnoreFiltersVisitor(_MutatingSequenceVisitor):
    def __init__(self):
      self.operator = IgnoreFilters()
      self._levelInTasks = 0
      super(type(self),self).__init__(self.operator)
    def enter(self,visitee):
      if isinstance(visitee, cms.Task):
        self._levelInTasks += 1
      self.operator.onTask = (self._levelInTasks > 0)
      super(IgnoreFiltersVisitor,self).enter(visitee)
    def leave(self,visitee):
      if self._levelInTasks > 0:
        if isinstance(visitee, cms.Task):
          self._levelInTasks -= 1
      super(IgnoreFiltersVisitor,self).leave(visitee)

  mutator = IgnoreFiltersVisitor()
  path.visit(mutator)
  if mutator._didApply():
    path._seq = mutator.result(path)[0]
    path._tasks.clear()
    path.associate(*mutator.result(path)[1])
  return path

def Utilities.loadHltConfiguration	(		process,
			menu,
			args
	)

Definition at line 24 of file Utilities.py.

References _build_options(), and FrontierConditions_GlobalTag_cff.dump.

def loadHltConfiguration(process, menu, **args):
  args['menu']     = menu
  args['fragment'] = True
  options = _build_options(**args)

  hlt = _imp.new_module('hlt')
  exec(_confdb.HLTProcess(options).dump(), globals(), hlt.__dict__)
  process.extend( hlt )

def Utilities.moduleLabelsInSequences

(

sequences

)

Definition at line 164 of file Utilities.py.

References modulesInSequences().

def moduleLabelsInSequences(* sequences):
  return [module.label() for module in modulesInSequences(* sequences)]

def Utilities.modulesInSequences

(

sequences

)

Definition at line 156 of file Utilities.py.

Referenced by moduleLabelsInSequences().

def modulesInSequences(* sequences):
  from FWCore.ParameterSet.SequenceTypes import ModuleNodeVisitor
  modules = []
  for sequence in sequences:
    sequence.visit(ModuleNodeVisitor(modules))
  return modules

Variable Documentation

Utilities.__lastCallUnary

private

Definition at line 10 of file Utilities.py.

Utilities._levelInTasks

private

Definition at line 25 of file Utilities.py.

Utilities.onTask

Definition at line 11 of file Utilities.py.

Utilities.operator

Definition at line 24 of file Utilities.py.

Referenced by AlcaBeamMonitorClient.AlcaBeamMonitorClient(), SiStripCalibLorentzAngle.algoBeginJob(), SiStripGainFromCalibTree.algoEndJob(), edm::debugging_allocator< T >.allocate(), DetIdSelectorTest.analyze(), BeamMonitorBx.BeamMonitorBx(), PFDQMEventSelector.beginJob(), GenericBenchmarkAnalyzer.beginJob(), DQMMessageLoggerClient.beginJob(), PFTester.beginJob(), DQMSourceExample.beginJob(), DQMFEDIntegrityClient.beginJob(), LaserDQM.beginJob(), DQMOfflineHLTEventInfoClient.beginJob(), DQMLumiMonitor.beginJob(), PixelVTXMonitor.beginJob(), DQMClientExample.beginJob(), SimplePhotonAnalyzer.beginJob(), APVValidationPlots.beginJob(), StripValidationPlots.beginJob(), TestSuite.beginJob(), SiPixelDQMRocLevelAnalyzer.beginJob(), DQMDaqInfo.beginJob(), FourVectorHLT.beginJob(), MEtoEDMConverter.beginJob(), MuonAlignment.beginJob(), DQMStoreStats.beginJob(), ESIntegrityTask.beginLuminosityBlock(), SiStripLAProfileBooker.beginRun(), SiStripCommissioningSource.beginRun(), SiStripDigiValid.bookHistograms(), TrackingTruthValid.bookHistograms(), SiPixelDigiValid.bookHistograms(), TrackerHitAnalyzer.bookHistograms(), TkConvValidator.bookHistograms(), SiPixelTrackingRecHitsValid.bookHistograms(), SiStripCondObjBuilderFromDb.buildAnalysisRelatedObjects(), ConversionPostprocessing.ConversionPostprocessing(), ConverterTester.ConverterTester(), SiStripCommissioningOfflineDbClient.createHistos(), CSCDCCUnpacker.CSCDCCUnpacker(), edm::debugging_allocator< T >.deallocate(), TrackerOfflineValidation::DirectoryWrapper.DirectoryWrapper(), OuterTrackerMCHarvester.dqmEndJob(), DQMGenericClient.dqmEndJob(), DQMFileReader.DQMFileReader(), DTnoiseDBValidation.DTnoiseDBValidation(), DTt0DBValidation.DTt0DBValidation(), EcalBarrelMonitorDbModule.EcalBarrelMonitorDbModule(), EcalBarrelSimHitsValidation.EcalBarrelSimHitsValidation(), EcalEndcapSimHitsValidation.EcalEndcapSimHitsValidation(), EcalPreshowerSimHitsValidation.EcalPreshowerSimHitsValidation(), EcalSimHitsValidation.EcalSimHitsValidation(), EgHLTOfflineSummaryClient.EgHLTOfflineSummaryClient(), DQMEDHarvester.endJob(), HarvestingAnalyzer.endJob(), DQMEDHarvester.endLuminosityBlock(), EDMtoMEConverter.endLuminosityBlockProduce(), MEtoEDMConverter.endLuminosityBlockProduce(), DQMRivetClient.endRun(), DQMGenericTnPClient.endRun(), ESIntegrityTask.endRun(), SiStripClassToMonitorCondData.endRun(), SiStripMonitorCondData.endRun(), OccupancyPlots.endRun(), EDMtoMEConverter.endRunProduce(), MEtoEDMConverter.endRunProduce(), ESDaqInfoTask.ESDaqInfoTask(), ESDataCertificationTask.ESDataCertificationTask(), ESDcsInfoTask.ESDcsInfoTask(), FedRawDataInputSource.FedRawDataInputSource(), SiStripQualityDQM.fillGrandSummaryMEs(), FourVectorHLT.FourVectorHLT(), GenFilterEfficiencyProducer.GenFilterEfficiencyProducer(), FWEvePtr< T >.get(), SiStripCablingDQM.getActiveDetIds(), ValidHitPairFilter.getCloseDets(), ClusterShapeTrackFilter.getGlobalPoss(), GlobalHitsProdHistStripper.GlobalHitsProdHistStripper(), GlobalMuonMatchAnalyzer.GlobalMuonMatchAnalyzer(), HLTInclusiveVBFClient.HLTInclusiveVBFClient(), HLTOverallSummary.HLTOverallSummary(), HLTPrescaler.HLTPrescaler(), HLTPrescaleRecorder.HLTPrescaleRecorder(), HLTScalersClient.HLTScalersClient(), DQMSourceExample.initialize(), DQMFEDIntegrityClient.initialize(), DQMOfflineHLTEventInfoClient.initialize(), DQMClientExample.initialize(), SiStripBadAPVandHotStripAlgorithmFromClusterOccupancy.initializeDQMHistograms(), L1ScalersClient.L1ScalersClient(), cond::persistency::PayloadProxy< cond::persistency::KeyList >.loadPayload(), TkHistoMap.loadServices(), SiStripFedCablingBuilderFromDb.make(), MEtoMEComparitor.MEtoMEComparitor(), MonitorElementsDb.MonitorElementsDb(), MuonTrackAnalyzer.MuonTrackAnalyzer(), MuonTrackResidualAnalyzer.MuonTrackResidualAnalyzer(), NoiseTask.NoiseTask(), PVSelector.operator()(), RunLumiSelector.operator()(), PVObjectSelector.operator()(), ElectronVPlusJetsIDSelectionFunctor.operator()(), PFElectronSelector.operator()(), PFMuonSelector.operator()(), funct::RootIntegrator.operator()(), MuonVPlusJetsIDSelectionFunctor.operator()(), JetIDSelectionFunctor.operator()(), PFJetIDSelectionFunctor.operator()(), SimpleCutBasedElectronIDSelectionFunctor.operator()(), de_rank< T >.operator()(), Indexed< T >.operator+(), EncodedTruthId.operator<(), EncodedTruthId.operator==(), PedsOnlyTask.PedsOnlyTask(), PFJetBenchmarkAnalyzer.PFJetBenchmarkAnalyzer(), PFMETBenchmarkAnalyzer.PFMETBenchmarkAnalyzer(), PFTauElecRejectionBenchmarkAnalyzer.PFTauElecRejectionBenchmarkAnalyzer(), PhotonPostprocessing.PhotonPostprocessing(), evf::FastMonitoringService.preBeginJob(), TkDetMapESProducer.produce(), SiStripBadModuleFedErrESSource.produce(), BuildTrackerMapPlugin.read(), recoBSVTagInfoValidationAnalyzer.recoBSVTagInfoValidationAnalyzer(), RecoMuonValidator.RecoMuonValidator(), SiStripCondObjBuilderFromDb.retrieveNumberAPVPairs(), RivetAnalyzer.RivetAnalyzer(), WPlusJetsEventSelector.scaleJets(), SiStripCondObjBuilderFromDb.setValuesApvLatency(), SiPixelFolderOrganizer.SiPixelFolderOrganizer(), SiStripCorrelateBadStripAndNoise.SiStripCorrelateBadStripAndNoise(), SiStripCorrelateNoise.SiStripCorrelateNoise(), SiStripFolderOrganizer.SiStripFolderOrganizer(), SiStripGainFromCalibTree.SiStripGainFromCalibTree(), SiStripGainFromData.SiStripGainFromData(), SiStripPlotGain.SiStripPlotGain(), SiStripQualityHotStripIdentifierRoot.SiStripQualityHotStripIdentifierRoot(), StatisticsFilter.StatisticsFilter(), TrackEfficiencyMonitor.TrackEfficiencyMonitor(), TrackerOfflineValidationSummary.TrackerOfflineValidationSummary(), TSCBLBuilderWithPropagator.TSCBLBuilderWithPropagator(), Clusterizer1D< reco::Track >.~Clusterizer1D(), DivisiveClusterizer1D< T >.~DivisiveClusterizer1D(), pixeltemp::DivisiveClusterizer1D< T >.~DivisiveClusterizer1D(), TSCBLBuilderWithPropagator.~TSCBLBuilderWithPropagator(), and TSCPBuilderNoMaterial.~TSCPBuilderNoMaterial().