CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Namespaces | Classes | Functions
Utilities Namespace Reference

Namespaces

 ReleaseScripts
 

Classes

class  InputTagLabelSet
 
class  TestModuleCommand
 

Functions

def _build_options
 
def convertToUnscheduled
 
def getHltConfiguration
 
def loadHltConfiguration
 
def removeModulesNotOnAPathExcluding
 
def traverseInputTags
 

Function Documentation

def Utilities._build_options (   args)
private

Definition at line 6 of file Utilities.py.

Referenced by getHltConfiguration(), and loadHltConfiguration().

6 
7 def _build_options(**args):
8  options = _options.HLTProcessOptions()
9  for key, val in args.iteritems():
10  setattr(options, key, val)
11  return options
12 
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.convertToUnscheduled (   proc)

Definition at line 69 of file Utilities.py.

References relativeConstraints.keys, and traverseInputTags().

Referenced by Utilities.TestModuleCommand.testNoSchedule(), and Utilities.TestModuleCommand.testWithSchedule().

69 
70 def convertToUnscheduled(proc):
71  import FWCore.ParameterSet.Config as cms
72  """Given a 'Process', convert the python configuration from scheduled execution to unscheduled. This is done by
73  1. Removing all modules not on Paths or EndPaths
74  2. Pulling EDProducers not dependent on EDFilters off of all Paths
75  3. Dropping any Paths which are now empty
76  4. Fixing up the Schedule if needed
77  """
78  # Warning: It is not always possible to convert a configuration
79  # where EDProducers are all run on Paths to an unscheduled
80  # configuration by modifying only the python configuration.
81  # There is more than one kind of pathological condition
82  # that can cause this conversion to produce a configuration
83  # that gives different results than the original configuration
84  # when run under cmsRun. One should view the converted configuration
85  # as a thing which needs to be validated. It is possible for there
86  # to be pathologies that cannot be resolved by modifying only the
87  # python configuration and may require redesign inside the C++ code
88  # of the modules and redesign of the logic. For example,
89  # an EDAnalyzer might try to get a product and check if the
90  # returned handle isValid. Then it could behave differently
91  # depending on whether or not the product was present.
92  # The behavior when the product is not present could
93  # be meaningful and important. In the unscheduled case,
94  # the EDProducer will always run and the product could
95  # always be there.
96 
97  # Remove all modules not on Paths or EndPaths
98  proc.prune()
99 
100  # Turn on unschedule mode
101  if not hasattr(proc,'options'):
102  proc.options = cms.untracked.PSet()
103  proc.options.allowUnscheduled = cms.untracked.bool(True)
104 
105  l = proc.paths
106  droppedPaths =[]
107  #have to get them now since switching them after the
108  # paths have been changed gives null labels
109  if proc.schedule:
110  pathNamesInScheduled = [p.label_() for p in proc.schedule]
111  else:
112  pathNamesInScheduled = False
113 
114  # Look for EDProducers that depend on an EDFilter, either
115  # directly or indirectly through another EDProducer. These
116  # EDProducers must stay in a path and run scheduled. The
117  # first loop will find all the direct dependencies, but then
118  # the loop must be repeated until no more dependencies
119  # are found in order to find the indirect dependencies.
120  # Note that here we are assuming that if a module
121  # has a configuration parameter that is an InputTag, then
122  # it depends on the module with the same module label as in
123  # the InputTag. In addition, there are a number of special
124  # cases where we have identified specific types of modules
125  # which use particular string parameters like InputTag
126  # module labels. And so we have some special code to
127  # handle those cases also. If there are other cases where
128  # the module gets things without using InputTags, then this
129  # conversion script can fail, which might result in ProductNotFound
130  # exceptions or other problems when the converted configuration is run.
131 
132  # The dictionary keys are are the types of the EDProducers
133  # The dictionary values are lists of parameter names that are strings
134  # used like InputTags.
135  knownStringInputLabels = {}
136  knownStringInputLabels['KProd'] = ['xSrc'] # a fake entry for a unit test below
137  knownStringInputLabels['SeedGeneratorFromRegionHitsEDProducer'] = ['vertexSrc']
138  knownStringInputLabels['PixelTrackProducer'] = ['vertexSrc']
139  knownStringInputLabels['PixelTracksProducer'] = ['vertexSrc']
140  knownStringInputLabels['SimpleTrackListMerger'] = ['TrackProducer1', 'TrackProducer2']
141 
142  allEDFilters = set(proc.filters_().keys())
143  allEDProducers = set(proc.producers_().keys())
144 
145  dependentProducers = set()
146  firstPass = True
147 
148  while True :
149 
150  dependentsFoundThisPass = False
151 
152  for producer in allEDProducers :
153  if producer not in dependentProducers :
154  iModule = getattr(proc,producer)
155 
156  stringInputLabels = []
157  moduleType = iModule.type_()
158  if moduleType in knownStringInputLabels :
159  stringInputLabels = knownStringInputLabels[moduleType]
160 
161  inputTagLabels = InputTagLabelSet()
162  traverseInputTags(getattr(proc,producer), inputTagLabels, stringInputLabels)
163 
164  if firstPass :
165  if not inputTagLabels.labels.isdisjoint(allEDFilters) :
166  dependentProducers.add(producer)
167  dependentsFoundThisPass = True
168 
169  if not inputTagLabels.labels.isdisjoint(dependentProducers) :
170  dependentProducers.add(producer)
171  dependentsFoundThisPass = True
172 
173  if not dependentsFoundThisPass :
174  break
175  firstPass = False
176 
177  # Loop over paths
178  # On each path we drop EDProducers except we
179  # keep the EDProducers that depend on EDFilters
180  for pName,p in l.iteritems():
181  nodes = []
182  v = cms.ModuleNodeVisitor(nodes)
183  p.visit(v)
184  names = [node.label_() for node in nodes]
185  remaining =[]
186 
187  for n in names:
188  if not isinstance(getattr(proc,n), cms.EDProducer):
189  remaining.append(n)
190  else :
191  if n in dependentProducers :
192  remaining.append(n)
193 
194  if remaining:
195  p=getattr(proc,remaining[0])
196  for m in remaining[1:]:
197  p+=getattr(proc,m)
198  setattr(proc,pName,cms.Path(p))
199  # drop empty paths
200  else:
201  setattr(proc,pName,cms.Path())
202 
203  # If there is a schedule then it needs to point at
204  # the new Path objects
205  if proc.schedule:
206  proc.schedule = cms.Schedule([getattr(proc,p) for p in pathNamesInScheduled])
207  return proc
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Utilities.convertToUnscheduled
def convertToUnscheduled
Definition: Utilities.py:69
def Utilities.getHltConfiguration (   menu,
  args 
)

Definition at line 13 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

13 
14 def getHltConfiguration(menu, **args):
15  args['menu'] = menu
16  args['fragment'] = False
17  options = _build_options(**args)
18 
19  hlt = _imp.new_module('hlt')
20  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
21  return hlt.process
22 
Utilities.getHltConfiguration
def getHltConfiguration
Definition: Utilities.py:13
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.loadHltConfiguration (   process,
  menu,
  args 
)

Definition at line 23 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

23 
24 def loadHltConfiguration(process, menu, **args):
25  args['menu'] = menu
26  args['fragment'] = True
27  options = _build_options(**args)
28 
29  hlt = _imp.new_module('hlt')
30  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
31  process.extend( hlt )
32 
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
Utilities.loadHltConfiguration
def loadHltConfiguration
Definition: Utilities.py:23
def Utilities.removeModulesNotOnAPathExcluding (   process,
  keepList = () 
) 
Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
and remove them if they do not appear on a Path or EndPath.  One can optionally pass in
a list of modules which even if they are not on a Path or EndPath you wish to have stay 
in the configuration [useful for unscheduled execution].

Definition at line 1 of file Utilities.py.

1 def removeModulesNotOnAPathExcluding( process, keepList=() ):
2  """Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
3  and remove them if they do not appear on a Path or EndPath. One can optionally pass in
4  a list of modules which even if they are not on a Path or EndPath you wish to have stay
5  in the configuration [useful for unscheduled execution].
6  """
7  allMods=set((x for x in process.producers_().iterkeys()))
8  allMods.update((x for x in process.filters_().iterkeys()))
9  allMods.update((x for x in process.analyzers_().iterkeys()))
10  allMods.update((x for x in process.outputModules_().iterkeys()))
11 
12  modulesOnPaths = set()
13  for p in process.paths_():
14  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
15  for p in process.endpaths_():
16  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
17 
18  notOnPaths = allMods.difference(modulesOnPaths)
19 
20  keepModuleNames = set( (x.label_() for x in keepList) )
21 
22  getRidOf = notOnPaths.difference(keepModuleNames)
23 
24  for n in getRidOf:
25  delattr(process,n)
26 
Utilities.removeModulesNotOnAPathExcluding
def removeModulesNotOnAPathExcluding
Definition: Utilities.py:1
def Utilities.traverseInputTags (   pset,
  visitor,
  stringInputLabels 
)

Definition at line 44 of file Utilities.py.

References relativeConstraints.keys.

Referenced by convertToUnscheduled().

44 
45 def traverseInputTags(pset, visitor, stringInputLabels):
46  from FWCore.ParameterSet.Mixins import _Parameterizable
47  from FWCore.ParameterSet.Types import VPSet, VInputTag, InputTag, string
48 
49  # Loop over parameters in a PSet
50  for name in pset.parameters_().keys() :
51  value = getattr(pset,name)
52  # Recursive calls into a PSet in a PSet
53  if isinstance(value, _Parameterizable) :
54  traverseInputTags(value, visitor, stringInputLabels)
55  # Recursive calls into PSets in a VPSet
56  elif isinstance(value, VPSet) :
57  for (n, psetInVPSet) in enumerate(value):
58  traverseInputTags(psetInVPSet, visitor, stringInputLabels)
59  # Get the labels from a VInputTag
60  elif isinstance(value, VInputTag) :
61  visitor(value)
62  # Get the label from an InputTag
63  elif isinstance(value, InputTag) :
64  visitor(value)
65  # Known module labels in string objects
66  elif stringInputLabels and isinstance(value, string) and name in stringInputLabels and value.value() :
67  visitor.labels.add(value.value())
68  #ignore the rest
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Generated for CMSSW Reference Manual by doxygen 1.8.5