CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Namespaces | Classes | Functions
Utilities Namespace Reference

Namespaces

 ReleaseScripts
 

Classes

class  InputTagLabelSet
 
class  TestModuleCommand
 

Functions

def _build_options
 
def convertToUnscheduled
 
def getHltConfiguration
 
def loadHltConfiguration
 
def removeModulesNotOnAPathExcluding
 
def traverseInputTags
 

Function Documentation

def Utilities._build_options (   args)
private

Definition at line 6 of file Utilities.py.

Referenced by getHltConfiguration(), and loadHltConfiguration().

6 
7 def _build_options(**args):
8  options = _options.HLTProcessOptions()
9  for key, val in args.iteritems():
10  setattr(options, key, val)
11  return options
12 
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.convertToUnscheduled (   proc)

Definition at line 69 of file Utilities.py.

References relativeConstraints.keys, and traverseInputTags().

Referenced by Utilities.TestModuleCommand.testNoSchedule(), and Utilities.TestModuleCommand.testWithSchedule().

69 
70 def convertToUnscheduled(proc):
71  import FWCore.ParameterSet.Config as cms
72  """Given a 'Process', convert the python configuration from scheduled execution to unscheduled. This is done by
73  1. Removing all modules not on Paths or EndPaths
74  2. Pulling EDProducers not dependent on EDFilters off of all Paths
75  3. Dropping any Paths which are now empty
76  4. Fixing up the Schedule if needed
77  """
78  # Warning: It is not always possible to convert a configuration
79  # where EDProducers are all run on Paths to an unscheduled
80  # configuration by modifying only the python configuration.
81  # There is more than one kind of pathological condition
82  # that can cause this conversion to produce a configuration
83  # that gives different results than the original configuration
84  # when run under cmsRun. One should view the converted configuration
85  # as a thing which needs to be validated. It is possible for there
86  # to be pathologies that cannot be resolved by modifying only the
87  # python configuration and may require redesign inside the C++ code
88  # of the modules and redesign of the logic. For example,
89  # an EDAnalyzer might try to get a product and check if the
90  # returned handle isValid. Then it could behave differently
91  # depending on whether or not the product was present.
92  # The behavior when the product is not present could
93  # be meaningful and important. In the unscheduled case,
94  # the EDProducer will always run and the product could
95  # always be there.
96 
97  # Remove all modules not on Paths or EndPaths
98  proc.prune()
99 
100  # Turn on unschedule mode
101  if not hasattr(proc,'options'):
102  proc.options = cms.untracked.PSet()
103  proc.options.allowUnscheduled = cms.untracked.bool(True)
104 
105  l = proc.paths
106  droppedPaths =[]
107  #have to get them now since switching them after the
108  # paths have been changed gives null labels
109  if proc.schedule:
110  pathNamesInScheduled = [p.label_() for p in proc.schedule]
111  else:
112  pathNamesInScheduled = False
113 
114  # Look for EDProducers that depend on an EDFilter, either
115  # directly or indirectly through another EDProducer. These
116  # EDProducers must stay in a path and run scheduled. The
117  # first loop will find all the direct dependencies, but then
118  # the loop must be repeated until no more dependencies
119  # are found in order to find the indirect dependencies.
120  # Note that here we are assuming that if a module
121  # has a configuration parameter that is an InputTag, then
122  # it depends on the module with the same module label as in
123  # the InputTag. In addition, there are a number of special
124  # cases where we have identified specific types of modules
125  # which use particular string parameters like InputTag
126  # module labels. And so we have some special code to
127  # handle those cases also. If there are other cases where
128  # the module gets things without using InputTags, then this
129  # conversion script can fail, which might result in ProductNotFound
130  # exceptions or other problems when the converted configuration is run.
131 
132  # The dictionary keys are are the types of the EDProducers
133  # The dictionary values are lists of parameter names that are strings
134  # used like InputTags.
135  knownStringInputLabels = {}
136  knownStringInputLabels['KProd'] = ['xSrc'] # a fake entry for a unit test below
137  knownStringInputLabels['SeedGeneratorFromRegionHitsEDProducer'] = ['vertexSrc']
138  knownStringInputLabels['PixelTrackProducer'] = ['vertexSrc']
139  knownStringInputLabels['SimpleTrackListMerger'] = ['TrackProducer1', 'TrackProducer2']
140 
141  allEDFilters = set(proc.filters_().keys())
142  allEDProducers = set(proc.producers_().keys())
143 
144  dependentProducers = set()
145  firstPass = True
146 
147  while True :
148 
149  dependentsFoundThisPass = False
150 
151  for producer in allEDProducers :
152  if producer not in dependentProducers :
153  iModule = getattr(proc,producer)
154 
155  stringInputLabels = []
156  moduleType = iModule.type_()
157  if moduleType in knownStringInputLabels :
158  stringInputLabels = knownStringInputLabels[moduleType]
159 
160  inputTagLabels = InputTagLabelSet()
161  traverseInputTags(getattr(proc,producer), inputTagLabels, stringInputLabels)
162 
163  if firstPass :
164  if not inputTagLabels.labels.isdisjoint(allEDFilters) :
165  dependentProducers.add(producer)
166  dependentsFoundThisPass = True
167 
168  if not inputTagLabels.labels.isdisjoint(dependentProducers) :
169  dependentProducers.add(producer)
170  dependentsFoundThisPass = True
171 
172  if not dependentsFoundThisPass :
173  break
174  firstPass = False
175 
176  # Loop over paths
177  # On each path we drop EDProducers except we
178  # keep the EDProducers that depend on EDFilters
179  for pName,p in l.iteritems():
180  nodes = []
181  v = cms.ModuleNodeVisitor(nodes)
182  p.visit(v)
183  names = [node.label_() for node in nodes]
184  remaining =[]
185 
186  for n in names:
187  if not isinstance(getattr(proc,n), cms.EDProducer):
188  remaining.append(n)
189  else :
190  if n in dependentProducers :
191  remaining.append(n)
192 
193  if remaining:
194  p=getattr(proc,remaining[0])
195  for m in remaining[1:]:
196  p+=getattr(proc,m)
197  setattr(proc,pName,cms.Path(p))
198  # drop empty paths
199  else:
200  setattr(proc,pName,cms.Path())
201 
202  # If there is a schedule then it needs to point at
203  # the new Path objects
204  if proc.schedule:
205  proc.schedule = cms.Schedule([getattr(proc,p) for p in pathNamesInScheduled])
206  return proc
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Utilities.convertToUnscheduled
def convertToUnscheduled
Definition: Utilities.py:69
def Utilities.getHltConfiguration (   menu,
  args 
)

Definition at line 13 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

13 
14 def getHltConfiguration(menu, **args):
15  args['menu'] = menu
16  args['fragment'] = False
17  options = _build_options(**args)
18 
19  hlt = _imp.new_module('hlt')
20  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
21  return hlt.process
22 
Utilities.getHltConfiguration
def getHltConfiguration
Definition: Utilities.py:13
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.loadHltConfiguration (   process,
  menu,
  args 
)

Definition at line 23 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

23 
24 def loadHltConfiguration(process, menu, **args):
25  args['menu'] = menu
26  args['fragment'] = True
27  options = _build_options(**args)
28 
29  hlt = _imp.new_module('hlt')
30  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
31  process.extend( hlt )
32 
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
Utilities.loadHltConfiguration
def loadHltConfiguration
Definition: Utilities.py:23
def Utilities.removeModulesNotOnAPathExcluding (   process,
  keepList = () 
) 
Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
and remove them if they do not appear on a Path or EndPath.  One can optionally pass in
a list of modules which even if they are not on a Path or EndPath you wish to have stay 
in the configuration [useful for unscheduled execution].

Definition at line 1 of file Utilities.py.

1 def removeModulesNotOnAPathExcluding( process, keepList=() ):
2  """Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
3  and remove them if they do not appear on a Path or EndPath. One can optionally pass in
4  a list of modules which even if they are not on a Path or EndPath you wish to have stay
5  in the configuration [useful for unscheduled execution].
6  """
7  allMods=set((x for x in process.producers_().iterkeys()))
8  allMods.update((x for x in process.filters_().iterkeys()))
9  allMods.update((x for x in process.analyzers_().iterkeys()))
10  allMods.update((x for x in process.outputModules_().iterkeys()))
11 
12  modulesOnPaths = set()
13  for p in process.paths_():
14  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
15  for p in process.endpaths_():
16  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
17 
18  notOnPaths = allMods.difference(modulesOnPaths)
19 
20  keepModuleNames = set( (x.label_() for x in keepList) )
21 
22  getRidOf = notOnPaths.difference(keepModuleNames)
23 
24  for n in getRidOf:
25  delattr(process,n)
26 
Utilities.removeModulesNotOnAPathExcluding
def removeModulesNotOnAPathExcluding
Definition: Utilities.py:1
def Utilities.traverseInputTags (   pset,
  visitor,
  stringInputLabels 
)

Definition at line 44 of file Utilities.py.

References relativeConstraints.keys.

Referenced by convertToUnscheduled().

44 
45 def traverseInputTags(pset, visitor, stringInputLabels):
46  from FWCore.ParameterSet.Mixins import _Parameterizable
47  from FWCore.ParameterSet.Types import VPSet, VInputTag, InputTag, string
48 
49  # Loop over parameters in a PSet
50  for name in pset.parameters_().keys() :
51  value = getattr(pset,name)
52  # Recursive calls into a PSet in a PSet
53  if isinstance(value, _Parameterizable) :
54  traverseInputTags(value, visitor, stringInputLabels)
55  # Recursive calls into PSets in a VPSet
56  elif isinstance(value, VPSet) :
57  for (n, psetInVPSet) in enumerate(value):
58  traverseInputTags(psetInVPSet, visitor, stringInputLabels)
59  # Get the labels from a VInputTag
60  elif isinstance(value, VInputTag) :
61  visitor(value)
62  # Get the label from an InputTag
63  elif isinstance(value, InputTag) :
64  visitor(value)
65  # Known module labels in string objects
66  elif stringInputLabels and isinstance(value, string) and name in stringInputLabels and value.value() :
67  visitor.labels.add(value.value())
68  #ignore the rest
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Generated for CMSSW Reference Manual by doxygen 1.8.5