CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Namespaces | Classes | Functions
Utilities Namespace Reference

Namespaces

 ReleaseScripts
 

Classes

class  InputTagLabelSet
 
class  TestModuleCommand
 

Functions

def _build_options
 
def cleanUnscheduled
 
def convertToUnscheduled
 
def getHltConfiguration
 
def loadHltConfiguration
 
def removeModulesNotOnAPathExcluding
 
def traverseInputTags
 

Function Documentation

def Utilities._build_options (   args)
private

Definition at line 6 of file Utilities.py.

Referenced by getHltConfiguration(), and loadHltConfiguration().

6 
7 def _build_options(**args):
8  options = _options.HLTProcessOptions()
9  for key, val in args.iteritems():
10  setattr(options, key, val)
11  return options
12 
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.cleanUnscheduled (   proc)

Definition at line 107 of file Utilities.py.

References relativeConstraints.keys, and traverseInputTags().

Referenced by convertToUnscheduled().

107 
108 def cleanUnscheduled(proc):
109  import FWCore.ParameterSet.Config as cms
110  l = proc.paths
111  droppedPaths =[]
112  #have to get them now since switching them after the
113  # paths have been changed gives null labels
114  if proc.schedule:
115  pathNamesInScheduled = [p.label_() for p in proc.schedule]
116  else:
117  pathNamesInScheduled = False
118 
119  # Look for EDProducers that depend on an EDFilter, either
120  # directly or indirectly through another EDProducer. These
121  # EDProducers must stay in a path and run scheduled. The
122  # first loop will find all the direct dependencies, but then
123  # the loop must be repeated until no more dependencies
124  # are found in order to find the indirect dependencies.
125  # Note that here we are assuming that if a module
126  # has a configuration parameter that is an InputTag, then
127  # it depends on the module with the same module label as in
128  # the InputTag. In addition, there are a number of special
129  # cases where we have identified specific types of modules
130  # which use particular string parameters like InputTag
131  # module labels. And so we have some special code to
132  # handle those cases also. If there are other cases where
133  # the module gets things without using InputTags, then this
134  # conversion script can fail, which might result in ProductNotFound
135  # exceptions or other problems when the converted configuration is run.
136 
137  # The dictionary keys are are the types of the EDProducers
138  # The dictionary values are lists of parameter names that are strings
139  # used like InputTags.
140  knownStringInputLabels = {}
141  knownStringInputLabels['KProd'] = ['xSrc'] # a fake entry for a unit test below
142  knownStringInputLabels['SeedGeneratorFromRegionHitsEDProducer'] = ['vertexSrc']
143  knownStringInputLabels['PixelTrackProducer'] = ['vertexSrc']
144  knownStringInputLabels['PixelTracksProducer'] = ['vertexSrc']
145  knownStringInputLabels['SimpleTrackListMerger'] = ['TrackProducer1', 'TrackProducer2']
146 
147  allEDFilters = set(proc.filters_().keys())
148  allEDProducers = set(proc.producers_().keys())
149 
150  dependentProducers = set()
151  firstPass = True
152 
153 
154  def getUnqualifiedName(name):
155  if name[0] in set(['!','-']):
156  return name[1:]
157  return name
158 
159  def getQualifiedModule(name,proc):
160  unqual_name = getUnqualifiedName(name)
161  p=getattr(proc,unqual_name)
162  if unqual_name != name:
163  if name[0] == '!':
164  p = ~p
165  elif name[0] == '-':
166  p = cms.ignore(p)
167  return p
168 
169  while True :
170 
171  dependentsFoundThisPass = False
172 
173  for producer in allEDProducers :
174  if producer not in dependentProducers :
175  iModule = getattr(proc,producer)
176 
177  stringInputLabels = []
178  moduleType = iModule.type_()
179  if moduleType in knownStringInputLabels :
180  stringInputLabels = knownStringInputLabels[moduleType]
181 
182  inputTagLabels = InputTagLabelSet()
183  traverseInputTags(getattr(proc,producer), inputTagLabels, stringInputLabels)
184 
185  if firstPass :
186  if not inputTagLabels.labels.isdisjoint(allEDFilters) :
187  dependentProducers.add(producer)
188  dependentsFoundThisPass = True
189 
190  if not inputTagLabels.labels.isdisjoint(dependentProducers) :
191  dependentProducers.add(producer)
192  dependentsFoundThisPass = True
193 
194  if not dependentsFoundThisPass :
195  break
196  firstPass = False
197 
198  # Loop over paths
199  # On each path we drop EDProducers except we
200  # keep the EDProducers that depend on EDFilters
201  for pName,p in l.iteritems():
202  qualified_names = []
203  v = cms.DecoratedNodeNameVisitor(qualified_names)
204  p.visit(v)
205  remaining =[]
206 
207  for n in qualified_names:
208  unqual_name = getUnqualifiedName(n)
209  if not isinstance(getattr(proc,unqual_name), cms.EDProducer):
210  remaining.append(n)
211  else :
212  if unqual_name in dependentProducers :
213  remaining.append(n)
214 
215  if remaining:
216  p = getQualifiedModule(remaining[0],proc)
217  for m in remaining[1:]:
218  p+=getQualifiedModule(m,proc)
219  setattr(proc,pName,cms.Path(p))
220  # drop empty paths
221  else:
222  setattr(proc,pName,cms.Path())
223 
224  # If there is a schedule then it needs to point at
225  # the new Path objects
226  if proc.schedule:
227  proc.schedule = cms.Schedule([getattr(proc,p) for p in pathNamesInScheduled])
228  return proc
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Utilities.cleanUnscheduled
def cleanUnscheduled
Definition: Utilities.py:107
def Utilities.convertToUnscheduled (   proc)

Definition at line 69 of file Utilities.py.

References cleanUnscheduled().

Referenced by Utilities.TestModuleCommand.testNoSchedule(), and Utilities.TestModuleCommand.testWithSchedule().

69 
70 def convertToUnscheduled(proc):
71  import FWCore.ParameterSet.Config as cms
72  """Given a 'Process', convert the python configuration from scheduled execution to unscheduled. This is done by
73  1. Removing all modules not on Paths or EndPaths
74  2. Pulling EDProducers not dependent on EDFilters off of all Paths
75  3. Dropping any Paths which are now empty
76  4. Fixing up the Schedule if needed
77  """
78  # Warning: It is not always possible to convert a configuration
79  # where EDProducers are all run on Paths to an unscheduled
80  # configuration by modifying only the python configuration.
81  # There is more than one kind of pathological condition
82  # that can cause this conversion to produce a configuration
83  # that gives different results than the original configuration
84  # when run under cmsRun. One should view the converted configuration
85  # as a thing which needs to be validated. It is possible for there
86  # to be pathologies that cannot be resolved by modifying only the
87  # python configuration and may require redesign inside the C++ code
88  # of the modules and redesign of the logic. For example,
89  # an EDAnalyzer might try to get a product and check if the
90  # returned handle isValid. Then it could behave differently
91  # depending on whether or not the product was present.
92  # The behavior when the product is not present could
93  # be meaningful and important. In the unscheduled case,
94  # the EDProducer will always run and the product could
95  # always be there.
96 
97  # Remove all modules not on Paths or EndPaths
98  proc.prune()
99 
100  # Turn on unschedule mode
101  if not hasattr(proc,'options'):
102  proc.options = cms.untracked.PSet()
103  proc.options.allowUnscheduled = cms.untracked.bool(True)
104 
105  proc=cleanUnscheduled(proc)
106  return proc
Utilities.convertToUnscheduled
def convertToUnscheduled
Definition: Utilities.py:69
Utilities.cleanUnscheduled
def cleanUnscheduled
Definition: Utilities.py:107
def Utilities.getHltConfiguration (   menu,
  args 
)

Definition at line 13 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

13 
14 def getHltConfiguration(menu, **args):
15  args['menu'] = menu
16  args['fragment'] = False
17  options = _build_options(**args)
18 
19  hlt = _imp.new_module('hlt')
20  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
21  return hlt.process
22 
Utilities.getHltConfiguration
def getHltConfiguration
Definition: Utilities.py:13
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.loadHltConfiguration (   process,
  menu,
  args 
)

Definition at line 23 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

23 
24 def loadHltConfiguration(process, menu, **args):
25  args['menu'] = menu
26  args['fragment'] = True
27  options = _build_options(**args)
28 
29  hlt = _imp.new_module('hlt')
30  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
31  process.extend( hlt )
32 
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
Utilities.loadHltConfiguration
def loadHltConfiguration
Definition: Utilities.py:23
def Utilities.removeModulesNotOnAPathExcluding (   process,
  keepList = () 
) 
Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
and remove them if they do not appear on a Path or EndPath.  One can optionally pass in
a list of modules which even if they are not on a Path or EndPath you wish to have stay 
in the configuration [useful for unscheduled execution].

Definition at line 1 of file Utilities.py.

1 def removeModulesNotOnAPathExcluding( process, keepList=() ):
2  """Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
3  and remove them if they do not appear on a Path or EndPath. One can optionally pass in
4  a list of modules which even if they are not on a Path or EndPath you wish to have stay
5  in the configuration [useful for unscheduled execution].
6  """
7  allMods=set((x for x in process.producers_().iterkeys()))
8  allMods.update((x for x in process.filters_().iterkeys()))
9  allMods.update((x for x in process.analyzers_().iterkeys()))
10  allMods.update((x for x in process.outputModules_().iterkeys()))
11 
12  modulesOnPaths = set()
13  for p in process.paths_():
14  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
15  for p in process.endpaths_():
16  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
17 
18  notOnPaths = allMods.difference(modulesOnPaths)
19 
20  keepModuleNames = set( (x.label_() for x in keepList) )
21 
22  getRidOf = notOnPaths.difference(keepModuleNames)
23 
24  for n in getRidOf:
25  delattr(process,n)
26 
Utilities.removeModulesNotOnAPathExcluding
def removeModulesNotOnAPathExcluding
Definition: Utilities.py:1
def Utilities.traverseInputTags (   pset,
  visitor,
  stringInputLabels 
)

Definition at line 44 of file Utilities.py.

References relativeConstraints.keys.

Referenced by cleanUnscheduled().

44 
45 def traverseInputTags(pset, visitor, stringInputLabels):
46  from FWCore.ParameterSet.Mixins import _Parameterizable
47  from FWCore.ParameterSet.Types import VPSet, VInputTag, InputTag, string
48 
49  # Loop over parameters in a PSet
50  for name in pset.parameters_().keys() :
51  value = getattr(pset,name)
52  # Recursive calls into a PSet in a PSet
53  if isinstance(value, _Parameterizable) :
54  traverseInputTags(value, visitor, stringInputLabels)
55  # Recursive calls into PSets in a VPSet
56  elif isinstance(value, VPSet) :
57  for (n, psetInVPSet) in enumerate(value):
58  traverseInputTags(psetInVPSet, visitor, stringInputLabels)
59  # Get the labels from a VInputTag
60  elif isinstance(value, VInputTag) :
61  visitor(value)
62  # Get the label from an InputTag
63  elif isinstance(value, InputTag) :
64  visitor(value)
65  # Known module labels in string objects
66  elif stringInputLabels and isinstance(value, string) and name in stringInputLabels and value.value() :
67  visitor.labels.add(value.value())
68  #ignore the rest
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Generated for CMSSW Reference Manual by doxygen 1.8.5