CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Namespaces | Classes | Functions
Utilities Namespace Reference

Namespaces

 ReleaseScripts
 

Classes

class  InputTagLabelSet
 
class  TestModuleCommand
 

Functions

def _build_options
 
def cleanUnscheduled
 
def convertToUnscheduled
 
def getHltConfiguration
 
def loadHltConfiguration
 
def removeModulesNotOnAPathExcluding
 
def traverseInputTags
 

Function Documentation

def Utilities._build_options (   args)
private

Definition at line 6 of file Utilities.py.

Referenced by getHltConfiguration(), and loadHltConfiguration().

6 
7 def _build_options(**args):
8  options = _options.HLTProcessOptions()
9  for key, val in args.iteritems():
10  setattr(options, key, val)
11  return options
12 
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.cleanUnscheduled (   proc)

Definition at line 107 of file Utilities.py.

References cmsPerfStripChart.dict, relativeConstraints.keys, and traverseInputTags().

Referenced by convertToUnscheduled().

107 
108 def cleanUnscheduled(proc):
109  import FWCore.ParameterSet.Config as cms
110  l = dict(proc.paths)
111  l.update( dict(proc.endpaths) )
112  droppedPaths =[]
113  #have to get them now since switching them after the
114  # paths have been changed gives null labels
115  if proc.schedule:
116  pathNamesInScheduled = [p.label_() for p in proc.schedule]
117  else:
118  pathNamesInScheduled = False
119 
120  # Look for EDProducers that depend on an EDFilter, either
121  # directly or indirectly through another EDProducer. These
122  # EDProducers must stay in a path and run scheduled. The
123  # first loop will find all the direct dependencies, but then
124  # the loop must be repeated until no more dependencies
125  # are found in order to find the indirect dependencies.
126  # Note that here we are assuming that if a module
127  # has a configuration parameter that is an InputTag, then
128  # it depends on the module with the same module label as in
129  # the InputTag. In addition, there are a number of special
130  # cases where we have identified specific types of modules
131  # which use particular string parameters like InputTag
132  # module labels. And so we have some special code to
133  # handle those cases also. If there are other cases where
134  # the module gets things without using InputTags, then this
135  # conversion script can fail, which might result in ProductNotFound
136  # exceptions or other problems when the converted configuration is run.
137 
138  # The dictionary keys are are the types of the EDProducers
139  # The dictionary values are lists of parameter names that are strings
140  # used like InputTags.
141  knownStringInputLabels = {}
142  knownStringInputLabels['KProd'] = ['xSrc'] # a fake entry for a unit test below
143  knownStringInputLabels['SeedGeneratorFromRegionHitsEDProducer'] = ['vertexSrc']
144  knownStringInputLabels['PixelTrackProducer'] = ['vertexSrc']
145  knownStringInputLabels['PixelTracksProducer'] = ['vertexSrc']
146  knownStringInputLabels['SimpleTrackListMerger'] = ['TrackProducer1', 'TrackProducer2']
147 
148  allEDFilters = set(proc.filters_().keys())
149  allEDProducers = set(proc.producers_().keys())
150 
151  dependentProducers = set()
152  firstPass = True
153 
154 
155  def getUnqualifiedName(name):
156  if name[0] in set(['!','-']):
157  return name[1:]
158  return name
159 
160  def getQualifiedModule(name,proc):
161  unqual_name = getUnqualifiedName(name)
162  p=getattr(proc,unqual_name)
163  if unqual_name != name:
164  if name[0] == '!':
165  p = ~p
166  elif name[0] == '-':
167  p = cms.ignore(p)
168  return p
169 
170  while True :
171 
172  dependentsFoundThisPass = False
173 
174  for producer in allEDProducers :
175  if producer not in dependentProducers :
176  iModule = getattr(proc,producer)
177 
178  stringInputLabels = []
179  moduleType = iModule.type_()
180  if moduleType in knownStringInputLabels :
181  stringInputLabels = knownStringInputLabels[moduleType]
182 
183  inputTagLabels = InputTagLabelSet()
184  traverseInputTags(getattr(proc,producer), inputTagLabels, stringInputLabels)
185 
186  if firstPass :
187  if not inputTagLabels.labels.isdisjoint(allEDFilters) :
188  dependentProducers.add(producer)
189  dependentsFoundThisPass = True
190 
191  if not inputTagLabels.labels.isdisjoint(dependentProducers) :
192  dependentProducers.add(producer)
193  dependentsFoundThisPass = True
194 
195  if not dependentsFoundThisPass :
196  break
197  firstPass = False
198 
199  # Loop over paths
200  # On each path we drop EDProducers except we
201  # keep the EDProducers that depend on EDFilters
202  for pName,p in l.iteritems():
203  qualified_names = []
204  v = cms.DecoratedNodeNameVisitor(qualified_names)
205  p.visit(v)
206  remaining =[]
207 
208  for n in qualified_names:
209  unqual_name = getUnqualifiedName(n)
210  if not isinstance(getattr(proc,unqual_name), cms.EDProducer):
211  remaining.append(n)
212  else :
213  if unqual_name in dependentProducers :
214  remaining.append(n)
215 
216  if remaining:
217  p = getQualifiedModule(remaining[0],proc)
218  for m in remaining[1:]:
219  p+=getQualifiedModule(m,proc)
220  setattr(proc,pName,type(getattr(proc,pName))(p))
221  # drop empty paths
222  else:
223  setattr(proc,pName,type(getattr(proc,pName))())
224 
225  # If there is a schedule then it needs to point at
226  # the new Path objects
227  if proc.schedule:
228  proc.schedule = cms.Schedule([getattr(proc,p) for p in pathNamesInScheduled])
229  return proc
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Utilities.cleanUnscheduled
def cleanUnscheduled
Definition: Utilities.py:107
def Utilities.convertToUnscheduled (   proc)

Definition at line 69 of file Utilities.py.

References cleanUnscheduled().

Referenced by Utilities.TestModuleCommand.testNoSchedule(), and Utilities.TestModuleCommand.testWithSchedule().

69 
70 def convertToUnscheduled(proc):
71  import FWCore.ParameterSet.Config as cms
72  """Given a 'Process', convert the python configuration from scheduled execution to unscheduled. This is done by
73  1. Removing all modules not on Paths or EndPaths
74  2. Pulling EDProducers not dependent on EDFilters off of all Paths
75  3. Dropping any Paths which are now empty
76  4. Fixing up the Schedule if needed
77  """
78  # Warning: It is not always possible to convert a configuration
79  # where EDProducers are all run on Paths to an unscheduled
80  # configuration by modifying only the python configuration.
81  # There is more than one kind of pathological condition
82  # that can cause this conversion to produce a configuration
83  # that gives different results than the original configuration
84  # when run under cmsRun. One should view the converted configuration
85  # as a thing which needs to be validated. It is possible for there
86  # to be pathologies that cannot be resolved by modifying only the
87  # python configuration and may require redesign inside the C++ code
88  # of the modules and redesign of the logic. For example,
89  # an EDAnalyzer might try to get a product and check if the
90  # returned handle isValid. Then it could behave differently
91  # depending on whether or not the product was present.
92  # The behavior when the product is not present could
93  # be meaningful and important. In the unscheduled case,
94  # the EDProducer will always run and the product could
95  # always be there.
96 
97  # Remove all modules not on Paths or EndPaths
98  proc.prune()
99 
100  # Turn on unschedule mode
101  if not hasattr(proc,'options'):
102  proc.options = cms.untracked.PSet()
103  proc.options.allowUnscheduled = cms.untracked.bool(True)
104 
105  proc=cleanUnscheduled(proc)
106  return proc
Utilities.convertToUnscheduled
def convertToUnscheduled
Definition: Utilities.py:69
Utilities.cleanUnscheduled
def cleanUnscheduled
Definition: Utilities.py:107
def Utilities.getHltConfiguration (   menu,
  args 
)

Definition at line 13 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

13 
14 def getHltConfiguration(menu, **args):
15  args['menu'] = menu
16  args['fragment'] = False
17  options = _build_options(**args)
18 
19  hlt = _imp.new_module('hlt')
20  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
21  return hlt.process
22 
Utilities.getHltConfiguration
def getHltConfiguration
Definition: Utilities.py:13
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
def Utilities.loadHltConfiguration (   process,
  menu,
  args 
)

Definition at line 23 of file Utilities.py.

References _build_options(), and hcal_timing_source_file_cfg.dump.

23 
24 def loadHltConfiguration(process, menu, **args):
25  args['menu'] = menu
26  args['fragment'] = True
27  options = _build_options(**args)
28 
29  hlt = _imp.new_module('hlt')
30  exec _confdb.HLTProcess(options).dump() in globals(), hlt.__dict__
31  process.extend( hlt )
32 
hcal_timing_source_file_cfg.dump
tuple dump
Definition: hcal_timing_source_file_cfg.py:72
Utilities._build_options
def _build_options
Definition: Utilities.py:6
Utilities.loadHltConfiguration
def loadHltConfiguration
Definition: Utilities.py:23
def Utilities.removeModulesNotOnAPathExcluding (   process,
  keepList = () 
) 
Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
and remove them if they do not appear on a Path or EndPath.  One can optionally pass in
a list of modules which even if they are not on a Path or EndPath you wish to have stay 
in the configuration [useful for unscheduled execution].

Definition at line 1 of file Utilities.py.

1 def removeModulesNotOnAPathExcluding( process, keepList=() ):
2  """Given a 'process', find all modules (EDProducers,EDFilters,EDAnalyzers,OutputModules)
3  and remove them if they do not appear on a Path or EndPath. One can optionally pass in
4  a list of modules which even if they are not on a Path or EndPath you wish to have stay
5  in the configuration [useful for unscheduled execution].
6  """
7  allMods=set((x for x in process.producers_().iterkeys()))
8  allMods.update((x for x in process.filters_().iterkeys()))
9  allMods.update((x for x in process.analyzers_().iterkeys()))
10  allMods.update((x for x in process.outputModules_().iterkeys()))
11 
12  modulesOnPaths = set()
13  for p in process.paths_():
14  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
15  for p in process.endpaths_():
16  modulesOnPaths.update( (x for x in getattr(process,p).moduleNames()))
17 
18  notOnPaths = allMods.difference(modulesOnPaths)
19 
20  keepModuleNames = set( (x.label_() for x in keepList) )
21 
22  getRidOf = notOnPaths.difference(keepModuleNames)
23 
24  for n in getRidOf:
25  delattr(process,n)
26 
Utilities.removeModulesNotOnAPathExcluding
def removeModulesNotOnAPathExcluding
Definition: Utilities.py:1
def Utilities.traverseInputTags (   pset,
  visitor,
  stringInputLabels 
)

Definition at line 44 of file Utilities.py.

References relativeConstraints.keys.

Referenced by cleanUnscheduled().

44 
45 def traverseInputTags(pset, visitor, stringInputLabels):
46  from FWCore.ParameterSet.Mixins import _Parameterizable
47  from FWCore.ParameterSet.Types import VPSet, VInputTag, InputTag, string
48 
49  # Loop over parameters in a PSet
50  for name in pset.parameters_().keys() :
51  value = getattr(pset,name)
52  # Recursive calls into a PSet in a PSet
53  if isinstance(value, _Parameterizable) :
54  traverseInputTags(value, visitor, stringInputLabels)
55  # Recursive calls into PSets in a VPSet
56  elif isinstance(value, VPSet) :
57  for (n, psetInVPSet) in enumerate(value):
58  traverseInputTags(psetInVPSet, visitor, stringInputLabels)
59  # Get the labels from a VInputTag
60  elif isinstance(value, VInputTag) :
61  visitor(value)
62  # Get the label from an InputTag
63  elif isinstance(value, InputTag) :
64  visitor(value)
65  # Known module labels in string objects
66  elif stringInputLabels and isinstance(value, string) and name in stringInputLabels and value.value() :
67  visitor.labels.add(value.value())
68  #ignore the rest
Utilities.traverseInputTags
def traverseInputTags
Definition: Utilities.py:44
Generated for CMSSW Reference Manual by doxygen 1.8.5