CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
confdb.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 from __future__ import absolute_import
4 import sys
5 import re
6 import os
7 from .pipe import pipe as _pipe
8 from .options import globalTag
9 from itertools import islice
10 
11 def splitter(iterator, n):
12  i = iterator.__iter__()
13  while True:
14  l = list(islice(i, n))
15  if l:
16  yield l
17  else:
18  break
19 
20 
21 class HLTProcess(object):
22 
23  def __init__(self, configuration):
24  self.config = configuration
25  self.data = None
26  self.source = []
27  self.parent = []
28 
29  self.options = {
30  'essources' : [],
31  'esmodules' : [],
32  'modules' : [],
33  'sequences' : [],
34  'services' : [],
35  'paths' : [],
36  'psets' : [],
37  'blocks' : [],
38  }
39 
40  self.labels = {}
41  if self.config.fragment:
42  self.labels['process'] = 'fragment'
43  self.labels['dict'] = 'fragment.__dict__'
44  else:
45  self.labels['process'] = 'process'
46  self.labels['dict'] = 'process.__dict__'
47 
48  if self.config.prescale and (self.config.prescale.lower() != 'none'):
49  self.labels['prescale'] = self.config.prescale
50 
51  # get the configuration from ConfdB
52  from .confdbOfflineConverter import OfflineConverter
53  self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database, proxy = self.config.proxy, proxyHost = self.config.proxy_host, proxyPort = self.config.proxy_port)
54  self.buildPathList()
55  self.buildOptions()
58  self.customize()
59 
61  if not self.config.setup:
62  return
63  ## if --setup is a python file, use directly that file as setup_cff.py
64  if ".py" in self.config.setup:
65  self.config.setupFile = self.config.setup.split(".py")[0]
66  return
67  args = ['--configName', self.config.setup ]
68  args.append('--noedsources')
69  args.append('--nopaths')
70  for key, vals in self.options.items():
71  if vals:
72  args.extend(('--'+key, ','.join(vals)))
73  args.append('--cff')
74  data, err = self.converter.query( *args )
75  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
76  sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
77  sys.stderr.write(err + "\n\n")
78  sys.exit(1)
79  self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
80  outfile = open(self.config.setupFile+".py","w+")
81  outfile.write("# This file is automatically generated by hltGetConfiguration.\n" + data)
82 
84  if self.config.menu.run:
85  args = ['--runNumber', self.config.menu.run]
86  else:
87  args = ['--configName', self.config.menu.name ]
88  if not self.config.hilton:
89  # keep the original Source when running on Hilton
90  args.append('--noedsources')
91  for key, vals in self.options.items():
92  if vals:
93  args.extend(('--'+key, ','.join(vals)))
94  data, err = self.converter.query( *args )
95  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
96  sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
97  sys.stderr.write(err + "\n\n")
98  sys.exit(1)
99  self.data = data
100 
101  def getPathList(self):
102  if self.config.menu.run:
103  args = ['--runNumber', self.config.menu.run]
104  else:
105  args = ['--configName', self.config.menu.name]
106  args.extend( (
107  '--cff',
108  '--noedsources',
109  '--noes',
110  '--noservices',
111  '--nosequences',
112  '--nomodules'
113  ) )
114 
115  data, err = self.converter.query( *args )
116  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
117  sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
118  sys.stderr.write(err + "\n\n")
119  sys.exit(1)
120  filter = re.compile(r' *= *cms.(End)?Path.*')
121  paths = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
122  return paths
123 
124 
125  @staticmethod
126  def expandWildcards(globs, collection):
127  # expand a list of unix-style wildcards matching a given collection
128  # wildcards with no matches are silently discarded
129  matches = []
130  for glob in globs:
131  negate = ''
132  if glob[0] == '-':
133  negate = '-'
134  glob = glob[1:]
135  # translate a unix-style glob expression into a regular expression
136  filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
137  matches.extend( negate + element for element in collection if filter.match(element) )
138  return matches
139 
140 
141  @staticmethod
142  def consolidateNegativeList(elements):
143  # consolidate a list of path exclusions and re-inclusions
144  # the result is the list of paths to be removed from the dump
145  result = set()
146  for element in elements:
147  if element[0] == '-':
148  result.add( element )
149  else:
150  result.discard( '-' + element )
151  return sorted( element for element in result )
152 
153  @staticmethod
154  def consolidatePositiveList(elements):
155  # consolidate a list of path selection and re-exclusions
156  # the result is the list of paths to be included in the dump
157  result = set()
158  for element in elements:
159  if element[0] == '-':
160  result.discard( element[1:] )
161  else:
162  result.add( element )
163  return sorted( element for element in result )
164 
165 
166  # dump the final configuration
167  def dump(self):
168  self.data = self.data % self.labels
169  if self.config.fragment:
170  self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
171  self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
172  return self.data
173 
174 
175  # add specific customizations
176  def specificCustomize(self):
177  # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
178  if self.config.fragment:
179  self.data += """
180 # add specific customizations
181 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
182 fragment = customizeHLTforAll(fragment,"%s")
183 """ % (self.config.type)
184  elif self.config.hilton:
185  # do not apply the STORM-specific customisation
186  pass
187  else:
188  if self.config.type=="Fake":
189  prefix = "run1"
190  elif self.config.type in ("Fake1","Fake2","2018"):
191  prefix = "run2"
192  else:
193  prefix = "run3"
194  _gtData = "auto:"+prefix+"_hlt_"+self.config.type
195  _gtMc = "auto:"+prefix+"_mc_" +self.config.type
196  self.data += """
197 # add specific customizations
198 _customInfo = {}
199 _customInfo['menuType' ]= "%s"
200 _customInfo['globalTags']= {}
201 _customInfo['globalTags'][True ] = "%s"
202 _customInfo['globalTags'][False] = "%s"
203 _customInfo['inputFiles']={}
204 _customInfo['inputFiles'][True] = "file:RelVal_Raw_%s_DATA.root"
205 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
206 _customInfo['maxEvents' ]= %s
207 _customInfo['globalTag' ]= "%s"
208 _customInfo['inputFile' ]= %s
209 _customInfo['realData' ]= %s
210 
211 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
212 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
213 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
214 
215  self.data += """
216 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
217 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
218 """ % (self.config.type)
219 
220  # Eras-based customisations
221  self.data += """
222 # Eras-based customisations
223 from HLTrigger.Configuration.Eras import modifyHLTforEras
224 modifyHLTforEras(%(process)s)
225 """
226  # add the user-defined customization functions, if any
227  if self.config.customise:
228  self.data += "\n"
229  self.data += "#User-defined customization functions\n"
230  for customise in self.config.customise.split(","):
231  customiseValues = customise.split(".")
232  if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
233  if len(customiseValues)==1:
234  customiseValues.append("customise")
235  customiseValues[0] = customiseValues[0].replace("/",".")
236  self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
237  self.data += "process = "+customiseValues[1]+"(process)\n"
238 
239 
240  # customize the configuration according to the options
241  def customize(self):
242 
243  # adapt the source to the current scenario
244  if not self.config.fragment:
245  self.build_source()
246 
247  # manual override some parameters
248  if self.config.type in ('HIon', ):
249  if self.config.data:
250  if not self.config.fragment:
251  self._fix_parameter( type = 'InputTag', value = 'rawDataCollector', replace = 'rawDataRepacker')
252 
253  # if requested, remove the HLT prescales
254  self.fixPrescales()
255 
256  # if requested, override all ED/HLTfilters to always pass ("open" mode)
257  self.instrumentOpenMode()
258 
259  # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
261 
262  # if requested, instrument the self with the modules and EndPath needed for timing studies
263  self.instrumentTiming()
264 
265  # if requested, override the L1 self from the GlobalTag (Xml)
266  self.overrideL1MenuXml()
267 
268  # if requested, run the L1 emulator
269  self.runL1Emulator()
270 
271  # add process.load("setup_cff")
272  self.loadSetupCff()
273 
274  if self.config.fragment:
275  self.data += """
276 # dummify hltGetConditions in cff's
277 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
278  %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
279  result = cms.bool( True )
280  )
281  %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
282 """
283 
284  # fix the Scouting EndPaths
285  for path in self.all_paths:
286  match = re.match(r'(Scouting\w+)Output$', path)
287  if match:
288  module = 'hltOutput' + match.group(1)
289  self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
290  self.data = self.data.replace(' + process.'+module, '')
291 
292  else:
293 
294  # override the process name and adapt the relevant filters
295  self.overrideProcessName()
296 
297  # select specific Eras
298  self.addEras()
299 
300  # override the output modules to output root files
301  self.overrideOutput()
302 
303  # add global options
304  self.addGlobalOptions()
305 
306  # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
307  self.overrideGlobalTag()
308 
309  # request summary informations from the MessageLogger
310  self.updateMessageLogger()
311 
312  # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
313  self.instrumentDQM()
314 
315  # add specific customisations
316  self.specificCustomize()
317 
318 
319  def addGlobalOptions(self):
320  # add global options
321  self.data += """
322 # limit the number of events to be processed
323 %%(process)s.maxEvents = cms.untracked.PSet(
324  input = cms.untracked.int32( %d )
325 )
326 """ % self.config.events
327 
328  self.data += """
329 # enable TrigReport, TimeReport and MultiThreading
330 %(process)s.options = cms.untracked.PSet(
331  wantSummary = cms.untracked.bool( True ),
332  numberOfThreads = cms.untracked.uint32( 4 ),
333  numberOfStreams = cms.untracked.uint32( 0 ),
334 )
335 """
336 
337  def _fix_parameter(self, **args):
338  """arguments:
339  name: parameter name (optional)
340  type: parameter type (look for tracked and untracked variants)
341  value: original value
342  replace: replacement value
343  """
344  if 'name' in args:
345  self.data = re.sub(
346  r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
347  r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
348  self.data)
349  else:
350  self.data = re.sub(
351  r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
352  r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
353  self.data)
354 
355 
356  def fixPrescales(self):
357  # update the PrescaleService to match the new list of paths
358  if self.options['paths']:
359  if self.options['paths'][0][0] == '-':
360  # drop requested paths
361  for minuspath in self.options['paths']:
362  path = minuspath[1:]
363  self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
364  else:
365  # keep requested paths
366  for path in self.all_paths:
367  if path not in self.options['paths']:
368  self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
369 
370  if self.config.prescale and (self.config.prescale.lower() != 'none'):
371  # TO DO: check that the requested prescale column is valid
372  self.data += """
373 # force the use of a specific HLT prescale column
374 if 'PrescaleService' in %(dict)s:
375  %(process)s.PrescaleService.forceDefault = True
376  %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
377 """
378 
379 
381  if self.config.open:
382  # find all EDfilters
383  filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
384  re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
385  # remove existing 'cms.ignore' and '~' modifiers
386  self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
387  self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
388  # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
389  for some in splitter(filters, 1000):
390  re_filters = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
391  self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
392 
393 
395  if self.config.errortype:
396  # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
397  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
398  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
399  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
400 
401 
402  def overrideGlobalTag(self):
403  # overwrite GlobalTag
404  # the logic is:
405  # - if a GlobalTag is specified on the command line:
406  # - override the global tag
407  # - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
408  # - if a GlobalTag is NOT specified on the command line:
409  # - when running on data, do nothing, and keep the global tag in the menu
410  # - when running on mc, take the GT from the configuration.type
411 
412  # override the GlobalTag connection string and pfnPrefix
413 
414  # when running on MC, override the global tag even if not specified on the command line
415  if not self.config.data and not self.config.globaltag:
416  if self.config.type in globalTag:
417  self.config.globaltag = globalTag[self.config.type]
418  else:
419  self.config.globaltag = globalTag['GRun']
420 
421  # if requested, override the L1 menu from the GlobalTag
422  if self.config.l1.override:
423  self.config.l1.tag = self.config.l1.override
424  self.config.l1.record = 'L1TUtmTriggerMenuRcd'
425  self.config.l1.connect = ''
426  self.config.l1.label = ''
427  if not self.config.l1.snapshotTime:
428  self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
429  self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
430  else:
431  self.config.l1cond = None
432 
433  if self.config.globaltag or self.config.l1cond:
434  text = """
435 # override the GlobalTag, connection string and pfnPrefix
436 if 'GlobalTag' in %(dict)s:
437  from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
438  %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
439  if self.config.globaltag:
440  text += ", globaltag = %s" % repr(self.config.globaltag)
441  if self.config.l1cond:
442  text += ", conditions = %s" % repr(self.config.l1cond)
443  text += ")\n"
444  self.data += text
445 
446  def overrideL1MenuXml(self):
447  # if requested, override the GlobalTag's L1T menu from an Xml file
448  if self.config.l1Xml.XmlFile:
449  text = """
450 # override the GlobalTag's L1T menu from an Xml file
451 from HLTrigger.Configuration.CustomConfigs import L1XML
452 %%(process)s = L1XML(%%(process)s,"%s")
453 """ % (self.config.l1Xml.XmlFile)
454  self.data += text
455 
456  def runL1Emulator(self):
457  # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
458  if self.config.emulator:
459  text = """
460 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
461 from HLTrigger.Configuration.CustomConfigs import L1REPACK
462 %%(process)s = L1REPACK(%%(process)s,"%s")
463 """ % (self.config.emulator)
464  self.data += text
465 
466  def overrideOutput(self):
467  # if not runnign on Hilton, override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
468  # note for Run3 ShmStreamConsumer has been replaced with EvFOutputModule
469  # so we also do a replace there
470  if not self.config.hilton:
471  self.data = re.sub(
472  r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"(ShmStreamConsumer)" *,',
473  r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),',
474  self.data
475  )
476  self.data = re.sub(
477  r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"EvFOutputModule" *,\n use_compression = cms.untracked.bool\( True \),\n compression_algorithm = cms.untracked.string\( "ZLIB" \),\n compression_level = cms.untracked.int32\( 1 \),\n lumiSection_interval = cms.untracked.int32\( 0 \),\n(.+?),\n psetMap = cms.untracked.InputTag\( "hltPSetMap" \)\n',
478  r'\1hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),\n\3\n',
479  self.data,0,re.DOTALL
480  )
481 
482  if not self.config.fragment and self.config.output == 'minimal':
483  # add a single output to keep the TriggerResults and TriggerEvent
484  self.data += """
485 # add a single "keep *" output
486 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
487  fileName = cms.untracked.string( "output.root" ),
488  fastCloning = cms.untracked.bool( False ),
489  dataset = cms.untracked.PSet(
490  dataTier = cms.untracked.string( 'AOD' ),
491  filterName = cms.untracked.string( '' )
492  ),
493  outputCommands = cms.untracked.vstring( 'drop *',
494  'keep edmTriggerResults_*_*_*',
495  'keep triggerTriggerEvent_*_*_*',
496  'keep GlobalAlgBlkBXVector_*_*_*',
497  'keep GlobalExtBlkBXVector_*_*_*',
498  'keep l1tEGammaBXVector_*_EGamma_*',
499  'keep l1tEtSumBXVector_*_EtSum_*',
500  'keep l1tJetBXVector_*_Jet_*',
501  'keep l1tMuonBXVector_*_Muon_*',
502  'keep l1tTauBXVector_*_Tau_*',
503  )
504 )
505 %(process)s.MinimalOutput = cms.EndPath( %(process)s.hltOutputMinimal )
506 %(process)s.schedule.append( %(process)s.MinimalOutput )
507 """
508  elif not self.config.fragment and self.config.output == 'full':
509  # add a single "keep *" output
510  self.data += """
511 # add a single "keep *" output
512 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
513  fileName = cms.untracked.string( "output.root" ),
514  fastCloning = cms.untracked.bool( False ),
515  dataset = cms.untracked.PSet(
516  dataTier = cms.untracked.string( 'RECO' ),
517  filterName = cms.untracked.string( '' )
518  ),
519  outputCommands = cms.untracked.vstring( 'keep *' )
520 )
521 %(process)s.FullOutput = cms.EndPath( %(process)s.hltOutputFull )
522 %(process)s.schedule.append( %(process)s.FullOutput )
523 """
524 
525  # select specific Eras
526  def addEras(self):
527  if self.config.eras is None:
528  return
529  from Configuration.StandardSequences.Eras import eras
530  erasSplit = self.config.eras.split(',')
531  self.data = re.sub(r'process = cms.Process\( *"\w+"', '\n'.join(eras.pythonCfgLines[era] for era in erasSplit)+'\n\g<0>, '+', '.join(era for era in erasSplit), self.data)
532 
533  # select specific Eras
534  def loadSetupCff(self):
535  if self.config.setup is None:
536  return
537  processLine = self.data.find("\n",self.data.find("cms.Process"))
538  self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
539 
540  # override the process name and adapt the relevant filters
542  if self.config.name is None:
543  return
544 
545  # sanitise process name
546  self.config.name = self.config.name.replace("_","")
547  # override the process name
548  quote = '[\'\"]'
549  self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
550 
551  # when --setup option is used, remove possible errors from PrescaleService due to missing HLT paths.
552  if self.config.setup: self.data += """
553 # avoid PrescaleService error due to missing HLT paths
554 if 'PrescaleService' in process.__dict__:
555  for pset in reversed(process.PrescaleService.prescaleTable):
556  if not hasattr(process,pset.pathName.value()):
557  process.PrescaleService.prescaleTable.remove(pset)
558 """
559 
560 
562  # request summary informations from the MessageLogger
563  self.data += """
564 # show summaries from trigger analysers used at HLT
565 if 'MessageLogger' in %(dict)s:
566  %(process)s.MessageLogger.TriggerSummaryProducerAOD = cms.untracked.PSet()
567  %(process)s.MessageLogger.L1GtTrigReport = cms.untracked.PSet()
568  %(process)s.MessageLogger.L1TGlobalSummary = cms.untracked.PSet()
569  %(process)s.MessageLogger.HLTrigReport = cms.untracked.PSet()
570  %(process)s.MessageLogger.FastReport = cms.untracked.PSet()
571  %(process)s.MessageLogger.ThroughputService = cms.untracked.PSet()
572 """
573 
574 
575  def loadAdditionalConditions(self, comment, *conditions):
576  # load additional conditions
577  self.data += """
578 # %s
579 if 'GlobalTag' in %%(dict)s:
580 """ % comment
581  for condition in conditions:
582  self.data += """ %%(process)s.GlobalTag.toGet.append(
583  cms.PSet(
584  record = cms.string( '%(record)s' ),
585  tag = cms.string( '%(tag)s' ),
586  label = cms.untracked.string( '%(label)s' ),
587  )
588  )
589 """ % condition
590 
591 
592  def loadCffCommand(self, module):
593  # load a cfi or cff module
594  if self.config.fragment:
595  return 'from %s import *\n' % module
596  else:
597  return 'process.load( "%s" )\n' % module
598 
599  def loadCff(self, module):
600  self.data += self.loadCffCommand(module)
601 
602 
603  def overrideParameters(self, module, parameters):
604  # override a module's parameter if the module is present in the configuration
605  self.data += "if '%s' in %%(dict)s:\n" % module
606  for (parameter, value) in parameters:
607  self.data += " %%(process)s.%s.%s = %s\n" % (module, parameter, value)
608  self.data += "\n"
609 
610 
612  if label in self.data:
613  label_re = r'\b(process\.)?' + label
614  self.data = re.sub(r' *(\+|,) *' + label_re, '', self.data)
615  self.data = re.sub(label_re + r' *(\+|,) *', '', self.data)
616  self.data = re.sub(label_re, '', self.data)
617 
618 
619  def instrumentTiming(self):
620 
621  if self.config.timing:
622  self.data += """
623 # instrument the menu with the modules and EndPath needed for timing studies
624 """
625 
626  self.data += '\n# configure the FastTimerService\n'
627  self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
628 
629  self.data += """# print a text summary at the end of the job
630 %(process)s.FastTimerService.printEventSummary = False
631 %(process)s.FastTimerService.printRunSummary = False
632 %(process)s.FastTimerService.printJobSummary = True
633 
634 # enable DQM plots
635 %(process)s.FastTimerService.enableDQM = True
636 
637 # enable per-path DQM plots (starting with CMSSW 9.2.3-patch2)
638 %(process)s.FastTimerService.enableDQMbyPath = True
639 
640 # enable per-module DQM plots
641 %(process)s.FastTimerService.enableDQMbyModule = True
642 
643 # enable per-event DQM plots vs lumisection
644 %(process)s.FastTimerService.enableDQMbyLumiSection = True
645 %(process)s.FastTimerService.dqmLumiSectionsRange = 2500
646 
647 # set the time resolution of the DQM plots
648 %(process)s.FastTimerService.dqmTimeRange = 2000.
649 %(process)s.FastTimerService.dqmTimeResolution = 10.
650 %(process)s.FastTimerService.dqmPathTimeRange = 1000.
651 %(process)s.FastTimerService.dqmPathTimeResolution = 5.
652 %(process)s.FastTimerService.dqmModuleTimeRange = 200.
653 %(process)s.FastTimerService.dqmModuleTimeResolution = 1.
654 
655 # set the base DQM folder for the plots
656 %(process)s.FastTimerService.dqmPath = 'HLT/TimerService'
657 %(process)s.FastTimerService.enableDQMbyProcesses = False
658 """
659 
660 
661  def instrumentDQM(self):
662  if not self.config.hilton:
663  # remove any reference to the hltDQMFileSaver and hltDQMFileSaverPB:
664  # note the convert options remove the module itself,
665  # here we are just removing the references in paths, sequences, etc
666  self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaverPB')
667  self.removeElementFromSequencesTasksAndPaths('hltDQMFileSaver')
668 
669  # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
670  dqmstore = "\n# load the DQMStore and DQMRootOutputModule\n"
671  dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
672  dqmstore += """
673 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
674  fileName = cms.untracked.string("DQMIO.root")
675 )
676 """
677 
678  empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.EndPath\( *\).*')
679  other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.EndPath\()(.*)')
680  if empty_path.search(self.data):
681  # replace an empty DQMOutput path
682  self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n', self.data)
683  elif other_path.search(self.data):
684  # prepend the dqmOutput to the DQMOutput path
685  self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<3>', self.data)
686  else:
687  # create a new DQMOutput path with the dqmOutput module
688  self.data += dqmstore
689  self.data += '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n'
690  self.data += '%(process)s.schedule.append( %(process)s.DQMOutput )\n'
691 
692 
693  @staticmethod
694  def dumppaths(paths):
695  sys.stderr.write('Path selection:\n')
696  for path in paths:
697  sys.stderr.write('\t%s\n' % path)
698  sys.stderr.write('\n\n')
699 
700  def buildPathList(self):
701  self.all_paths = self.getPathList()
702 
703  if self.config.paths:
704  # no path list was requested, dump the full table, minus unsupported / unwanted paths
705  paths = self.config.paths.split(',')
706  else:
707  # dump only the requested paths, plus the eventual output endpaths
708  paths = []
709 
710  # 'none' should remove all outputs
711  # 'dqm' should remove all outputs but DQMHistograms
712  # 'minimal' should remove all outputs but DQMHistograms, and add a single output module to keep the TriggerResults and TriggerEvent
713  # 'full' should remove all outputs but DQMHistograms, and add a single output module to "keep *"
714  # See also the `overrideOutput` method
715  if self.config.fragment or self.config.output in ('none', ):
716  if self.config.paths:
717  # keep only the Paths and EndPaths requested explicitly
718  pass
719  else:
720  # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring and DQMHistograms
721  paths.append( "-*Output" )
722  paths.append( "-RatesMonitoring")
723  paths.append( "-DQMHistograms")
724  if self.config.fragment: paths.append( "Scouting*Output" )
725 
726  elif self.config.output in ('dqm', 'minimal', 'full'):
727  if self.config.paths:
728  # keep only the Paths and EndPaths requested explicitly, and the DQMHistograms
729  paths.append( "DQMHistograms" )
730  else:
731  # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring
732  paths.append( "-*Output" )
733  paths.append( "-RatesMonitoring")
734  if self.config.fragment: paths.append( "Scouting*Output" )
735 
736  else:
737  if self.config.paths:
738  # keep all output EndPaths, including the DQMHistograms
739  paths.append( "*Output" )
740  paths.append( "DQMHistograms" )
741  else:
742  # keep all Paths and EndPaths
743  pass
744 
745  # drop unwanted paths for profiling (and timing studies)
746  if self.config.profiling:
747  paths.append( "-HLTAnalyzerEndpath" )
748 
749  # this should never be in any dump (nor online menu)
750  paths.append( "-OfflineOutput" )
751 
752  # expand all wildcards
753  paths = self.expandWildcards(paths, self.all_paths)
754 
755  if self.config.paths:
756  # do an "additive" consolidation
757  paths = self.consolidatePositiveList(paths)
758  if not paths:
759  raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
760  else:
761  # do a "subtractive" consolidation
762  paths = self.consolidateNegativeList(paths)
763  self.options['paths'] = paths
764 
765  def buildOptions(self):
766  # common configuration for all scenarios
767  self.options['services'].append( "-DQM" )
768  self.options['services'].append( "-FUShmDQMOutputService" )
769  self.options['services'].append( "-MicroStateService" )
770  self.options['services'].append( "-ModuleWebRegistry" )
771  self.options['services'].append( "-TimeProfilerService" )
772 
773  # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
774  # unless a hilton-like configuration has been requested
775  if not self.config.hilton:
776  self.options['services'].append( "-EvFDaqDirector" )
777  self.options['services'].append( "-FastMonitoringService" )
778  self.options['services'].append( "-DQMStore" )
779  self.options['modules'].append( "-hltDQMFileSaver" )
780  self.options['modules'].append( "-hltDQMFileSaverPB" )
781 
782  if self.config.fragment:
783  # extract a configuration file fragment
784  self.options['essources'].append( "-GlobalTag" )
785  self.options['essources'].append( "-HepPDTESSource" )
786  self.options['essources'].append( "-XMLIdealGeometryESSource" )
787  self.options['essources'].append( "-eegeom" )
788  self.options['essources'].append( "-es_hardcode" )
789  self.options['essources'].append( "-magfield" )
790 
791  self.options['esmodules'].append( "-SlaveField0" )
792  self.options['esmodules'].append( "-SlaveField20" )
793  self.options['esmodules'].append( "-SlaveField30" )
794  self.options['esmodules'].append( "-SlaveField35" )
795  self.options['esmodules'].append( "-SlaveField38" )
796  self.options['esmodules'].append( "-SlaveField40" )
797  self.options['esmodules'].append( "-VBF0" )
798  self.options['esmodules'].append( "-VBF20" )
799  self.options['esmodules'].append( "-VBF30" )
800  self.options['esmodules'].append( "-VBF35" )
801  self.options['esmodules'].append( "-VBF38" )
802  self.options['esmodules'].append( "-VBF40" )
803  self.options['esmodules'].append( "-CSCGeometryESModule" )
804  self.options['esmodules'].append( "-CaloGeometryBuilder" )
805  self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
806  self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
807  self.options['esmodules'].append( "-DTGeometryESModule" )
808  self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
809  self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
810  self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
811  self.options['esmodules'].append( "-EcalLaserCorrectionService" )
812  self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
813  self.options['esmodules'].append( "-GEMGeometryESModule" )
814  self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
815  self.options['esmodules'].append( "-HcalTopologyIdealEP" )
816  self.options['esmodules'].append( "-MuonNumberingInitialization" )
817  self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
818  self.options['esmodules'].append( "-RPCGeometryESModule" )
819  self.options['esmodules'].append( "-SiStripGainESProducer" )
820  self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
821  self.options['esmodules'].append( "-SiStripQualityESProducer" )
822  self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
823  self.options['esmodules'].append( "-TrackerAdditionalParametersPerDetESModule" )
824  self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
825  self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
826  self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
827  self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
828  self.options['esmodules'].append( "-hcal_db_producer" )
829  self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
830  self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
831  self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
832  self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
833  self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
834  self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
835  self.options['esmodules'].append( "-trackerTopology" )
836 
837  self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
838  self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
839  self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
840  self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
841  self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
842  self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
843  self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
844  self.options['esmodules'].append( "-XMLFromDBSource" )
845  self.options['esmodules'].append( "-sistripconn" )
846 
847  self.options['services'].append( "-MessageLogger" )
848 
849  self.options['psets'].append( "-maxEvents" )
850  self.options['psets'].append( "-options" )
851 
852  # remove Scouting OutputModules even though the EndPaths are kept
853  self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
854  self.options['modules'].append( "-hltOutputScoutingPF" )
855 
856  if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
857  self.options['services'].append( "-PrescaleService" )
858 
859  if self.config.fragment or self.config.timing:
860  self.options['services'].append( "-FastTimerService" )
861 
862 
863  def append_filenames(self, name, filenames):
864  if len(filenames) > 255:
865  token_open = "( *("
866  token_close = ") )"
867  else:
868  token_open = "("
869  token_close = ")"
870 
871  self.data += " %s = cms.untracked.vstring%s\n" % (name, token_open)
872  for line in filenames:
873  self.data += " '%s',\n" % line
874  self.data += " %s,\n" % (token_close)
875 
876 
877  def expand_filenames(self, input):
878  # check if the input is a dataset or a list of files
879  if input[0:8] == 'dataset:':
880  from .dasFileQuery import dasFileQuery
881  # extract the dataset name, and use DAS to fine the list of LFNs
882  dataset = input[8:]
883  files = dasFileQuery(dataset)
884  else:
885  # assume a comma-separated list of input files
886  files = input.split(',')
887  return files
888 
889  def build_source(self):
890  if self.config.hilton:
891  # use the DAQ source
892  return
893 
894  if self.config.input:
895  # if a dataset or a list of input files was given, use it
896  self.source = self.expand_filenames(self.config.input)
897  elif self.config.data:
898  # offline we can run on data...
899  self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
900  else:
901  # ...or on mc
902  self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
903 
904  if self.config.parent:
905  # if a dataset or a list of input files was given for the parent data, use it
906  self.parent = self.expand_filenames(self.config.parent)
907 
908  self.data += """
909 # source module (EDM inputs)
910 %(process)s.source = cms.Source( "PoolSource",
911 """
912  self.append_filenames("fileNames", self.source)
913  if (self.parent):
914  self.append_filenames("secondaryFileNames", self.parent)
915  self.data += """\
916  inputCommands = cms.untracked.vstring(
917  'keep *'
918  )
919 )
920 """
def _fix_parameter
Definition: confdb.py:337
def overrideL1MenuXml
Definition: confdb.py:446
def addGlobalOptions
Definition: confdb.py:319
def getSetupConfigurationFromDB
Definition: confdb.py:60
def overrideOutput
Definition: confdb.py:466
def expand_filenames
Definition: confdb.py:877
boost::dynamic_bitset append(const boost::dynamic_bitset<> &bs1, const boost::dynamic_bitset<> &bs2)
this method takes two bitsets bs1 and bs2 and returns result of bs2 appended to the end of bs1 ...
def splitter
Definition: confdb.py:11
def overrideProcessName
Definition: confdb.py:541
def specificCustomize
Definition: confdb.py:176
def consolidateNegativeList
Definition: confdb.py:142
def instrumentErrorEventType
Definition: confdb.py:394
def append_filenames
Definition: confdb.py:863
def consolidatePositiveList
Definition: confdb.py:154
def expandWildcards
Definition: confdb.py:126
def overrideGlobalTag
Definition: confdb.py:402
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
def removeElementFromSequencesTasksAndPaths
Definition: confdb.py:611
def getRawConfigurationFromDB
Definition: confdb.py:83
def overrideParameters
Definition: confdb.py:603
def loadCffCommand
Definition: confdb.py:592
def instrumentTiming
Definition: confdb.py:619
def instrumentOpenMode
Definition: confdb.py:380
def updateMessageLogger
Definition: confdb.py:561
def loadAdditionalConditions
Definition: confdb.py:575