CMS 3D CMS Logo

confdb.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import sys
4 import re
5 import os
6 import urllib, urllib2
7 from pipe import pipe as _pipe
8 from options import globalTag
9 from itertools import islice
10 
11 def splitter(iterator, n):
12  i = iterator.__iter__()
13  while True:
14  l = list(islice(i, n))
15  if l:
16  yield l
17  else:
18  break
19 
20 
22 
23  def __init__(self, configuration):
24  self.config = configuration
25  self.data = None
26  self.source = []
27  self.parent = []
28 
29  self.options = {
30  'essources' : [],
31  'esmodules' : [],
32  'modules' : [],
33  'sequences' : [],
34  'services' : [],
35  'paths' : [],
36  'psets' : [],
37  'blocks' : [],
38  }
39 
40  self.labels = {}
41  if self.config.fragment:
42  self.labels['process'] = 'fragment'
43  self.labels['dict'] = 'fragment.__dict__'
44  else:
45  self.labels['process'] = 'process'
46  self.labels['dict'] = 'process.__dict__'
47 
48  if self.config.prescale and (self.config.prescale.lower() != 'none'):
49  self.labels['prescale'] = self.config.prescale
50 
51  # get the configuration from ConfdB
52  from confdbOfflineConverter import OfflineConverter
53  self.converter = OfflineConverter(version = self.config.menu.version, database = self.config.menu.database)
54  self.buildPathList()
55  self.buildOptions()
58  self.customize()
59 
61  if not self.config.setup:
62  return
63  ## if --setup is a python file, use directly that file as setup_cff.py
64  if ".py" in self.config.setup:
65  self.config.setupFile = self.config.setup.split(".py")[0]
66  return
67  args = ['--configName', self.config.setup ]
68  args.append('--noedsources')
69  args.append('--nopaths')
70  for key, vals in self.options.iteritems():
71  if vals:
72  args.extend(('--'+key, ','.join(vals)))
73  args.append('--cff')
74  data, err = self.converter.query( *args )
75  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
76  sys.stderr.write("%s: error while retrieving the HLT setup menu\n\n" % os.path.basename(sys.argv[0]))
77  sys.stderr.write(err + "\n\n")
78  sys.exit(1)
79  self.config.setupFile = "setup_"+self.config.setup[1:].replace("/","_")+"_cff"
80  file(self.config.setupFile+".py","w+").write("# This file is automatically generated by hltGetConfiguration.\n" + data)
81 
83  if self.config.menu.run:
84  args = ['--runNumber', self.config.menu.run]
85  else:
86  args = ['--configName', self.config.menu.name ]
87  args.append('--noedsources')
88  for key, vals in self.options.iteritems():
89  if vals:
90  args.extend(('--'+key, ','.join(vals)))
91 
92  data, err = self.converter.query( *args )
93  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
94  sys.stderr.write("%s: error while retrieving the HLT menu\n\n" % os.path.basename(sys.argv[0]))
95  sys.stderr.write(err + "\n\n")
96  sys.exit(1)
97  self.data = data
98 
99  def getPathList(self):
100  if self.config.menu.run:
101  args = ['--runNumber', self.config.menu.run]
102  else:
103  args = ['--configName', self.config.menu.name]
104  args.extend( (
105  '--cff',
106  '--noedsources',
107  '--noes',
108  '--noservices',
109  '--nosequences',
110  '--nomodules'
111  ) )
112 
113  data, err = self.converter.query( *args )
114  if 'ERROR' in err or 'Exhausted Resultset' in err or 'CONFIG_NOT_FOUND' in err:
115  sys.stderr.write("%s: error while retrieving the list of paths from the HLT menu\n\n" % os.path.basename(sys.argv[0]))
116  sys.stderr.write(err + "\n\n")
117  sys.exit(1)
118  filter = re.compile(r' *= *cms.(End)?Path.*')
119  paths = [ filter.sub('', line) for line in data.splitlines() if filter.search(line) ]
120  return paths
121 
122 
123  @staticmethod
124  def expandWildcards(globs, collection):
125  # expand a list of unix-style wildcards matching a given collection
126  # wildcards with no matches are silently discarded
127  matches = []
128  for glob in globs:
129  negate = ''
130  if glob[0] == '-':
131  negate = '-'
132  glob = glob[1:]
133  # translate a unix-style glob expression into a regular expression
134  filter = re.compile(r'^' + glob.replace('?', '.').replace('*', '.*').replace('[!', '[^') + r'$')
135  matches.extend( negate + element for element in collection if filter.match(element) )
136  return matches
137 
138 
139  @staticmethod
140  def consolidateNegativeList(elements):
141  # consolidate a list of path exclusions and re-inclusions
142  # the result is the list of paths to be removed from the dump
143  result = set()
144  for element in elements:
145  if element[0] == '-':
146  result.add( element )
147  else:
148  result.discard( '-' + element )
149  return sorted( element for element in result )
150 
151  @staticmethod
152  def consolidatePositiveList(elements):
153  # consolidate a list of path selection and re-exclusions
154  # the result is the list of paths to be included in the dump
155  result = set()
156  for element in elements:
157  if element[0] == '-':
158  result.discard( element[1:] )
159  else:
160  result.add( element )
161  return sorted( element for element in result )
162 
163 
164  # dump the final configuration
165  def dump(self):
166  self.data = self.data % self.labels
167  if self.config.fragment:
168  self.data = re.sub( r'\bprocess\b', 'fragment', self.data )
169  self.data = re.sub( r'\bProcess\b', 'ProcessFragment', self.data )
170  return self.data
171 
172 
173  # add specific customizations
174  def specificCustomize(self):
175  # specific customizations now live in HLTrigger.Configuration.customizeHLTforALL.customizeHLTforAll(.,.)
176  if self.config.fragment:
177  self.data += """
178 # add specific customizations
179 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
180 fragment = customizeHLTforAll(fragment,"%s")
181 """ % (self.config.type)
182  else:
183  if self.config.type=="Fake":
184  prefix = "run1"
185  else:
186  prefix = "run2"
187  _gtData = "auto:"+prefix+"_hlt_"+self.config.type
188  _gtMc = "auto:"+prefix+"_mc_" +self.config.type
189  self.data += """
190 # add specific customizations
191 _customInfo = {}
192 _customInfo['menuType' ]= "%s"
193 _customInfo['globalTags']= {}
194 _customInfo['globalTags'][True ] = "%s"
195 _customInfo['globalTags'][False] = "%s"
196 _customInfo['inputFiles']={}
197 _customInfo['inputFiles'][True] = "file:RelVal_Raw_%s_DATA.root"
198 _customInfo['inputFiles'][False] = "file:RelVal_Raw_%s_MC.root"
199 _customInfo['maxEvents' ]= %s
200 _customInfo['globalTag' ]= "%s"
201 _customInfo['inputFile' ]= %s
202 _customInfo['realData' ]= %s
203 from HLTrigger.Configuration.customizeHLTforALL import customizeHLTforAll
204 %%(process)s = customizeHLTforAll(%%(process)s,"%s",_customInfo)
205 """ % (self.config.type,_gtData,_gtMc,self.config.type,self.config.type,self.config.events,self.config.globaltag,self.source,self.config.data,self.config.type)
206 
207  self.data += """
208 from HLTrigger.Configuration.customizeHLTforCMSSW import customizeHLTforCMSSW
209 %%(process)s = customizeHLTforCMSSW(%%(process)s,"%s")
210 """ % (self.config.type)
211 
212  # Eras-based customisations
213  self.data += """
214 # Eras-based customisations
215 from HLTrigger.Configuration.Eras import modifyHLTforEras
216 modifyHLTforEras(%(process)s)
217 """
218  # add the user-defined customization functions, if any
219  if self.config.customise:
220  self.data += "\n"
221  self.data += "#User-defined customization functions\n"
222  for customise in self.config.customise.split(","):
223  customiseValues = customise.split(".")
224  if len(customiseValues)>=3: raise Exception("--customise option cannot contain more than one dot.")
225  if len(customiseValues)==1:
226  customiseValues.append("customise")
227  customiseValues[0] = customiseValues[0].replace("/",".")
228  self.data += "from "+customiseValues[0]+" import "+customiseValues[1]+"\n"
229  self.data += "process = "+customiseValues[1]+"(process)\n"
230 
231  # customize the configuration according to the options
232  def customize(self):
233 
234  # adapt the source to the current scenario
235  if not self.config.fragment:
236  self.build_source()
237 
238  # manual override some parameters
239  if self.config.type in ('HIon', ):
240  if self.config.data:
241  if not self.config.fragment:
242  self._fix_parameter( type = 'InputTag', value = 'rawDataCollector', replace = 'rawDataRepacker')
243 
244  # if requested, remove the HLT prescales
245  self.fixPrescales()
246 
247  # if requested, override all ED/HLTfilters to always pass ("open" mode)
248  self.instrumentOpenMode()
249 
250  # if requested, change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
252 
253  # if requested, instrument the self with the modules and EndPath needed for timing studies
254  self.instrumentTiming()
255 
256  # if requested, override the L1 self from the GlobalTag (Xml)
257  self.overrideL1MenuXml()
258 
259  # if requested, run the L1 emulator
260  self.runL1Emulator()
261 
262  # add process.load("setup_cff")
263  self.loadSetupCff()
264 
265  if self.config.fragment:
266  self.data += """
267 # dummyfy hltGetConditions in cff's
268 if 'hltGetConditions' in %(dict)s and 'HLTriggerFirstPath' in %(dict)s :
269  %(process)s.hltDummyConditions = cms.EDFilter( "HLTBool",
270  result = cms.bool( True )
271  )
272  %(process)s.HLTriggerFirstPath.replace(%(process)s.hltGetConditions,%(process)s.hltDummyConditions)
273 """
274 
275  # fix the Scouting EndPaths
276  for path in self.all_paths:
277  match = re.match(r'(Scouting\w+)Output$', path)
278  if match:
279  module = 'hltOutput' + match.group(1)
280  self.data = self.data.replace(path+' = cms.EndPath', path+' = cms.Path')
281  self.data = self.data.replace(' + process.'+module, '')
282 
283  else:
284 
285  # override the process name and adapt the relevant filters
286  self.overrideProcessName()
287 
288  # select specific Eras
289  self.addEras()
290 
291  # override the output modules to output root files
292  self.overrideOutput()
293 
294  # add global options
295  self.addGlobalOptions()
296 
297  # if requested or necessary, override the GlobalTag and connection strings (incl. L1!)
298  self.overrideGlobalTag()
299 
300  # request summary informations from the MessageLogger
301  self.updateMessageLogger()
302 
303  # replace DQMStore and DQMRootOutputModule with a configuration suitable for running offline
304  self.instrumentDQM()
305 
306  # add specific customisations
307  self.specificCustomize()
308 
309 
310  def addGlobalOptions(self):
311  # add global options
312  self.data += """
313 # limit the number of events to be processed
314 %%(process)s.maxEvents = cms.untracked.PSet(
315  input = cms.untracked.int32( %d )
316 )
317 """ % self.config.events
318 
319  self.data += """
320 # enable TrigReport, TimeReport and MultiThreading
321 %(process)s.options = cms.untracked.PSet(
322  wantSummary = cms.untracked.bool( True ),
323  numberOfThreads = cms.untracked.uint32( 4 ),
324  numberOfStreams = cms.untracked.uint32( 0 ),
325  sizeOfStackForThreadsInKB = cms.untracked.uint32( 10*1024 )
326 )
327 """
328 
329  def _fix_parameter(self, **args):
330  """arguments:
331  name: parameter name (optional)
332  type: parameter type (look for tracked and untracked variants)
333  value: original value
334  replace: replacement value
335  """
336  if 'name' in args:
337  self.data = re.sub(
338  r'%(name)s = cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
339  r'%(name)s = cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
340  self.data)
341  else:
342  self.data = re.sub(
343  r'cms(?P<tracked>(?:\.untracked)?)\.%(type)s\( (?P<quote>["\']?)%(value)s(?P=quote)' % args,
344  r'cms\g<tracked>.%(type)s( \g<quote>%(replace)s\g<quote>' % args,
345  self.data)
346 
347 
348  def fixPrescales(self):
349  # update the PrescaleService to match the new list of paths
350  if self.options['paths']:
351  if self.options['paths'][0][0] == '-':
352  # drop requested paths
353  for minuspath in self.options['paths']:
354  path = minuspath[1:]
355  self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
356  else:
357  # keep requested paths
358  for path in self.all_paths:
359  if path not in self.options['paths']:
360  self.data = re.sub(r' cms.PSet\( pathName = cms.string\( "%s" \),\n prescales = cms.vuint32\( .* \)\n \),?\n' % path, '', self.data)
361 
362  if self.config.prescale and (self.config.prescale.lower() != 'none'):
363  # TO DO: check that the requested prescale column is valid
364  self.data += """
365 # force the use of a specific HLT prescale column
366 if 'PrescaleService' in %(dict)s:
367  %(process)s.PrescaleService.forceDefault = True
368  %(process)s.PrescaleService.lvl1DefaultLabel = '%(prescale)s'
369 """
370 
371 
373  if self.config.open:
374  # find all EDfilters
375  filters = [ match[1] for match in re.findall(r'(process\.)?\b(\w+) = cms.EDFilter', self.data) ]
376  re_sequence = re.compile( r'cms\.(Path|Sequence)\((.*)\)' )
377  # remove existing 'cms.ignore' and '~' modifiers
378  self.data = re_sequence.sub( lambda line: re.sub( r'cms\.ignore *\( *((process\.)?\b(\w+)) *\)', r'\1', line.group(0) ), self.data )
379  self.data = re_sequence.sub( lambda line: re.sub( r'~', '', line.group(0) ), self.data )
380  # wrap all EDfilters with "cms.ignore( ... )", 1000 at a time (python 2.6 complains for too-big regular expressions)
381  for some in splitter(filters, 1000):
382  re_filters = re.compile( r'\b((process\.)?(' + r'|'.join(some) + r'))\b' )
383  self.data = re_sequence.sub( lambda line: re_filters.sub( r'cms.ignore( \1 )', line.group(0) ), self.data )
384 
385 
387  if self.config.errortype:
388  # change all HLTTriggerTypeFilter EDFilters to accept only error events (SelectedTriggerType = 0)
389  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '1', replace = '0')
390  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '2', replace = '0')
391  self._fix_parameter(name = 'SelectedTriggerType', type ='int32', value = '3', replace = '0')
392 
393 
394  def overrideGlobalTag(self):
395  # overwrite GlobalTag
396  # the logic is:
397  # - if a GlobalTag is specified on the command line:
398  # - override the global tag
399  # - if the GT is "auto:...", insert the code to read it from Configuration.AlCa.autoCond
400  # - if a GlobalTag is NOT specified on the command line:
401  # - when running on data, do nothing, and keep the global tag in the menu
402  # - when running on mc, take the GT from the configuration.type
403 
404  # override the GlobalTag connection string and pfnPrefix
405 
406  # when running on MC, override the global tag even if not specified on the command line
407  if not self.config.data and not self.config.globaltag:
408  if self.config.type in globalTag:
409  self.config.globaltag = globalTag[self.config.type]
410  else:
411  self.config.globaltag = globalTag['GRun']
412 
413  # if requested, override the L1 menu from the GlobalTag
414  if self.config.l1.override:
415  self.config.l1.tag = self.config.l1.override
416  self.config.l1.record = 'L1TUtmTriggerMenuRcd'
417  self.config.l1.connect = ''
418  self.config.l1.label = ''
419  if not self.config.l1.snapshotTime:
420  self.config.l1.snapshotTime = '9999-12-31 23:59:59.000'
421  self.config.l1cond = '%(tag)s,%(record)s,%(connect)s,%(label)s,%(snapshotTime)s' % self.config.l1.__dict__
422  else:
423  self.config.l1cond = None
424 
425  if self.config.globaltag or self.config.l1cond:
426  text = """
427 # override the GlobalTag, connection string and pfnPrefix
428 if 'GlobalTag' in %(dict)s:
429  from Configuration.AlCa.GlobalTag import GlobalTag as customiseGlobalTag
430  %(process)s.GlobalTag = customiseGlobalTag(%(process)s.GlobalTag"""
431  if self.config.globaltag:
432  text += ", globaltag = %s" % repr(self.config.globaltag)
433  if self.config.l1cond:
434  text += ", conditions = %s" % repr(self.config.l1cond)
435  text += ")\n"
436  self.data += text
437 
438  def overrideL1MenuXml(self):
439  # if requested, override the GlobalTag's L1T menu from an Xml file
440  if self.config.l1Xml.XmlFile:
441  text = """
442 # override the GlobalTag's L1T menu from an Xml file
443 from HLTrigger.Configuration.CustomConfigs import L1XML
444 %%(process)s = L1XML(%%(process)s,"%s")
445 """ % (self.config.l1Xml.XmlFile)
446  self.data += text
447 
448  def runL1Emulator(self):
449  # if requested, run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
450  if self.config.emulator:
451  text = """
452 # run the Full L1T emulator, then repack the data into a new RAW collection, to be used by the HLT
453 from HLTrigger.Configuration.CustomConfigs import L1REPACK
454 %%(process)s = L1REPACK(%%(process)s,"%s")
455 """ % (self.config.emulator)
456  self.data += text
457 
458  def overrideOutput(self):
459  # override the "online" ShmStreamConsumer output modules with "offline" PoolOutputModule's
460  self.data = re.sub(
461  r'\b(process\.)?hltOutput(\w+) *= *cms\.OutputModule\( *"ShmStreamConsumer" *,',
462  r'%(process)s.hltOutput\2 = cms.OutputModule( "PoolOutputModule",\n fileName = cms.untracked.string( "output\2.root" ),\n fastCloning = cms.untracked.bool( False ),\n dataset = cms.untracked.PSet(\n filterName = cms.untracked.string( "" ),\n dataTier = cms.untracked.string( "RAW" )\n ),',
463  self.data
464  )
465 
466  if not self.config.fragment and self.config.output == 'minimal':
467  # add a single output to keep the TriggerResults and TriggerEvent
468  self.data += """
469 # add a single "keep *" output
470 %(process)s.hltOutputMinimal = cms.OutputModule( "PoolOutputModule",
471  fileName = cms.untracked.string( "output.root" ),
472  fastCloning = cms.untracked.bool( False ),
473  dataset = cms.untracked.PSet(
474  dataTier = cms.untracked.string( 'AOD' ),
475  filterName = cms.untracked.string( '' )
476  ),
477  outputCommands = cms.untracked.vstring( 'drop *',
478  'keep edmTriggerResults_*_*_*',
479  'keep triggerTriggerEvent_*_*_*'
480  )
481 )
482 %(process)s.MinimalOutput = cms.EndPath( %(process)s.hltOutputMinimal )
483 """
484  elif not self.config.fragment and self.config.output == 'full':
485  # add a single "keep *" output
486  self.data += """
487 # add a single "keep *" output
488 %(process)s.hltOutputFull = cms.OutputModule( "PoolOutputModule",
489  fileName = cms.untracked.string( "output.root" ),
490  fastCloning = cms.untracked.bool( False ),
491  dataset = cms.untracked.PSet(
492  dataTier = cms.untracked.string( 'RECO' ),
493  filterName = cms.untracked.string( '' )
494  ),
495  outputCommands = cms.untracked.vstring( 'keep *' )
496 )
497 %(process)s.FullOutput = cms.EndPath( %(process)s.hltOutputFull )
498 """
499 
500  # select specific Eras
501  def addEras(self):
502  if self.config.eras is None:
503  return
504  self.data = re.sub(r'process = cms.Process\( *"\w+"', 'from Configuration.StandardSequences.Eras import eras\n\g<0>, '+', '.join('eras.' + era for era in self.config.eras.split(',')), self.data)
505 
506  # select specific Eras
507  def loadSetupCff(self):
508  if self.config.setup is None:
509  return
510  processLine = self.data.find("\n",self.data.find("cms.Process"))
511  self.data = self.data[:processLine]+'\nprocess.load("%s")'%self.config.setupFile+self.data[processLine:]
512 
513  # override the process name and adapt the relevant filters
515  if self.config.name is None:
516  return
517 
518  # sanitise process name
519  self.config.name = self.config.name.replace("_","")
520  # override the process name
521  quote = '[\'\"]'
522  self.data = re.compile(r'^(process\s*=\s*cms\.Process\(\s*' + quote + r')\w+(' + quote + r'\s*\).*)$', re.MULTILINE).sub(r'\1%s\2' % self.config.name, self.data, 1)
523 
524  # when --setup option is used, remove possible errors from PrescaleService due to missing HLT paths.
525  if self.config.setup: self.data += """
526 # avoid PrescaleService error due to missing HLT paths
527 if 'PrescaleService' in process.__dict__:
528  for pset in reversed(process.PrescaleService.prescaleTable):
529  if not hasattr(process,pset.pathName.value()):
530  process.PrescaleService.prescaleTable.remove(pset)
531 """
532 
533 
535  # request summary informations from the MessageLogger
536  self.data += """
537 if 'MessageLogger' in %(dict)s:
538  %(process)s.MessageLogger.categories.append('TriggerSummaryProducerAOD')
539  %(process)s.MessageLogger.categories.append('L1GtTrigReport')
540  %(process)s.MessageLogger.categories.append('L1TGlobalSummary')
541  %(process)s.MessageLogger.categories.append('HLTrigReport')
542  %(process)s.MessageLogger.categories.append('FastReport')
543 """
544 
545 
546  def loadAdditionalConditions(self, comment, *conditions):
547  # load additional conditions
548  self.data += """
549 # %s
550 if 'GlobalTag' in %%(dict)s:
551 """ % comment
552  for condition in conditions:
553  self.data += """ %%(process)s.GlobalTag.toGet.append(
554  cms.PSet(
555  record = cms.string( '%(record)s' ),
556  tag = cms.string( '%(tag)s' ),
557  label = cms.untracked.string( '%(label)s' ),
558  )
559  )
560 """ % condition
561 
562 
563  def loadCffCommand(self, module):
564  # load a cfi or cff module
565  if self.config.fragment:
566  return 'from %s import *\n' % module
567  else:
568  return 'process.load( "%s" )\n' % module
569 
570  def loadCff(self, module):
571  self.data += self.loadCffCommand(module)
572 
573 
574  def overrideParameters(self, module, parameters):
575  # override a module's parameter if the module is present in the configuration
576  self.data += "if '%s' in %%(dict)s:\n" % module
577  for (parameter, value) in parameters:
578  self.data += " %%(process)s.%s.%s = %s\n" % (module, parameter, value)
579  self.data += "\n"
580 
581 
582  def instrumentTiming(self):
583 
584  if self.config.timing:
585  self.data += """
586 # instrument the menu with the modules and EndPath needed for timing studies
587 """
588 
589  self.data += '\n# configure the FastTimerService\n'
590  self.loadCff('HLTrigger.Timer.FastTimerService_cfi')
591 
592  self.data += """# print a text summary at the end of the job
593 %(process)s.FastTimerService.printEventSummary = False
594 %(process)s.FastTimerService.printRunSummary = False
595 %(process)s.FastTimerService.printJobSummary = True
596 
597 # enable DQM plots
598 %(process)s.FastTimerService.enableDQM = True
599 
600 # enable per-path DQM plots (starting with CMSSW 9.2.3-patch2)
601 %(process)s.FastTimerService.enableDQMbyPath = True
602 
603 # enable per-module DQM plots
604 %(process)s.FastTimerService.enableDQMbyModule = True
605 
606 # enable per-event DQM plots vs lumisection
607 %(process)s.FastTimerService.enableDQMbyLumiSection = True
608 %(process)s.FastTimerService.dqmLumiSectionsRange = 2500
609 
610 # set the time resolution of the DQM plots
611 %(process)s.FastTimerService.dqmTimeRange = 2000.
612 %(process)s.FastTimerService.dqmTimeResolution = 10.
613 %(process)s.FastTimerService.dqmPathTimeRange = 1000.
614 %(process)s.FastTimerService.dqmPathTimeResolution = 5.
615 %(process)s.FastTimerService.dqmModuleTimeRange = 200.
616 %(process)s.FastTimerService.dqmModuleTimeResolution = 1.
617 
618 # set the base DQM folder for the plots
619 %(process)s.FastTimerService.dqmPath = 'HLT/TimerService'
620 %(process)s.FastTimerService.enableDQMbyProcesses = False
621 """
622 
623 
624  def instrumentDQM(self):
625  if not self.config.hilton:
626  # remove any reference to the hltDQMFileSaver
627  if 'hltDQMFileSaver' in self.data:
628  self.data = re.sub(r'\b(process\.)?hltDQMFileSaver \+ ', '', self.data)
629  self.data = re.sub(r' \+ \b(process\.)?hltDQMFileSaver', '', self.data)
630  self.data = re.sub(r'\b(process\.)?hltDQMFileSaver', '', self.data)
631 
632  # instrument the HLT menu with DQMStore and DQMRootOutputModule suitable for running offline
633  dqmstore = "\n# load the DQMStore and DQMRootOutputModule\n"
634  dqmstore += self.loadCffCommand('DQMServices.Core.DQMStore_cfi')
635  dqmstore += "%(process)s.DQMStore.enableMultiThread = True\n"
636  dqmstore += """
637 %(process)s.dqmOutput = cms.OutputModule("DQMRootOutputModule",
638  fileName = cms.untracked.string("DQMIO.root")
639 )
640 """
641 
642  empty_path = re.compile(r'.*\b(process\.)?DQMOutput = cms\.EndPath\( *\).*')
643  other_path = re.compile(r'(.*\b(process\.)?DQMOutput = cms\.EndPath\()(.*)')
644  if empty_path.search(self.data):
645  # replace an empty DQMOutput path
646  self.data = empty_path.sub(dqmstore + '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n', self.data)
647  elif other_path.search(self.data):
648  # prepend the dqmOutput to the DQMOutput path
649  self.data = other_path.sub(dqmstore + r'\g<1> %(process)s.dqmOutput +\g<3>', self.data)
650  else:
651  # ceate a new DQMOutput path with the dqmOutput module
652  self.data += dqmstore
653  self.data += '\n%(process)s.DQMOutput = cms.EndPath( %(process)s.dqmOutput )\n'
654 
655 
656  @staticmethod
657  def dumppaths(paths):
658  sys.stderr.write('Path selection:\n')
659  for path in paths:
660  sys.stderr.write('\t%s\n' % path)
661  sys.stderr.write('\n\n')
662 
663  def buildPathList(self):
664  self.all_paths = self.getPathList()
665 
666  if self.config.paths:
667  # no path list was requested, dump the full table, minus unsupported / unwanted paths
668  paths = self.config.paths.split(',')
669  else:
670  # dump only the requested paths, plus the eventual output endpaths
671  paths = []
672 
673  # 'none' should remove all outputs
674  # 'dqm' should remove all outputs but DQMHistograms
675  # 'minimal' should remove all outputs but DQMHistograms, and add a single output module to keep the TriggerResults and TriggerEvent
676  # 'full' should remove all outputs but DQMHistograms, and add a single output module to "keep *"
677  # See also the `overrideOutput` method
678  if self.config.fragment or self.config.output in ('none', ):
679  if self.config.paths:
680  # keep only the Paths and EndPaths requested explicitly
681  pass
682  else:
683  # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring and DQMHistograms
684  paths.append( "-*Output" )
685  paths.append( "Scouting*Output" )
686  paths.append( "-RatesMonitoring")
687  paths.append( "-DQMHistograms")
688 
689  elif self.config.output in ('dqm', 'minimal', 'full'):
690  if self.config.paths:
691  # keep only the Paths and EndPaths requested explicitly, and the DQMHistograms
692  paths.append( "DQMHistograms" )
693  else:
694  # drop all output EndPaths but the Scouting ones, and drop the RatesMonitoring
695  paths.append( "-*Output" )
696  paths.append( "Scouting*Output" )
697  paths.append( "-RatesMonitoring")
698 
699  else:
700  if self.config.paths:
701  # keep all output EndPaths, including the DQMHistograms
702  paths.append( "*Output" )
703  paths.append( "DQMHistograms" )
704  else:
705  # keep all Paths and EndPaths
706  pass
707 
708  # drop unwanted paths for profiling (and timing studies)
709  if self.config.profiling:
710  paths.append( "-HLTAnalyzerEndpath" )
711 
712  # this should never be in any dump (nor online menu)
713  paths.append( "-OfflineOutput" )
714 
715  # expand all wildcards
716  paths = self.expandWildcards(paths, self.all_paths)
717 
718  if self.config.paths:
719  # do an "additive" consolidation
720  paths = self.consolidatePositiveList(paths)
721  if not paths:
722  raise RuntimeError('Error: option "--paths %s" does not select any valid paths' % self.config.paths)
723  else:
724  # do a "subtractive" consolidation
725  paths = self.consolidateNegativeList(paths)
726  self.options['paths'] = paths
727 
728  def buildOptions(self):
729  # common configuration for all scenarios
730  self.options['services'].append( "-DQM" )
731  self.options['services'].append( "-FUShmDQMOutputService" )
732  self.options['services'].append( "-MicroStateService" )
733  self.options['services'].append( "-ModuleWebRegistry" )
734  self.options['services'].append( "-TimeProfilerService" )
735 
736  # remove the DAQ modules and the online definition of the DQMStore and DQMFileSaver
737  # unless a hilton-like configuration has been requested
738  if not self.config.hilton:
739  self.options['services'].append( "-EvFDaqDirector" )
740  self.options['services'].append( "-FastMonitoringService" )
741  self.options['services'].append( "-DQMStore" )
742  self.options['modules'].append( "-hltDQMFileSaver" )
743 
744  if self.config.fragment:
745  # extract a configuration file fragment
746  self.options['essources'].append( "-GlobalTag" )
747  self.options['essources'].append( "-HepPDTESSource" )
748  self.options['essources'].append( "-XMLIdealGeometryESSource" )
749  self.options['essources'].append( "-eegeom" )
750  self.options['essources'].append( "-es_hardcode" )
751  self.options['essources'].append( "-magfield" )
752 
753  self.options['esmodules'].append( "-AutoMagneticFieldESProducer" )
754  self.options['esmodules'].append( "-SlaveField0" )
755  self.options['esmodules'].append( "-SlaveField20" )
756  self.options['esmodules'].append( "-SlaveField30" )
757  self.options['esmodules'].append( "-SlaveField35" )
758  self.options['esmodules'].append( "-SlaveField38" )
759  self.options['esmodules'].append( "-SlaveField40" )
760  self.options['esmodules'].append( "-VBF0" )
761  self.options['esmodules'].append( "-VBF20" )
762  self.options['esmodules'].append( "-VBF30" )
763  self.options['esmodules'].append( "-VBF35" )
764  self.options['esmodules'].append( "-VBF38" )
765  self.options['esmodules'].append( "-VBF40" )
766  self.options['esmodules'].append( "-CSCGeometryESModule" )
767  self.options['esmodules'].append( "-CaloGeometryBuilder" )
768  self.options['esmodules'].append( "-CaloTowerHardcodeGeometryEP" )
769  self.options['esmodules'].append( "-CastorHardcodeGeometryEP" )
770  self.options['esmodules'].append( "-DTGeometryESModule" )
771  self.options['esmodules'].append( "-EcalBarrelGeometryEP" )
772  self.options['esmodules'].append( "-EcalElectronicsMappingBuilder" )
773  self.options['esmodules'].append( "-EcalEndcapGeometryEP" )
774  self.options['esmodules'].append( "-EcalLaserCorrectionService" )
775  self.options['esmodules'].append( "-EcalPreshowerGeometryEP" )
776  self.options['esmodules'].append( "-HcalHardcodeGeometryEP" )
777  self.options['esmodules'].append( "-HcalTopologyIdealEP" )
778  self.options['esmodules'].append( "-MuonNumberingInitialization" )
779  self.options['esmodules'].append( "-ParametrizedMagneticFieldProducer" )
780  self.options['esmodules'].append( "-RPCGeometryESModule" )
781  self.options['esmodules'].append( "-SiStripGainESProducer" )
782  self.options['esmodules'].append( "-SiStripRecHitMatcherESProducer" )
783  self.options['esmodules'].append( "-SiStripQualityESProducer" )
784  self.options['esmodules'].append( "-StripCPEfromTrackAngleESProducer" )
785  self.options['esmodules'].append( "-TrackerDigiGeometryESModule" )
786  self.options['esmodules'].append( "-TrackerGeometricDetESModule" )
787  self.options['esmodules'].append( "-VolumeBasedMagneticFieldESProducer" )
788  self.options['esmodules'].append( "-ZdcHardcodeGeometryEP" )
789  self.options['esmodules'].append( "-hcal_db_producer" )
790  self.options['esmodules'].append( "-L1GtTriggerMaskAlgoTrigTrivialProducer" )
791  self.options['esmodules'].append( "-L1GtTriggerMaskTechTrigTrivialProducer" )
792  self.options['esmodules'].append( "-hltESPEcalTrigTowerConstituentsMapBuilder" )
793  self.options['esmodules'].append( "-hltESPGlobalTrackingGeometryESProducer" )
794  self.options['esmodules'].append( "-hltESPMuonDetLayerGeometryESProducer" )
795  self.options['esmodules'].append( "-hltESPTrackerRecoGeometryESProducer" )
796  self.options['esmodules'].append( "-trackerTopology" )
797 
798  self.options['esmodules'].append( "-CaloTowerGeometryFromDBEP" )
799  self.options['esmodules'].append( "-CastorGeometryFromDBEP" )
800  self.options['esmodules'].append( "-EcalBarrelGeometryFromDBEP" )
801  self.options['esmodules'].append( "-EcalEndcapGeometryFromDBEP" )
802  self.options['esmodules'].append( "-EcalPreshowerGeometryFromDBEP" )
803  self.options['esmodules'].append( "-HcalGeometryFromDBEP" )
804  self.options['esmodules'].append( "-ZdcGeometryFromDBEP" )
805  self.options['esmodules'].append( "-XMLFromDBSource" )
806  self.options['esmodules'].append( "-sistripconn" )
807 
808  self.options['services'].append( "-MessageLogger" )
809 
810  self.options['psets'].append( "-maxEvents" )
811  self.options['psets'].append( "-options" )
812 
813  # remove Scouting OutputModules even though the EndPaths are kept
814  self.options['modules'].append( "-hltOutputScoutingCaloMuon" )
815  self.options['modules'].append( "-hltOutputScoutingPF" )
816 
817  if self.config.fragment or (self.config.prescale and (self.config.prescale.lower() == 'none')):
818  self.options['services'].append( "-PrescaleService" )
819 
820  if self.config.fragment or self.config.timing:
821  self.options['services'].append( "-FastTimerService" )
822 
823 
824  def append_filenames(self, name, filenames):
825  if len(filenames) > 255:
826  token_open = "( *("
827  token_close = ") )"
828  else:
829  token_open = "("
830  token_close = ")"
831 
832  self.data += " %s = cms.untracked.vstring%s\n" % (name, token_open)
833  for line in filenames:
834  self.data += " '%s',\n" % line
835  self.data += " %s,\n" % (token_close)
836 
837 
838  def expand_filenames(self, input):
839  # check if the input is a dataset or a list of files
840  if input[0:8] == 'dataset:':
841  from dasFileQuery import dasFileQuery
842  # extract the dataset name, and use DAS to fine the list of LFNs
843  dataset = input[8:]
844  files = dasFileQuery(dataset)
845  else:
846  # assume a comma-separated list of input files
847  files = input.split(',')
848  return files
849 
850  def build_source(self):
851  if self.config.input:
852  # if a dataset or a list of input files was given, use it
853  self.source = self.expand_filenames(self.config.input)
854  elif self.config.data:
855  # offline we can run on data...
856  self.source = [ "file:RelVal_Raw_%s_DATA.root" % self.config.type ]
857  else:
858  # ...or on mc
859  self.source = [ "file:RelVal_Raw_%s_MC.root" % self.config.type ]
860 
861  if self.config.parent:
862  # if a dataset or a list of input files was given for the parent data, use it
863  self.parent = self.expand_filenames(self.config.parent)
864 
865  self.data += """
866 %(process)s.source = cms.Source( "PoolSource",
867 """
868  self.append_filenames("fileNames", self.source)
869  if (self.parent):
870  self.append_filenames("secondaryFileNames", self.parent)
871  self.data += """\
872  inputCommands = cms.untracked.vstring(
873  'keep *'
874  )
875 )
876 """
def __init__(self, configuration)
Definition: confdb.py:23
def splitter(iterator, n)
Definition: confdb.py:11
def loadSetupCff(self)
Definition: confdb.py:507
def addGlobalOptions(self)
Definition: confdb.py:310
def getSetupConfigurationFromDB(self)
Definition: confdb.py:60
def overrideProcessName(self)
Definition: confdb.py:514
def runL1Emulator(self)
Definition: confdb.py:448
def instrumentErrorEventType(self)
Definition: confdb.py:386
def buildOptions(self)
Definition: confdb.py:728
def replace(string, replacements)
def overrideOutput(self)
Definition: confdb.py:458
def dump(self)
Definition: confdb.py:165
def getPathList(self)
Definition: confdb.py:99
def customize(self)
Definition: confdb.py:232
def addEras(self)
Definition: confdb.py:501
def expand_filenames(self, input)
Definition: confdb.py:838
def updateMessageLogger(self)
Definition: confdb.py:534
def overrideL1MenuXml(self)
Definition: confdb.py:438
def overrideGlobalTag(self)
Definition: confdb.py:394
def specificCustomize(self)
Definition: confdb.py:174
def consolidatePositiveList(elements)
Definition: confdb.py:152
def overrideParameters(self, module, parameters)
Definition: confdb.py:574
def instrumentOpenMode(self)
Definition: confdb.py:372
def dumppaths(paths)
Definition: confdb.py:657
def getRawConfigurationFromDB(self)
Definition: confdb.py:82
def loadAdditionalConditions(self, comment, conditions)
Definition: confdb.py:546
def build_source(self)
Definition: confdb.py:850
def loadCffCommand(self, module)
Definition: confdb.py:563
def instrumentDQM(self)
Definition: confdb.py:624
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def loadCff(self, module)
Definition: confdb.py:570
def buildPathList(self)
Definition: confdb.py:663
def fixPrescales(self)
Definition: confdb.py:348
def consolidateNegativeList(elements)
Definition: confdb.py:140
def write(self, setup)
def instrumentTiming(self)
Definition: confdb.py:582
def expandWildcards(globs, collection)
Definition: confdb.py:124
def _fix_parameter(self, args)
Definition: confdb.py:329
def append_filenames(self, name, filenames)
Definition: confdb.py:824
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run