hltFindDuplicates.py

Go to the documentation of this file.

#! /usr/bin/env python

import sys, imp, re, itertools
from HLTrigger.Configuration.Tools.frozendict import frozendict
import FWCore.ParameterSet.Config as cms

debug = False

whitelist_types = [
  'HLTPrescaler',
  'HLTBool', 
]

whitelist_labels = [
  'hltPreExpressSmart',
  'hltPreEventDisplaySmart',
]

def whitelist(module):
  return module.label in whitelist_labels or module.type in whitelist_types


def freeze(arg):
  if type(arg) == dict:
    return frozendict((k, freeze(v)) for (k, v) in arg.iteritems())
  elif '__iter__' in dir(arg):
    return tuple( freeze(v) for v in arg )
  else:
    return arg

def unfreeze(arg):
  if type(arg) == frozendict:
    return dict((k, unfreeze(v)) for (k, v) in arg.iteritems())
  elif '__iter__' in dir(arg):
    return list( unfreeze(v) for v in arg )
  else:
    return arg

def pythonize(arg):
  if 'parameters_' in dir(arg):
    arg = arg.parameters_()

  if 'value' in dir(arg):
    arg = arg.value()

  if type(arg) == dict:
    return frozendict((k, pythonize(v)) for (k, v) in arg.iteritems())
  elif '__iter__' in dir(arg):
    return tuple( pythonize(v) for v in arg )
  else:
    return arg


class Module(object):
  type   = ''
  label  = ''
  params = frozendict()
  hash   = 0

  def __init__(self, module):
    self.label  = module.label_()
    self.type   = module.type_()
    self.params = pythonize(module.parameters_())
    self.__rehash()


  def key(self):
    return self.hash

  def __rehash(self):
    self.hash = (hash(self.type) << 4) + hash(self.params)

  def __check(self, value, group):
    return type(value) is str and bool(group.match(value))

  def __sub(self, value, group, label):
    if type(value) is str:
      return group.sub(r'%s\2' % label, value)
    else:
      return value

  def apply_rename(self, groups):
    modified = False
    newparams = unfreeze(self.params)
    for label, (group, check) in groups.iteritems():
      for k, p in newparams.iteritems():
        if '__iter__' in dir(p):
          if any(self.__check(v, check) for v in p):
            newparams[k] = tuple(self.__sub(v, check, label) for v in p)
            modified = True
        else:
          if self.__check(p, check):
            newparams[k] = self.__sub(p, check, label)
            modified = True

    if modified:
      self.params = frozendict(newparams)
      self.__rehash()




class ModuleList(object):
  modules = []

  def append(self, module):
    m = Module(module)
    if not whitelist(m):
      self.modules.append(m)

  def extend(self, modules):
    for module in modules:
      self.append(module)

  def __init__(self, *args):
    for arg in args:
      if '__iter__' in dir(arg):
        self.extend(arg)
      else:
        self.append(arg)

  def sort(self):
    self.modules.sort(key = Module.key)

  def group(self):
    groups = dict()
    self.sort()
    i = 0
    for v, g in itertools.groupby(self.modules, Module.key):
      group = list(g)
      if len(group) > 1:
        i = i + 1
        g = [ m.label for m in group ]
        g.sort()
        l = 'hltGroup%d' %i
        r = re.compile(r'^(%s)($|:)' % r'|'.join(g))
        groups[l] = (g, r)
    return groups

  def apply_rename(self, groups):
    for module in self.modules:
      module.apply_rename(groups)

  def dump(self):
    for m in self.modules:
      print "%s = (%s) {" % (m.label, m.type)
      for k, v in m.params.iteritems():
        print "\t%s = %s" % (k, v)
      print '}'
      print



def findDuplicates(process):
  modules = ModuleList( 
    process._Process__analyzers.itervalues(), 
    process._Process__producers.itervalues(),
    process._Process__filters.itervalues() 
  )

  oldups = 0
  groups = modules.group()
  dups   = sum(len(g[0]) for g in groups.itervalues()) - len(groups)

  index = 1
  while(dups != oldups):
    if debug:
      dump = open('step%d.sed' % index, 'w')
      for target, (group, regexp) in groups.iteritems():
        dump.write('s#\\<\\(%s\\)\\>#%s#\n' % ('\\|'.join(group), target))
      dump.close()
    print "found %d duplicates" % dups
    oldups = dups
    modules.apply_rename(groups)
    groups = modules.group()
    dups   = sum(len(g[0]) for g in groups.itervalues()) - len(groups)
    index  = index + 1

  dump = open('groups.sed', 'w')
  for target, (group, regexp) in groups.iteritems():
    dump.write('s#\\<\\(%s\\)\\>#%s#\n' % ('\\|'.join(group), target))
  dump.close()

  dump = open('groups.txt', 'w')
  for target, (group, regexp) in groups.iteritems():
    dump.write('#%s\n%s\n\n' % ( target, '\n'.join(group)))
  dump.close()



def main():
  # parse the HLT configuration from standard input or from the given file
  hlt = imp.new_module('hlt')
  try:
    configname = sys.argv[1]
  except:
    config = sys.stdin
  else:
    config = open(configname)
  exec config in globals(), hlt.__dict__
  config.close()
  findDuplicates(hlt.process)


if __name__ == "__main__":
    main()