CMS 3D CMS Logo

dqmMemoryStats.py
Go to the documentation of this file.
1 #!/bin/env python
2 
3 from __future__ import print_function
4 import sys
5 
6 from DQMServices.FileIO.DQM import DQMReader
7 from collections import namedtuple
8 
9 HistogramEntry = namedtuple('HistogramEntry', ['type', 'bin_size', 'bin_count', 'extra', 'total_bytes'])
10 
12  BIN_SIZE = {
13  'TH1S': 2, 'TH2S': 2,
14  'TH1F': 4, 'TH2F': 4,
15  'TH1D': 8, 'TH2D': 8,
16  'TH3F': 4,
17  'TProfile': 8, 'TProfile2D': 8,
18  }
19 
20  def __init__(self):
21  self._all = {}
22 
23  def analyze(self, fn, obj):
24  name = fn.split("/")[-1]
25 
26  if hasattr(obj, 'ClassName'):
27  # this is a root type
28  t = str(obj.ClassName())
29  bin_size = self.BIN_SIZE.get(t, None)
30  if bin_size is None:
31  sys.stderr.write("warning: unknown root type: %s\n" % t)
32  sys.stderr.flush()
33  bin_size = 8
34 
35  bin_count = obj.GetNcells()
36  extra = len(fn)
37  total_bytes = bin_count * bin_size + extra
38 
39  self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
40  else:
41  t = str(type(obj))
42  #bin_count, bin_size, extra = 0, 0, len(str(obj)) + len(fn)
43  # assume constant size for strings
44  bin_count, bin_size, extra = 0, 0, 10 + len(fn)
45  total_bytes = bin_count * bin_size + extra
46 
47  self._all[fn] = HistogramEntry(t, bin_size, bin_count, extra, total_bytes)
48 
49  def group(self, level, countObjects):
50  group_stats = {}
51 
52  for k, v in self._all.items():
53  group_key = "/".join(k.split("/")[:level])
54 
55  current = group_stats.get(group_key, 0)
56  group_stats[group_key] = current + (1 if countObjects else v.total_bytes)
57 
58  return group_stats
59 
60  def difference(self, ref):
61  results = HistogramAnalyzer()
62  results._all = dict(self._all)
63 
64  zero = HistogramEntry("null", 0, 0, 0, 0)
65  def cmp(a, b):
66  return HistogramEntry(b.type, b.bin_size,
67  a.bin_count - b.bin_count,
68  a.extra - b.extra,
69  a.total_bytes - b.total_bytes )
70 
71  for k, refv in ref._all.items():
72  results._all[k] = cmp(self._all.get(k, zero), refv)
73 
74  return results
75 
76 
77 def kibisize(num,args):
78  if args.count:
79  return str(num)
80  pStr="%."+str(args.precision)+"f %s"
81  for prefix in ['KiB','MiB','GiB']:
82  num /= 1024.0
83 
84  if num < 1024.0 or args.units == prefix:
85  return pStr % (num, prefix)
86  return pStr % (num, prefix)
87 
88 def displayDirectoryStatistics(stats, args):
89  group_stats = stats.group(args.depth, args.count)
90 
91  cutoff, display = args.cutoff * 1024, args.display
92 
93  as_list = [(v, k, ) for (k, v) in group_stats.items()]
94  as_list.sort(reverse=True, key=lambda v_k1: abs(v_k1[0]))
95 
96  if cutoff is not None:
97  as_list = [v_k for v_k in as_list if abs(v_k[0]) > cutoff]
98 
99  if display is not None:
100  as_list = as_list[:display]
101 
102  if args.human:
103  print("*" * 80)
104  print((" DQM level %d folder breakdown " % args.depth).center(80, "*"))
105  if cutoff:
106  print(("* Size cutoff: %s" % kibisize(cutoff,args)).ljust(79) + "*")
107  if display:
108  print(("* Showing top %d entries." % display).ljust(79) + "*")
109  print("*" * 80)
110 
111  for v, k in as_list:
112  if args.human:
113  print(kibisize(v,args).ljust(16, " "), k)
114  else:
115  print(v, k)
116 
117 if __name__ == '__main__':
118  import argparse
119 
120  parser = argparse.ArgumentParser()
121  parser.add_argument("-i", "--input", help = "Input DQM ROOT file")
122  parser.add_argument("-r", "--ref", help = "Reference DQM ROOT file (to diff)")
123  parser.add_argument("--summary", help = "Dump summary", action = "store_true")
124  parser.add_argument("--count", help = "Count Histograms", action = "store_true")
125  parser.add_argument("-x", "--human", help = "Human readable output.", action = "store_true")
126  parser.add_argument("-n", "--display", help = "Max entries to display in --summary.", type = int, default = None)
127  parser.add_argument("-c", "--cutoff", help = "Max cutoff to display in --summary.", type = float, default = 512, metavar="KiB")
128  parser.add_argument("-d", "--depth", help = "Folder depth in --summary.", type = int, default = 2)
129  parser.add_argument("-u", "--units", help = "Memory units to use (KiB,MiB,GiB) if fixed output desired", type = str, default = "None")
130  parser.add_argument("-p", "--precision", help = "Places after decimal to display.", type = int, default = 2)
131 
132  args = parser.parse_args()
133 
135  reader = DQMReader(args.input)
136  for (fn, v) in reader.read_objects():
137  stats.analyze(fn, v)
138  reader.close()
139 
140  if args.ref:
141  reader = DQMReader(args.ref)
142  ref_stats = HistogramAnalyzer()
143  for (fn, v) in reader.read_objects():
144  ref_stats.analyze(fn, v)
145  reader.close()
146 
147  stats = stats.difference(ref_stats)
148 
149  if args.summary:
150  displayDirectoryStatistics(stats, args)
151 
152  total = stats.group(0, args.count)
153  if args.human:
154  print("Total bytes: %s" % kibisize(total[""],args))
155  else:
156  print(total[""])
dqmMemoryStats.HistogramAnalyzer.__init__
def __init__(self)
Definition: dqmMemoryStats.py:20
resolutioncreator_cfi.object
object
Definition: resolutioncreator_cfi.py:4
dqmMemoryStats.HistogramAnalyzer._all
_all
Definition: dqmMemoryStats.py:21
dqmMemoryStats.str
str
Definition: dqmMemoryStats.py:129
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
dqmMemoryStats.HistogramEntry
HistogramEntry
Definition: dqmMemoryStats.py:9
dqmMemoryStats.HistogramAnalyzer.group
def group(self, level, countObjects)
Definition: dqmMemoryStats.py:49
dqmMemoryStats.HistogramAnalyzer.BIN_SIZE
BIN_SIZE
Definition: dqmMemoryStats.py:12
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
dqmMemoryStats.displayDirectoryStatistics
def displayDirectoryStatistics(stats, args)
Definition: dqmMemoryStats.py:88
dqmMemoryStats.HistogramAnalyzer.analyze
def analyze(self, fn, obj)
Definition: dqmMemoryStats.py:23
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:46
dqmMemoryStats.HistogramAnalyzer.difference
def difference(self, ref)
Definition: dqmMemoryStats.py:60
dqmMemoryStats.HistogramAnalyzer
Definition: dqmMemoryStats.py:11
fileCollector.cmp
cmp
Definition: fileCollector.py:125
funct::abs
Abs< T >::type abs(const T &t)
Definition: Abs.h:22
dqmMemoryStats.kibisize
def kibisize(num, args)
Definition: dqmMemoryStats.py:77