CMS 3D CMS Logo

compare_using_files_v2.py
Go to the documentation of this file.
1 #! /usr/bin/env python
2 """
3 The script compares two ROOT files and fills specified database file with
4 comparison information.
5 
6 Author: Albertas Gimbutas, Vilnius University (LT)
7 e-mail: albertasgim@gmail.com
8 
9 Note: balcklist support not implemented.
10 """
11 from __future__ import print_function
12 import sys
13 import sqlite3
14 from datetime import datetime
15 from multiprocessing import Pool, Queue, Process
16 from optparse import OptionParser, OptionGroup
17 from os import makedirs
18 from os.path import basename, join, exists
19 from optparse import OptionParser
20 
21 from Utilities.RelMon.utils_v2 import ComparisonError, tests, init_database, get_version
22 from Utilities.RelMon.web.app_utils import get_release, get_stats, get_percentage, get_img_url, get_dataset_name
23 
24 
25 parser = OptionParser(usage='Usage: %prog <file1> <file2> --db DB_NAME [options]')
26 parser.add_option('--dir', action='store', dest='dir', default='.',
27  help='Directory to store static html and .db file.')
28 parser.add_option('--db', action='store', dest='db_name', default=None, help='path to SQLite3 database file.')
29 parser.add_option('--st_test', action='store', dest='st_test', default='KS',
30  help='Statistical test to use for the comparison.')
31 parser.add_option('--th', action='store', dest='threshold', default=1e-5,
32  help='Threshold to use in static HTML. Default: %default.')
33 parser.add_option('--html', action='store_true', dest='html', default=False,
34  help='Generate static html. Default: %default.')
35 parser.add_option('--cl', action='store_true', dest='clear_db', default=False,
36  help='Clear database before using.')
37 
39  def __init__(self, db_name, work_path=None, do_html=False):
40  self.db_name = db_name
41  self.work_path = work_path
42  self.do_html = do_html
43 
44  def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path=''):
45  c.execute('''INSERT INTO Directory(name, parent_id) VALUES (?, ?)''',
46  (directory.GetName(), parent_id))
47  dir_id = c.lastrowid
48  from_id, till_id = None, None
49  for elem in directory.GetListOfKeys():
50  elem_name = elem.GetName()
51  subdir = directory.Get(elem_name)
52  if subdir:
53  if subdir.IsFolder():
54  subdir_from_id, subdir_till_id, subdir_id = self.walk_through(c, subdir,
55  f1, f2, st_test, dir_id, path=join(path, elem_name))
56  if subdir_till_id and (not till_id or subdir_till_id > till_id):
57  till_id = subdir_till_id
58  if subdir_from_id and (not from_id or subdir_from_id < from_id):
59  from_id = subdir_from_id
60  else:
61  hist1 = f1.Get(join(directory.GetPath(), elem_name))
62  hist2 = f2.Get(join(directory.GetPath(), elem_name))
63  try:
64  p_value = st_test.do_test(hist1, hist2)
65  c.execute('''INSERT INTO HistogramComparison(name, p_value, directory_id)
66  VALUES (?, ?, ?)''', (elem_name, p_value, dir_id))
67  comp_id = c.lastrowid
68  if comp_id > till_id:
69  till_id = comp_id
70  if not from_id or comp_id < from_id:
71  from_id = comp_id
72  except ComparisonError as e:
73  print('Error comparing %s: %s' % (hist1, e))
74  if from_id and till_id:
75  c.execute('''UPDATE Directory SET from_histogram_id=?, till_histogram_id=?
76  WHERE id=?''', (from_id, till_id, dir_id))
77  return from_id, till_id, dir_id
78 
79  def compare(self, filename1, filename2, st_test):
80  if not 'TFile' in globals():
81  from ROOT import TFile
82  f1 = TFile(filename1)
83  f2 = TFile(filename2)
84 
85  conn = sqlite3.connect(self.db_name)
86  c = conn.cursor()
87 
88  ## Create Directory and HistogramComparison structure in the DB
89  dir_DQMData = f1.GetDirectory("DQMData")
90  dir_Run = None
91  for elem in dir_DQMData.GetListOfKeys():
92  elem_name = elem.GetName()
93  if elem_name.startswith('Run '):
94  dir_Run = dir_DQMData.Get(elem_name)
95 
96  fid, tid, dir_id = self.walk_through(c, dir_Run, f1, f2, st_test)
97 
98  c.execute('''DELETE FROM Directory WHERE from_histogram_id IS NULL
99  AND till_histogram_id IS NULL''')
100  c.execute('''INSERT INTO RootFileComparison(filename1, filename2, directory_id)
101  VALUES (?, ?, ?)''', (basename(filename1), basename(filename2), dir_id))
102  root_file_comparison_id = c.lastrowid
103 
104  conn.commit()
105  conn.close()
106  f1.Close()
107  f2.Close()
108  return root_file_comparison_id
109 
110  def was_compared(self, filename1, filename2, st_test_name):
111  conn = sqlite3.connect(self.db_name)
112  c = conn.cursor()
113  c.execute('''SELECT release_comparison_id FROM RootFileComparison WHERE (filename1=? and filename2=?)
114  OR (filename1=? and filename2=?)''', (filename1, filename2, filename2, filename1))
115  file_comparison = c.fetchall()
116 
117  for release_comparison_id in file_comparison:
118  c.execute('''SELECT statistical_test FROM ReleaseComparison WHERE
119  id = ?''', release_comparison_id)
120  statistical_test = c.fetchone()
121  if statistical_test and statistical_test[0] == st_test_name:
122  conn.close()
123  return True
124  conn.close()
125  return False
126 
127 
128 if __name__ == '__main__':
129  opts, args = parser.parse_args()
130  if len(args) != 2:
131  parser.error('Specify two files to use for the comparison.')
132  if not opts.db_name:
133  parser.error('Specify SQLite3 database file for the comparison.')
134 
135  if opts.clear_db:
136  print('Clearing DB: %s...' % opts.db_name, end=' ')
137  open(opts.db_name, 'w').close()
138  print('Done.')
139 
140  init_database(opts.db_name)
141 
142  print('Comparing files:\n%s\n%s\n' % (basename(args[0]), basename(args[1])))
143  file_cmp = RootFileComparison(opts.db_name, opts.dir)
144  file_cmp.compare(args[0], args[1], tests[opts.st_test]())
145 
146  if opts.html:
147  dbfile2html(opts.db_name, opts.threshold)
def was_compared(self, filename1, filename2, st_test_name)
def compare(self, filename1, filename2, st_test)
def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path='')
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def init_database(db_path)
Utils.
Definition: utils_v2.py:391
def dbfile2html(db_name, work_path, threshold=1e-5)
Definition: dbfile2html.py:36
def __init__(self, db_name, work_path=None, do_html=False)