3 The script compares two ROOT files and fills specified database file with 4 comparison information. 6 Author: Albertas Gimbutas, Vilnius University (LT) 7 e-mail: albertasgim@gmail.com 9 Note: balcklist support not implemented. 11 from __future__
import print_function
14 from datetime
import datetime
15 from multiprocessing
import Pool, Queue, Process
16 from optparse
import OptionParser, OptionGroup
17 from os
import makedirs
18 from os.path
import basename, join, exists
19 from optparse
import OptionParser
22 from Utilities.RelMon.web.app_utils
import get_release, get_stats, get_percentage, get_img_url, get_dataset_name
25 parser = OptionParser(usage=
'Usage: %prog <file1> <file2> --db DB_NAME [options]')
26 parser.add_option(
'--dir', action=
'store', dest=
'dir', default=
'.',
27 help=
'Directory to store static html and .db file.')
28 parser.add_option(
'--db', action=
'store', dest=
'db_name', default=
None, help=
'path to SQLite3 database file.')
29 parser.add_option(
'--st_test', action=
'store', dest=
'st_test', default=
'KS',
30 help=
'Statistical test to use for the comparison.')
31 parser.add_option(
'--th', action=
'store', dest=
'threshold', default=1e-5,
32 help=
'Threshold to use in static HTML. Default: %default.')
33 parser.add_option(
'--html', action=
'store_true', dest=
'html', default=
False,
34 help=
'Generate static html. Default: %default.')
35 parser.add_option(
'--cl', action=
'store_true', dest=
'clear_db', default=
False,
36 help=
'Clear database before using.')
39 def __init__(self, db_name, work_path=None, do_html=False):
44 def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path=''):
45 c.execute(
'''INSERT INTO Directory(name, parent_id) VALUES (?, ?)''',
46 (directory.GetName(), parent_id))
48 from_id, till_id =
None,
None 49 for elem
in directory.GetListOfKeys():
50 elem_name = elem.GetName()
51 subdir = directory.Get(elem_name)
54 subdir_from_id, subdir_till_id, subdir_id = self.
walk_through(c, subdir,
55 f1, f2, st_test, dir_id, path=
join(path, elem_name))
56 if subdir_till_id
and (
not till_id
or subdir_till_id > till_id):
57 till_id = subdir_till_id
58 if subdir_from_id
and (
not from_id
or subdir_from_id < from_id):
59 from_id = subdir_from_id
61 hist1 = f1.Get(
join(directory.GetPath(), elem_name))
62 hist2 = f2.Get(
join(directory.GetPath(), elem_name))
64 p_value = st_test.do_test(hist1, hist2)
65 c.execute(
'''INSERT INTO HistogramComparison(name, p_value, directory_id) 66 VALUES (?, ?, ?)''', (elem_name, p_value, dir_id))
70 if not from_id
or comp_id < from_id:
72 except ComparisonError
as e:
73 print(
'Error comparing %s: %s' % (hist1, e))
74 if from_id
and till_id:
75 c.execute(
'''UPDATE Directory SET from_histogram_id=?, till_histogram_id=? 76 WHERE id=?''', (from_id, till_id, dir_id))
77 return from_id, till_id, dir_id
79 def compare(self, filename1, filename2, st_test):
80 if not 'TFile' in globals():
81 from ROOT
import TFile
85 conn = sqlite3.connect(self.
db_name)
89 dir_DQMData = f1.GetDirectory(
"DQMData")
91 for elem
in dir_DQMData.GetListOfKeys():
92 elem_name = elem.GetName()
93 if elem_name.startswith(
'Run '):
94 dir_Run = dir_DQMData.Get(elem_name)
96 fid, tid, dir_id = self.
walk_through(c, dir_Run, f1, f2, st_test)
98 c.execute(
'''DELETE FROM Directory WHERE from_histogram_id IS NULL 99 AND till_histogram_id IS NULL''')
100 c.execute(
'''INSERT INTO RootFileComparison(filename1, filename2, directory_id) 102 root_file_comparison_id = c.lastrowid
108 return root_file_comparison_id
111 conn = sqlite3.connect(self.
db_name)
113 c.execute(
'''SELECT release_comparison_id FROM RootFileComparison WHERE (filename1=? and filename2=?) 114 OR (filename1=? and filename2=?)''', (filename1, filename2, filename2, filename1))
115 file_comparison = c.fetchall()
117 for release_comparison_id
in file_comparison:
118 c.execute(
'''SELECT statistical_test FROM ReleaseComparison WHERE 119 id = ?''', release_comparison_id)
120 statistical_test = c.fetchone()
121 if statistical_test
and statistical_test[0] == st_test_name:
128 if __name__ ==
'__main__':
129 opts, args = parser.parse_args()
131 parser.error(
'Specify two files to use for the comparison.')
133 parser.error(
'Specify SQLite3 database file for the comparison.')
136 print(
'Clearing DB: %s...' % opts.db_name, end=
' ')
137 open(opts.db_name,
'w').close()
144 file_cmp.compare(args[0], args[1], tests[opts.st_test]())
def was_compared(self, filename1, filename2, st_test_name)
def compare(self, filename1, filename2, st_test)
def walk_through(self, c, directory, f1, f2, st_test, parent_id=None, path='')
S & print(S &os, JobReport::InputFile const &f)
static std::string join(char **cmd)
def init_database(db_path)
Utils.
def dbfile2html(db_name, work_path, threshold=1e-5)
def __init__(self, db_name, work_path=None, do_html=False)