Functions
def	db_list_with_releases
def	get_dataset_name
def	get_directory_summary_stats
def	get_folders
def	get_img_path
def	get_img_url
def	get_percentage
def	get_release
def	get_release_list
def	get_release_summary_stats
def	get_stats
def	join_ranges
Variables
dictionary	renaming

Function Documentation

def web::app_utils::db_list_with_releases ( path = '.' )

Returns available database list and their releases.

Definition at line 149 of file app_utils.py.

00150                                    :
00151     '''Returns available database list and their releases.'''
00152     db_list = [db for db in listdir(path) if db.endswith('.db')]
00153     db_list_with_releases = []
00154     for db in db_list:
00155         conn = sqlite3.connect(join(path, db))
00156         releases = get_release_list(conn.cursor())
00157         db_list_with_releases.append((db[:-3], releases))
00158         conn.close()
00159     return db_list_with_releases
00160 
00161 # -------------------     Template Context generators     --------------------

def web::app_utils::get_dataset_name ( name )

Returns extracted dataset name from the given ROOT filename.

Definition at line 66 of file app_utils.py.

00067                           :
00068     '''Returns extracted dataset name from the given ROOT filename.'''
00069     if re.search('RelVal', name):
00070         run = str(int(re.findall('_R(\d{9})_', name)[0]))
00071         ds = re.findall('GR_R_\d*_V\d*C?_(?:RelVal)?_([\w\d]*-v\d+)_', name)[0]
00072     else:
00073         run, ds = re.findall('R(\d{9})__([\w\d]*)__CMSSW_', name)[0:1]
00074     return '_'.join([ds, str(int(run))])
00075

def web::app_utils::get_directory_summary_stats	(	c,
		url_args,
		file_id,
		threshold
	)

Returns context for ``directory_summary.html`` template.

Definition at line 282 of file app_utils.py.

00283                                                                 :
00284     '''Returns context for ``directory_summary.html`` template.'''
00285     context = dict()
00286     c.execute('''SELECT directory_id, filename1, filename2 FROM RootFileComparison
00287                  WHERE id = ?''', (file_id,))
00288     dir_id, f1, f2 = c.fetchone()
00289     context['release1'] = get_release(f1)
00290     context['release2'] = get_release(f2)
00291     if not url_args:
00292         dir_name = get_dataset_name(f1)
00293     else:
00294         #### Select DQMData/Run directory.
00295         directory_names = []
00296 
00297         for dir_name in url_args:
00298             c.execute('''SELECT id, name FROM Directory WHERE name = ? AND
00299                     parent_id = ?''', (dir_name, dir_id))
00300             dir_id, name = c.fetchone()
00301             directory_names.append(name)
00302         context['parent_name'] = '/'.join(directory_names)
00303 
00304     ## Select stats
00305     c.execute('''SELECT from_histogram_id, till_histogram_id FROM
00306                  Directory WHERE id = ?''', (dir_id,))
00307     ranges = c.fetchone()
00308     successes, nulls, fails = get_stats(c, threshold, (ranges,))
00309     success, null, fail = get_percentage(successes, nulls, fails)
00310     context.update({
00311             'successes': successes, 'nulls': nulls, 'fails': fails,
00312             'success': success, 'null': null, 'fail': fail,
00313             'total': successes + nulls + fails, 'dir_name': dir_name
00314         })
00315     # subdirs: name, total, success, fail, null
00316     c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM Directory
00317                  WHERE parent_id = ?''', (dir_id,))
00318     subdirs = c.fetchall()
00319     subdir_stats = []
00320     for name, from_id, till_id in subdirs:
00321         successes, nulls, fails = get_stats(c, threshold, [(from_id, till_id,)])
00322         success, null, fail = get_percentage(successes, nulls, fails)
00323         subdir_stats.append((name, successes + nulls + fails, successes,
00324                              nulls, fails, success, null, fail))
00325     context['subdirs'] = sorted(subdir_stats, key=lambda x: x[4], reverse=True)
00326 
00327     # histograms: name, p_value
00328     c.execute('''SELECT name, p_value FROM HistogramComparison
00329                  WHERE directory_id = ?''', (dir_id,))
00330     failed_histos = []
00331     successful_histos = []
00332     null_histos = []
00333     for name, p_value in c.fetchall():
00334         path = quote('%s/%s' % ('/'.join(url_args), name))
00335         url1 = get_img_url(path, f1)
00336         url2 = get_img_url(path, f2)
00337         overlay = get_img_url(path, f1, f2)
00338         if p_value < 0:
00339             null_histos.append((name, p_value, url1, url2, overlay))
00340         elif p_value <= threshold:
00341             failed_histos.append((name, p_value, url1, url2, overlay))
00342         else:
00343             successful_histos.append((name, p_value, url1, url2, overlay))
00344 
00345     context['failed_histos'] = sorted(failed_histos, key=lambda x: x[1], reverse=True)
00346     context['null_histos'] = null_histos
00347     context['successful_histos'] = sorted(successful_histos, key=lambda x: x[1], reverse=True)
00348     return context

def web::app_utils::get_folders	(	c,
		file_id,
		filename,
		dir_id,
		threshold
	)

Returns file folder stats for one "summary table" column.

Definition at line 112 of file app_utils.py.

00112                                                         :  # TODO: If folder [Egamma|JetMet] analyse their subdirs
00113     '''Returns file folder stats for one "summary table" column.'''
00114     ds_name = get_dataset_name(filename)
00115     c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM
00116                  Directory WHERE parent_id=?''', (dir_id,))
00117     dirs = c.fetchall()
00118     file_folders = dict()
00119     total_successes, total_nulls, total_fails = 0, 0, 0
00120     for name, from_id, till_id in dirs:
00121         successes, nulls, fails = get_stats(c, threshold, ((from_id, till_id),))
00122         total_successes += successes
00123         total_nulls += nulls
00124         total_fails += fails
00125         if file_folders.has_key(name):
00126             file_folders[name].append([file_id, ds_name, successes, nulls, fails])
00127         else:
00128             file_folders[name] = [file_id, ds_name, successes, nulls, fails]
00129     return [('Summary', [file_id, ds_name, total_successes, total_nulls, total_fails])] + file_folders.items()
00130 
00131

def web::app_utils::get_img_path	(	filename,
		path
	)

Returns image path for https://cmsweb.cern.ch/dqm histogram
visualisation service

Definition at line 47 of file app_utils.py.

00048                                 :
00049     '''Returns image path for https://cmsweb.cern.ch/dqm histogram
00050     visualisation service'''
00051     run = int(re.findall('_R(\d*)__', filename)[0])
00052     parts = [e.rstrip('.root') for e in filename.split('__')]
00053     path = path.replace('Run summary/', '')
00054     return 'archive/%s/%s/%s/%s/%s' % (run, parts[1], parts[2], parts[3], path)
00055

def web::app_utils::get_img_url	(	path,
		f1,
		f2 = `None`,
		w = `250`,
		h = `250`
	)

Returns full URL of histogram (or histogram overlay) image for
https://cmsweb.cern.ch/dqm visualisation service.

Definition at line 56 of file app_utils.py.

00057                                                 :
00058     '''Returns full URL of histogram (or histogram overlay) image for
00059     https://cmsweb.cern.ch/dqm visualisation service.'''
00060     base = 'https://cmsweb.cern.ch/dqm/relval/plotfairy'
00061     if not f2:
00062         return '%s/%s?w=%s;h=%s' % (base, get_img_path(f1, path), w, h)
00063     return '%s/overlay?obj=%s;obj=%s;w=%s;h=%s' % (base,
00064                  get_img_path(f1, path), get_img_path(f2, path), w, h)
00065

def web::app_utils::get_percentage	(	successes,
		nulls,
		fails
	)

Converts integers ``successes``, ``nulls`` and ``fails`` to percents.

Definition at line 99 of file app_utils.py.

00100                                            :
00101     '''Converts integers ``successes``, ``nulls`` and ``fails`` to percents.'''
00102     if successes is None:
00103         return None, None, None
00104     total = successes + fails + nulls
00105     if not total:
00106         return None, None, None
00107     success =  round(100. * successes / total, 2)
00108     null =  round(100. * nulls / total, 2)
00109     fail =  round(100. * fails / total, 2)
00110     return success, null, fail
00111

def web::app_utils::get_release ( name )

Returns extracted release from the given ROOT filename.

Definition at line 76 of file app_utils.py.

00077                      :
00078     '''Returns extracted release from the given ROOT filename.'''
00079     return re.findall('R\d{9}__([\w\d_-]*)__DQM.root', name)[0]
00080

def web::app_utils::get_release_list ( c )

Returns all ``ReleaseComparisons`` found on database.

Definition at line 143 of file app_utils.py.

00144                        :
00145     '''Returns all ``ReleaseComparisons`` found on database.'''
00146     c.execute('SELECT title, statistical_test FROM ReleaseComparison')
00147     return c.fetchall()
00148

def web::app_utils::get_release_summary_stats	(	c,
		release_title,
		st_test,
		threshold = `1e-5`
	)

Returns context for ``release_summary.html`` template.

Definition at line 162 of file app_utils.py.

00163                                                                         :
00164     '''Returns context for ``release_summary.html`` template.'''
00165     ## Summary
00166     context = dict()
00167     c.execute('''SELECT release1, release2, id FROM ReleaseComparison
00168                  WHERE title = ? AND statistical_test = ?''', (release_title, st_test))
00169     context['release1'], context['release2'], release_comp_id = c.fetchone()
00170 
00171     # All directory ranges
00172     c.execute('''SELECT from_histogram_id, till_histogram_id FROM Directory
00173                    WHERE id IN (SELECT directory_id FROM RootFileComparison
00174                    WHERE release_comparison_id = ?)''', (release_comp_id,))
00175     dir_ranges = c.fetchall()
00176 
00177     if len(dir_ranges) > 1:
00178         dir_ranges = reduce(join_ranges, dir_ranges)
00179 
00180     context['successes'], context['nulls'], context['fails'], = get_stats(c, threshold, dir_ranges)
00181 
00182     context['total'] = context['successes'] + context['fails'] + context['nulls']
00183     if context['total']:
00184         context['success'], context['null'], context['fail'] = \
00185             get_percentage(context['successes'], context['nulls'], context['fails'])
00186 
00187     ## Data needed for the all the statistics:
00188     c.execute('''SELECT id, filename1, directory_id FROM RootFileComparison
00189                  WHERE release_comparison_id = ?''', (release_comp_id,))
00190     files = c.fetchall()
00191 
00192     ## folders: [(folder_name, [folder: (file_id, filename, success, null, fail)]), ...]
00193     folders = dict()
00194     for file_id, filename, dir_id in files:
00195         # file_folders: [(folder_name, [(file_id, file_name, success, null, fail)]), ...]
00196         file_folders = get_folders(c, file_id, filename, dir_id, threshold)
00197         for folder_name, file_folder_stats in file_folders:
00198             if folders.has_key(folder_name):
00199                 # Add folder stats
00200                 folders[folder_name].append(file_folder_stats)
00201                 # Update folder summary
00202                 folders[folder_name][0][2] += file_folder_stats[2]
00203                 folders[folder_name][0][3] += file_folder_stats[3]
00204                 folders[folder_name][0][4] += file_folder_stats[4]
00205             else:
00206                 folder_summary = [None, 'Summary', file_folder_stats[2],
00207                                     file_folder_stats[3], file_folder_stats[4]]
00208                 folders[folder_name] = [folder_summary, file_folder_stats]
00209 
00210     ## Calculate ratios
00211     folders = [('Summary', folders.pop('Summary'))] + sorted(folders.items(), key=lambda x: x[0])
00212     for folder, file_stats in folders:
00213         # Insert N/A if histo is missing
00214         if len(file_stats) != len(files)+1:
00215             for i, file_ in enumerate(files):
00216                 if file_[0] != file_stats[i][0]:
00217                     file_stats = file_stats[:i] + [[None, "N/A", None, None, None]] + file_stats[i:]
00218         # Count the ratios
00219         for i, stats in enumerate(file_stats):
00220             stats[2], stats[3], stats[4] = get_percentage(*stats[2:5])
00221     context['folders'] = folders
00222 
00223 
00224     ## Select Summary Barchart, Detailed Barchart
00225     for folder in folders:
00226         print folder
00227     #   detailed_ratios: (name, success_ratio)
00228     #   summary_ratios: (name, success_ratio)
00229 
00230 
00231     ## Summary Barchart
00232     # TODO: optimise not to fetch from DB again.
00233     c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM Directory
00234                  WHERE parent_id IN (SELECT directory_id FROM RootFileComparison
00235                  WHERE release_comparison_id = ?)''', (release_comp_id,))
00236     lvl3_dir_ranges = c.fetchall()
00237 
00238     cum_lvl3_dir_ranges = dict()
00239     for name, from_id, till_id in lvl3_dir_ranges:
00240         if cum_lvl3_dir_ranges.has_key(name):
00241             cum_lvl3_dir_ranges[name].append((from_id, till_id))
00242         else:
00243             cum_lvl3_dir_ranges[name] = [(from_id, till_id)]
00244 
00245     # Fetch stats
00246     summary_stats = dict()
00247     detailed_stats = dict()
00248     for name, ranges in cum_lvl3_dir_ranges.iteritems():
00249         successes, nulls, fails = get_stats(c, threshold, ranges)
00250         if detailed_stats.has_key(name):
00251             detailed_stats[name][0] += successes
00252             detailed_stats[name][1] += nulls
00253             detailed_stats[name][2] += fails
00254         else:
00255             detailed_stats[name] = [successes, nulls, fails]
00256         if renaming.has_key(name):
00257             if summary_stats.has_key(renaming[name]):
00258                 summary_stats[renaming[name]][0] += successes
00259                 summary_stats[renaming[name]][1] += nulls
00260                 summary_stats[renaming[name]][2] += fails
00261             else:
00262                 summary_stats[renaming[name]] = [successes, nulls, fails]
00263 
00264     # Calculate ratio
00265     summary_ratios = []
00266     for name, stats in summary_stats.iteritems():
00267         total = sum(stats)
00268         if total:
00269             ratio = float(stats[0]) / sum(stats)
00270             summary_ratios.append((name, ratio))
00271     detailed_ratios = []
00272     for name, stats in detailed_stats.iteritems():
00273         total = sum(stats)
00274         if total:
00275             ratio = float(stats[0]) / sum(stats)
00276             detailed_ratios.append((name, ratio))
00277 
00278     context['summary_ratios'] = sorted(summary_ratios, key=lambda x: x[0])
00279     context['detailed_ratios'] = sorted(detailed_ratios, key=lambda x: x[0])
00280     return context
00281

def web::app_utils::get_stats	(	c,
		threshold,
		dir_ranges
	)

Returns ``successes``, ``fails``, ``nulls`` for the given dir_ranges.

Definition at line 81 of file app_utils.py.

00082                                        :
00083     '''Returns ``successes``, ``fails``, ``nulls`` for the given dir_ranges.'''
00084     successes, nulls, fails = 0, 0, 0
00085     for from_id, till_id in dir_ranges:
00086         c.execute('''SELECT count(*) FROM HistogramComparison
00087                      WHERE p_value >= 0 AND p_value > ? AND
00088                      id >= ? and id <= ?''', (threshold, from_id, till_id))
00089         successes += c.fetchone()[0]
00090         c.execute('''SELECT count(*) FROM HistogramComparison WHERE
00091                      p_value < 0 AND id >= ? AND id <= ?''', (from_id, till_id))
00092         nulls += c.fetchone()[0]
00093         c.execute('''SELECT count(*) FROM HistogramComparison
00094                      WHERE p_value >= 0 AND p_value <= ? AND
00095                      id >= ? AND id <= ?''', (threshold, from_id, till_id))
00096         fails += c.fetchone()[0]
00097     return successes, nulls, fails
00098

def web::app_utils::join_ranges	(	ranges,
		elem
	)

To do less DB calls, joins [(from_id, till_id), ...] ranges.

Definition at line 132 of file app_utils.py.

00133                              :
00134     '''To do less DB calls, joins [(from_id, till_id), ...] ranges.'''
00135     if type(ranges) == tuple:
00136         ranges = [ranges]
00137     if ranges[-1][-1] + 1 == elem[0]:
00138         ranges[-1] = (ranges[-1][0], elem[1])
00139     else:
00140         ranges.append(elem)
00141     return ranges
00142

Variable Documentation

dictionary web::app_utils::renaming

Initial value:

00001 {
00002         'MessageLogger': 'Miscellanea', 'FourVector': 'Generic',
00003         'Castor': 'Castor Calorimeter', 'RPCDigisV': 'Resistive Plate Chambers',
00004         'GlobalRecHitsV': 'Miscellanea: Sim.', 'Top': 'Top', 'HLTJETMET': 'JetMet',
00005         'GlobalDigisV': 'Miscellanea: Sim.', 'L1TEMU': 'Level 1 Trigger',
00006         'TrackerRecHitsV': 'Tracking System', 'MuonDTHitsV': 'Muon Objects',
00007         'EcalDigisV': 'Ecal Calorimeter', 'EcalHitsV': 'Ecal Calorimeter',
00008         'Muons': 'Muon Objects', 'DT': 'Drift Tubes', 'TrackerDigisV': 'Tracking System',
00009         'Pixel': 'Tracking System', 'EcalPreshower': 'Ecal Calorimeter',
00010         'EgammaV': 'Photons', 'AlCaEcalPi0': 'Alca', 'SusyExo': 'SusyExo',
00011         'MuonDTDigisV': 'Muon Objects', 'TauRelVal': 'Tau',
00012         'HcalHitsV': 'Hcal Calorimeter', 'RPC': 'Resistive Plate Chambers',
00013         'EcalRecHitsV': 'Ecal Calorimeter', 'EgOffline': 'EGamma',
00014         'MuonCSCDigisV': 'Muon Objects', 'ParticleFlow': 'Miscellanea',
00015         'Info': 'Miscellanea', 'Tracking': 'Tracking',
00016         'NoiseRatesV': 'Miscellanea: Sim.', 'Generator': 'Miscellanea: Sim.',
00017         'Btag': 'B Tagging', 'Higgs': 'Higgs', 'GlobalHitsV': 'Miscellanea: Sim.',
00018         'HcalRecHitsV': 'Hcal Calorimeter', 'TrackerHitsV': 'Tracking System',
00019         'CSC': 'Cathode Strip Chambers', 'Muon,HLTMonMuon': 'Muon',
00020         'Hcal': 'Hcal Calorimeter', 'TauOffline': 'Tau',
00021         'HeavyFlavor': 'HeavyFlavor', 'JetMET': 'Jet', 'Physics': 'Miscellanea',
00022         'CaloTowersV': 'Hcal Calorimeter', 'SiStrip': 'Tracking System',
00023         'EcalClusterV': 'Ecal Calorimeter', 'HLTEgammaValidation': 'EGamma',
00024         'EcalPhiSym': 'Alca', 'L1T': 'Level 1 Trigger', 'MixingV': 'Miscellanea: Sim.',
00025         'FourVector_Val': 'Generic', 'EcalEndcap': 'Ecal Calorimeter',
00026         'TauOnline': 'Tau', 'Egamma': 'Photons', 'HcalIsoTrack': 'Alca',
00027         'EcalBarrel': 'Ecal Calorimeter'
00028 }

Definition at line 17 of file app_utils.py.

web::app_utils Namespace Reference

Functions

Variables

Function Documentation

Variable Documentation