web.app_utils Namespace Reference

Functions
def	db_list_with_releases (path='.')
def	get_dataset_name (name)
def	get_directory_summary_stats (c, url_args, file_id, threshold)
def	get_folders (c, file_id, filename, dir_id, threshold)
def	get_img_path (filename, path)
def	get_img_url (path, f1, f2=None, w=250, h=250)
def	get_percentage (successes, nulls, fails)
def	get_release (name)
def	get_release_list (c)
def	get_release_summary_stats (c, release_title, st_test, threshold=1e-5)
def	get_stats (c, threshold, dir_ranges)
def	join_ranges (ranges, elem)

Variables
	renaming

Function Documentation

def web.app_utils.db_list_with_releases

(

path = '.'

)

Returns available database list and their releases.

Definition at line 150 of file app_utils.py.

References web.app_utils.get_release_list(), and join().

Referenced by web.dbfile2html.dbfile2html(), and web.browse_db.BrowseDB.default().

def db_list_with_releases(path='.'):
    '''Returns available database list and their releases.'''
    db_list = [db for db in listdir(path) if db.endswith('.db')]
    db_list_with_releases = []
    for db in db_list:
        conn = sqlite3.connect(join(path, db))
        releases = get_release_list(conn.cursor())
        db_list_with_releases.append((db[:-3], releases))
        conn.close()
    return db_list_with_releases

# -------------------     Template Context generators     --------------------

def web.app_utils.get_dataset_name

(

name

)

Returns extracted dataset name from the given ROOT filename.

Definition at line 67 of file app_utils.py.

References createfilelist.int, join(), and harvestTrackValidationPlots.str.

Referenced by web.app_utils.get_directory_summary_stats(), and web.app_utils.get_folders().

def get_dataset_name(name):
    '''Returns extracted dataset name from the given ROOT filename.'''
    if re.search('RelVal', name):
        run = str(int(re.findall('_R(\d{9})_', name)[0]))
        ds = re.findall('GR_R_\d*_V\d*C?_(?:RelVal)?_([\w\d]*-v\d+)_', name)[0]
    else:
        run, ds = re.findall('R(\d{9})__([\w\d]*)__CMSSW_', name)[0:1]
    return '_'.join([ds, str(int(run))])

def web.app_utils.get_directory_summary_stats	(		c,
			url_args,
			file_id,
			threshold
	)

Returns context for ``directory_summary.html`` template.

Definition at line 283 of file app_utils.py.

References cmsPerfStripChart.dict, web.app_utils.get_dataset_name(), web.app_utils.get_img_url(), web.app_utils.get_percentage(), web.app_utils.get_release(), web.app_utils.get_stats(), and join().

Referenced by web.dbfile2html.dbfile2html(), and web.browse_db.BrowseDB.default().

def get_directory_summary_stats(c, url_args, file_id, threshold):
    '''Returns context for ``directory_summary.html`` template.'''
    context = dict()
    c.execute('''SELECT directory_id, filename1, filename2 FROM RootFileComparison
                 WHERE id = ?''', (file_id,))
    dir_id, f1, f2 = c.fetchone()
    context['release1'] = get_release(f1)
    context['release2'] = get_release(f2)
    if not url_args:
        dir_name = get_dataset_name(f1)
    else:
        #### Select DQMData/Run directory.
        directory_names = []

        for dir_name in url_args:
            c.execute('''SELECT id, name FROM Directory WHERE name = ? AND
                    parent_id = ?''', (dir_name, dir_id))
            dir_id, name = c.fetchone()
            directory_names.append(name)
        context['parent_name'] = '/'.join(directory_names)

    ## Select stats
    c.execute('''SELECT from_histogram_id, till_histogram_id FROM
                 Directory WHERE id = ?''', (dir_id,))
    ranges = c.fetchone()
    successes, nulls, fails = get_stats(c, threshold, (ranges,))
    success, null, fail = get_percentage(successes, nulls, fails)
    context.update({
            'successes': successes, 'nulls': nulls, 'fails': fails,
            'success': success, 'null': null, 'fail': fail,
            'total': successes + nulls + fails, 'dir_name': dir_name
        })
    # subdirs: name, total, success, fail, null
    c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM Directory
                 WHERE parent_id = ?''', (dir_id,))
    subdirs = c.fetchall()
    subdir_stats = []
    for name, from_id, till_id in subdirs:
        successes, nulls, fails = get_stats(c, threshold, [(from_id, till_id,)])
        success, null, fail = get_percentage(successes, nulls, fails)
        subdir_stats.append((name, successes + nulls + fails, successes,
                             nulls, fails, success, null, fail))
    context['subdirs'] = sorted(subdir_stats, key=lambda x: x[4], reverse=True)

    # histograms: name, p_value
    c.execute('''SELECT name, p_value FROM HistogramComparison
                 WHERE directory_id = ?''', (dir_id,))
    failed_histos = []
    successful_histos = []
    null_histos = []
    for name, p_value in c.fetchall():
        path = quote('%s/%s' % ('/'.join(url_args), name))
        url1 = get_img_url(path, f1)
        url2 = get_img_url(path, f2)
        overlay = get_img_url(path, f1, f2)
        if p_value < 0:
            null_histos.append((name, p_value, url1, url2, overlay))
        elif p_value <= threshold:
            failed_histos.append((name, p_value, url1, url2, overlay))
        else:
            successful_histos.append((name, p_value, url1, url2, overlay))

    context['failed_histos'] = sorted(failed_histos, key=lambda x: x[1], reverse=True)
    context['null_histos'] = null_histos
    context['successful_histos'] = sorted(successful_histos, key=lambda x: x[1], reverse=True)
    return context

def web.app_utils.get_folders	(		c,
			file_id,
			filename,
			dir_id,
			threshold
	)

Returns file folder stats for one "summary table" column.

Definition at line 113 of file app_utils.py.

References mps_alisetup.append, cmsPerfStripChart.dict, web.app_utils.get_dataset_name(), and web.app_utils.get_stats().

Referenced by web.app_utils.get_release_summary_stats().

def get_folders(c, file_id, filename, dir_id, threshold):  # TODO: If folder [Egamma|JetMet] analyse their subdirs
    '''Returns file folder stats for one "summary table" column.'''
    ds_name = get_dataset_name(filename)
    c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM
                 Directory WHERE parent_id=?''', (dir_id,))
    dirs = c.fetchall()
    file_folders = dict()
    total_successes, total_nulls, total_fails = 0, 0, 0
    for name, from_id, till_id in dirs:
        successes, nulls, fails = get_stats(c, threshold, ((from_id, till_id),))
        total_successes += successes
        total_nulls += nulls
        total_fails += fails
        if name in file_folders:
            file_folders[name].append([file_id, ds_name, successes, nulls, fails])
        else:
            file_folders[name] = [file_id, ds_name, successes, nulls, fails]
    return [('Summary', [file_id, ds_name, total_successes, total_nulls, total_fails])] + file_folders.items()

def web.app_utils.get_img_path	(		filename,
			path
	)

Returns image path for https://cmsweb.cern.ch/dqm histogram
visualisation service

Definition at line 48 of file app_utils.py.

References createfilelist.int.

Referenced by web.app_utils.get_img_url().

def get_img_path(filename, path):
    '''Returns image path for https://cmsweb.cern.ch/dqm histogram
    visualisation service'''
    run = int(re.findall('_R(\d*)__', filename)[0])
    parts = [e.rstrip('.root') for e in filename.split('__')]
    path = path.replace('Run summary/', '')
    return 'archive/%s/%s/%s/%s/%s' % (run, parts[1], parts[2], parts[3], path)

def web.app_utils.get_img_url	(		path,
			f1,
			f2 = None,
			w = 250,
			h = 250
	)

Returns full URL of histogram (or histogram overlay) image for
https://cmsweb.cern.ch/dqm visualisation service.

Definition at line 57 of file app_utils.py.

References web.app_utils.get_img_path().

Referenced by web.app_utils.get_directory_summary_stats().

def get_img_url(path, f1, f2=None, w=250, h=250):
    '''Returns full URL of histogram (or histogram overlay) image for
    https://cmsweb.cern.ch/dqm visualisation service.'''
    base = 'https://cmsweb.cern.ch/dqm/relval/plotfairy'
    if not f2:
        return '%s/%s?w=%s;h=%s' % (base, get_img_path(f1, path), w, h)
    return '%s/overlay?obj=%s;obj=%s;w=%s;h=%s' % (base,
                 get_img_path(f1, path), get_img_path(f2, path), w, h)

def web.app_utils.get_percentage	(		successes,
			nulls,
			fails
	)

Converts integers ``successes``, ``nulls`` and ``fails`` to percents.

Definition at line 100 of file app_utils.py.

Referenced by web.app_utils.get_directory_summary_stats(), and web.app_utils.get_release_summary_stats().

def get_percentage(successes, nulls, fails):
    '''Converts integers ``successes``, ``nulls`` and ``fails`` to percents.'''
    if successes is None:
        return None, None, None
    total = successes + fails + nulls
    if not total:
        return None, None, None
    success =  round(100. * successes / total, 2)
    null =  round(100. * nulls / total, 2)
    fail =  round(100. * fails / total, 2)
    return success, null, fail

def web.app_utils.get_release

(

name

)

Returns extracted release from the given ROOT filename.

Definition at line 77 of file app_utils.py.

Referenced by web.app_utils.get_directory_summary_stats().

def get_release(name):
    '''Returns extracted release from the given ROOT filename.'''
    return re.findall('R\d{9}__([\w\d_-]*)__DQM.root', name)[0]

def web.app_utils.get_release_list

(

c

)

Returns all ``ReleaseComparisons`` found on database.

Definition at line 144 of file app_utils.py.

Referenced by web.app_utils.db_list_with_releases(), and web.browse_db.BrowseDB.default().

def get_release_list(c):
    '''Returns all ``ReleaseComparisons`` found on database.'''
    c.execute('SELECT title, statistical_test FROM ReleaseComparison')
    return c.fetchall()

def web.app_utils.get_release_summary_stats	(		c,
			release_title,
			st_test,
			threshold = 1e-5
	)

Returns context for ``release_summary.html`` template.

Definition at line 163 of file app_utils.py.

References mps_alisetup.append, cmsPerfStripChart.dict, objects.autophobj.float, web.app_utils.get_folders(), web.app_utils.get_percentage(), and web.app_utils.get_stats().

Referenced by web.dbfile2html.dbfile2html(), and web.browse_db.BrowseDB.default().

def get_release_summary_stats(c, release_title, st_test, threshold=1e-5):
    '''Returns context for ``release_summary.html`` template.'''
    ## Summary
    context = dict()
    c.execute('''SELECT release1, release2, id FROM ReleaseComparison
                 WHERE title = ? AND statistical_test = ?''', (release_title, st_test))
    context['release1'], context['release2'], release_comp_id = c.fetchone()

    # All directory ranges
    c.execute('''SELECT from_histogram_id, till_histogram_id FROM Directory
                   WHERE id IN (SELECT directory_id FROM RootFileComparison
                   WHERE release_comparison_id = ?)''', (release_comp_id,))
    dir_ranges = c.fetchall()

    if len(dir_ranges) > 1:
        dir_ranges = reduce(join_ranges, dir_ranges)

    context['successes'], context['nulls'], context['fails'], = get_stats(c, threshold, dir_ranges)

    context['total'] = context['successes'] + context['fails'] + context['nulls']
    if context['total']:
        context['success'], context['null'], context['fail'] = \
            get_percentage(context['successes'], context['nulls'], context['fails'])

    ## Data needed for the all the statistics:
    c.execute('''SELECT id, filename1, directory_id FROM RootFileComparison
                 WHERE release_comparison_id = ?''', (release_comp_id,))
    files = c.fetchall()

    ## folders: [(folder_name, [folder: (file_id, filename, success, null, fail)]), ...]
    folders = dict()
    for file_id, filename, dir_id in files:
        # file_folders: [(folder_name, [(file_id, file_name, success, null, fail)]), ...]
        file_folders = get_folders(c, file_id, filename, dir_id, threshold)
        for folder_name, file_folder_stats in file_folders:
            if folder_name in folders:
                # Add folder stats
                folders[folder_name].append(file_folder_stats)
                # Update folder summary
                folders[folder_name][0][2] += file_folder_stats[2]
                folders[folder_name][0][3] += file_folder_stats[3]
                folders[folder_name][0][4] += file_folder_stats[4]
            else:
                folder_summary = [None, 'Summary', file_folder_stats[2],
                                    file_folder_stats[3], file_folder_stats[4]]
                folders[folder_name] = [folder_summary, file_folder_stats]

    ## Calculate ratios
    folders = [('Summary', folders.pop('Summary'))] + sorted(folders.items(), key=lambda x: x[0])
    for folder, file_stats in folders:
        # Insert N/A if histo is missing
        if len(file_stats) != len(files)+1:
            for i, file_ in enumerate(files):
                if file_[0] != file_stats[i][0]:
                    file_stats = file_stats[:i] + [[None, "N/A", None, None, None]] + file_stats[i:]
        # Count the ratios
        for i, stats in enumerate(file_stats):
            stats[2], stats[3], stats[4] = get_percentage(*stats[2:5])
    context['folders'] = folders


    ## Select Summary Barchart, Detailed Barchart
    for folder in folders:
        print folder
    #   detailed_ratios: (name, success_ratio)
    #   summary_ratios: (name, success_ratio)


    ## Summary Barchart
    # TODO: optimise not to fetch from DB again.
    c.execute('''SELECT name, from_histogram_id, till_histogram_id FROM Directory
                 WHERE parent_id IN (SELECT directory_id FROM RootFileComparison
                 WHERE release_comparison_id = ?)''', (release_comp_id,))
    lvl3_dir_ranges = c.fetchall()

    cum_lvl3_dir_ranges = dict()
    for name, from_id, till_id in lvl3_dir_ranges:
        if name in cum_lvl3_dir_ranges:
            cum_lvl3_dir_ranges[name].append((from_id, till_id))
        else:
            cum_lvl3_dir_ranges[name] = [(from_id, till_id)]

    # Fetch stats
    summary_stats = dict()
    detailed_stats = dict()
    for name, ranges in cum_lvl3_dir_ranges.iteritems():
        successes, nulls, fails = get_stats(c, threshold, ranges)
        if name in detailed_stats:
            detailed_stats[name][0] += successes
            detailed_stats[name][1] += nulls
            detailed_stats[name][2] += fails
        else:
            detailed_stats[name] = [successes, nulls, fails]
        if name in renaming:
            if renaming[name] in summary_stats:
                summary_stats[renaming[name]][0] += successes
                summary_stats[renaming[name]][1] += nulls
                summary_stats[renaming[name]][2] += fails
            else:
                summary_stats[renaming[name]] = [successes, nulls, fails]

    # Calculate ratio
    summary_ratios = []
    for name, stats in summary_stats.iteritems():
        total = sum(stats)
        if total:
            ratio = float(stats[0]) / sum(stats)
            summary_ratios.append((name, ratio))
    detailed_ratios = []
    for name, stats in detailed_stats.iteritems():
        total = sum(stats)
        if total:
            ratio = float(stats[0]) / sum(stats)
            detailed_ratios.append((name, ratio))

    context['summary_ratios'] = sorted(summary_ratios, key=lambda x: x[0])
    context['detailed_ratios'] = sorted(detailed_ratios, key=lambda x: x[0])
    return context

def web.app_utils.get_stats	(		c,
			threshold,
			dir_ranges
	)

Returns ``successes``, ``fails``, ``nulls`` for the given dir_ranges.

Definition at line 82 of file app_utils.py.

Referenced by web.app_utils.get_directory_summary_stats(), web.app_utils.get_folders(), and web.app_utils.get_release_summary_stats().

def get_stats(c, threshold, dir_ranges):
    '''Returns ``successes``, ``fails``, ``nulls`` for the given dir_ranges.'''
    successes, nulls, fails = 0, 0, 0
    for from_id, till_id in dir_ranges:
        c.execute('''SELECT count(*) FROM HistogramComparison
                     WHERE p_value >= 0 AND p_value > ? AND
                     id >= ? and id <= ?''', (threshold, from_id, till_id))
        successes += c.fetchone()[0]
        c.execute('''SELECT count(*) FROM HistogramComparison WHERE
                     p_value < 0 AND id >= ? AND id <= ?''', (from_id, till_id))
        nulls += c.fetchone()[0]
        c.execute('''SELECT count(*) FROM HistogramComparison
                     WHERE p_value >= 0 AND p_value <= ? AND
                     id >= ? AND id <= ?''', (threshold, from_id, till_id))
        fails += c.fetchone()[0]
    return successes, nulls, fails

def web.app_utils.join_ranges	(		ranges,
			elem
	)

To do less DB calls, joins [(from_id, till_id), ...] ranges.

Definition at line 133 of file app_utils.py.

def join_ranges(ranges, elem):
    '''To do less DB calls, joins [(from_id, till_id), ...] ranges.'''
    if type(ranges) == tuple:
        ranges = [ranges]
    if ranges[-1][-1] + 1 == elem[0]:
        ranges[-1] = (ranges[-1][0], elem[1])
    else:
        ranges.append(elem)
    return ranges

Variable Documentation

web.app_utils.renaming

Definition at line 18 of file app_utils.py.

---

Generated for CMSSW Reference Manual by  1.8.11