directories2html.py

Go to the documentation of this file.

from __future__ import print_function
from __future__ import absolute_import


#
#                                                                              
# Danilo Piparo CERN - danilo.piparo@cern.ch                                   
#                                                                              


from builtins import range
from os import chdir,getcwd,listdir,makedirs
from os.path import basename,join,exists
import html

import sys
theargv=sys.argv
sys.argv=[]
from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
sys.argv=theargv

import os
import hashlib
import random
from matplotlib.colors import to_hex

if "RELMON_SA" in os.environ:
  from .dirstructure import Comparison,Directory
  from .definitions import *
  from .utils import unpickler
else:
  from Utilities.RelMon.dirstructure import Comparison,Directory
  from Utilities.RelMon.definitions import *
  from Utilities.RelMon.utils import unpickler
  
#-------------------------------------------------------------------------------

def encode_obj_url(url):
  for old,new in url_encode_dict.items():
    url=url.replace(old,new)
  return url

def plot_size(h=250,w=200):
  return "w=%s;h=%s" %(h,w)

def build_obj_addr(run,sample,version,plot_path,tier):
  slash="/"
  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
  obj_url+=plot_path
  while obj_url.endswith(slash):
    obj_url=obj_url[:-1]  
  while slash*2 in obj_url:
    obj_url=obj_url.replace(slash*2,slash)
  return obj_url
  
def build_obj(run,sample,version,plot_path,tier):
  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
  return encode_obj_url(obj_url)

def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
  if len(draw_opts)>0:
    fairy_url+="drawopts=%s;" %draw_opts
  fairy_url+= plot_size(h,w)
  fairy_url += ";ref=ratiooverlay"
  return fairy_url

def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
  fairy_url+= "?%s"%plot_size(h,w)  
  if len(draw_opts)>0:
    fairy_url+="drawopts=%s;" %draw_opts  
  return fairy_url
 
#-------------------------------------------------------------------------------
style_location="/cms-service-reldqm"
def get_page_header(directory=None, standalone=False, additional_header=""):
  style_location="/cms-service-reldqm"
  if standalone:
    style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2 
  javascripts=''
  style=''
  tablestyle=''
  thead_h = 400 
  wrapper_h = 1500
  if directory!=None and len(directory.comparisons)>0:
    meta=directory.meta
    style='img.fail {border:1px solid #ff0000;}\n'+\
          'img.succes {border:1px solid #00ff00;}\n'+\
          'img.null {border:1px solid #ffff00;}\n'+\
          'img.skiped {border:1px solid #7a7a7a;}\n'+\
          'a.black_link:link {color: #333333}\n'+\
          'a.black_link:hover {color: #737373}\n'+\
          'a.black_link:visited {color: #333333}\n'+\
          'a.black_link:active {color: #333333}\n'
    
    
    wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)

  tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
  tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
  tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
  tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
  tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
  tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
  tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
  tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
  tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
  javascripts=""

  

  html='<html>'+\
       '<head>'+\
       '<title>RelMon Summary</title>'+\
       '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
       '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
       '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
       '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
       '<style type="text/css">'+\
       '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
       '%s'%style+\
       '%s'%tablestyle +\
       '</style>'+\
       '%s'%javascripts+\
       '%s'%additional_header+\
       '</head>'+\
       '<body>'+\
       '<div class="container">'
  
  return html
  
#-------------------------------------------------------------------------------

def get_page_footer():
    return '</div></body></html>'

#-------------------------------------------------------------------------------

def get_title_section(directory, hashing_flag, standalone, depth=2):
  
  mother_name=basename(directory.mother_dir)
  mother_file_name=""
  if depth==1:
    mother_file_name="../RelMonSummary.html"
    if mother_name!="":
      mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
  elif depth==2:
    mother_file_name="RelMonSummary.html"
    #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
    if mother_name!="":
      mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
  else:
      if hashing_flag:
          files = directory.mother_dir.split("/")
          if len(files) != 1:
              dir_name = files[-2]+files[-1] 
          else:
              dir_name = files[-1]
          dir_name = directory.mother_dir
          mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
      else:
          mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
          mother_file_name=mother_file_name.strip("_")
      
  link_to_mother='<a href="%s">..</a>' %mother_file_name
  html= '<div class="span-20">'+\
        '<h1>%s</h1>'%directory.name+\
        '</div>'+\
        '<div class="span-1">'+\
        '<h1>%s</h1>'%link_to_mother+\
        '</div>'+\
        '<div class="span-3 last">'+\
        '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
        '</div>'+\
        '<hr>' 
  if len(mother_name)>0:
    html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
          '<hr>' 
  return html
 
#-------------------------------------------------------------------------------
def get_dir_stats(directory):
  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
  html+='<ul>'
  if directory.n_successes>0:
    html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
  if directory.n_nulls>0:
    html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
  if directory.n_fails>0:
    html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
  if directory.n_skiped>0:
    html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
  if directory.n_missing_objs>0:
    html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
  html+='</ul>'
  return html

#-------------------------------------------------------------------------------

def get_subdirs_section(directory, hashing_flag): 
  if len(directory.subdirs)==0:
    return ""
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt">Sub-Directories</h2>'
  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
  # so reverse :)
  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
  for subdir in sorted_subdirs:
    name=subdir.name
    if hashing_flag:
        link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
    else:
        link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
        link=link.strip("_")
    html+='<div class="span-4 prepend-2 colborder">'
    html+='<h3>%s</h3>'%name
    html+='</div>'
    
    html+='<div class="span-7">'
    html+=get_dir_stats(subdir)
    html+='</div>'
    
    html+='<div class="span-6 last">'
    html+= subdir.get_piechart_js(120,link)
    html+='</div>'
    
    html+='<hr>'
  return html+'</div>'

 
#-------------------------------------------------------------------------------

def get_summary_section(directory,matrix_page=True):
  
  # Hack find the first comparison and fill test and threshold
  # shall this be put in meta?
  test_name=""
  test_threshold=0
  for comparison in directory.comparisons:
      test_name=comparison.test_name
      test_threshold=comparison.test_thr
      break
  if len(test_name)==0:
    for subdir in directory.subdirs:  
      for comparison in subdir.comparisons:
        test_name=comparison.test_name
        test_threshold=comparison.test_thr
        break
      if len(test_name)!=0:break      
      if len(test_name)==0:  
        for subsubdir in subdir.subdirs:          
          for comparison in subsubdir.comparisons:
            test_name=comparison.test_name
            test_threshold=comparison.test_thr
            break
          if len(test_name)!=0:break
        if len(test_name)==0:      
          for subsubsubdir in subsubdir.subdirs:
            for comparison in subsubsubdir.comparisons:
              test_name=comparison.test_name
              test_threshold=comparison.test_thr
              break
            if len(test_name)!=0:break


  meta=directory.meta
  
  html= '<div class="span-6">'+\
        '<h3>Summary</h3>'
  html+=get_dir_stats(directory)
  html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
  html+='</div>'
        
  html+='<div class="span-7 colborder">'+\
        '%s'%(directory.get_piechart_js(150)) +\
        '</div>'+\
        '<div class="span-9 last">'
  if matrix_page:
    html+='<h3>Sample:</h3>'+\
          '<p class="caps">%s</p>'%meta.sample1+\
          '<h3>Run1 and Run2:</h3>'+\
          '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
  html+='<h3>Releases:</h3>'+\
        '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
        '<h3>Statistical Test (Pvalue threshold):</h3>'+\
        '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
        '</div>'+\
        '<hr>'
  return html

#-------------------------------------------------------------------------------

def get_comparisons(category,directory):
  """Prepare the comparisons between histograms and organise them in the page.
  Moreover create separate pages with the overlay and the single plots.
  """
  counter=1
  tot_counter=1
  
  # get the right ones
  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]] 
  n_comparisons=len(comparisons)    

  is_reverse=True
  if category == FAIL:
    is_reverse=False
  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)

  
  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
  html_comparisons=""
  for comparison in comparisons:
    plot_name=comparison.img_name
    if "http://" not in plot_name:
      plot_name= basename(comparison.img_name)
    class_type="colborder"    
    if counter==3 or tot_counter==n_comparisons:
      class_type=" colborder last"
    comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)


    if directory.do_pngs:
      png_link=comparison.img_name
      html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
                      '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
                      '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
    else:
      big_fairy=fairy_url(directory.meta.run1,
                        directory.meta.run2,
                        directory.meta.sample1,
                        directory.meta.sample2,
                        directory.meta.release1,
                        directory.meta.release2,
                        comp_abs_path,
                        directory.meta.tier1,
                        directory.meta.tier2,
                        "",600,600)
      small_fairy=fairy_url(directory.meta.run1,
                        directory.meta.run2,
                        directory.meta.sample1,
                        directory.meta.sample2,
                        directory.meta.release1,
                        directory.meta.release2,
                        comp_abs_path,
                        directory.meta.tier1,
                        directory.meta.tier2)    

      single_fairy1=fairy_url_single(directory.meta.run1,
                                   directory.meta.sample1,
                                   directory.meta.release1,
                                   comp_abs_path,
                                   directory.meta.tier1,
                                   "",500,500)
      single_fairy2=fairy_url_single(directory.meta.run2,
                                   directory.meta.sample2,
                                   directory.meta.release2,
                                   comp_abs_path,
                                   directory.meta.tier2,
                                   "",500,500)

      html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
                      '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
                      '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
                      '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
                      '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])

    if counter==3:                    
      html_comparisons+="<hr>"
      counter=0
    counter+=1
    tot_counter+=1

  if len(html_comparisons)!=0:
    html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
    html+=html_comparisons
    html+='<hr>'
    return html
  return ""

#-------------------------------------------------------------------------------

def get_rank_section(directory):
# do Rank histo png
    imgname="RankSummary.png"
    gStyle.SetPadTickY(0)
    c=TCanvas("ranks","ranks",500,400)
    #gStyle.SetOptStat(0)
    c.cd()

    h=directory.rank_histo
    rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
    rank_histof.SetLineWidth(2)
    for i in range(0,h.GetNbinsX()+1):
      rank_histof.SetBinContent(i,h.GetBinContent(i))
    h.SetTitle("Ranks Summary;Rank;Frequency")
    h.Draw("Hist")
    c.Update()
    rank_histof.ComputeIntegral()
    integral = rank_histof.GetIntegral()
    rank_histof.SetContent(integral)

    rightmax = 1.1*rank_histof.GetMaximum()
    scale = gPad.GetUymax()/rightmax
    rank_histof.SetLineColor(kRed)
    rank_histof.Scale(scale)
    rank_histof.Draw("same")

    #draw an axis on the right side
    axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
    axis.SetTitle("Cumulative")
    axis.SetTitleColor(kRed)
    axis.SetLineColor(kRed)
    axis.SetLabelColor(kRed)
    axis.Draw()

    rank_histof.Draw("Same");
    

    c.Print(imgname)

    page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
    page_html+='<div class="span-19"><img src="%s"></div>' %imgname
    page_html+='</div> <a href="#top">Top...</a><hr>'

    return page_html
    
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------
def get_missing_objs_section(directory):
    """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
    """
    page_html = "Unpaired in %s</br>"%(directory.filename1)
    for elem in directory.different_histograms['file1']:
        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
    page_html +="</br>"
    page_html += "Unpaired in %s</br>"%(directory.filename2)
    for elem in directory.different_histograms['file2']:
        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
    return page_html
#-------------------------------------------------------------------------------

def directory2html(directory, hashing, standalone, depth=0):
  """Converts a directory tree into html pages, very nice ones.
  """
  #print "d2html: depth", str(depth)," dir ",directory.name
  depth+=1
  #old_cwd=getcwd()
  #if not exists(directory.name) and len(directory.name)>0:
    #makedirs(directory.name)
  
  #if len(directory.name)>0:
    #chdir(directory.name)
  
  for subdir in directory.subdirs:
    directory2html(subdir,hashing,standalone, depth)
  
  page_html=get_page_header(directory, standalone)+\
            get_title_section(directory,hashing, standalone, depth)+\
            get_summary_section(directory)+\
            get_subdirs_section(directory, hashing)

  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
                     (directory.n_comp_nulls >0,NULL ),
                     
                     (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
                     (directory.n_comp_skiped >0,SKIPED)):
    if do_cat:
      page_html+=get_comparisons(cat,directory)
    
  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
    page_html += get_missing_objs_section(directory)

  # Distribution of ranks

  if depth==1:
    page_html+=get_rank_section(directory)


  page_html+=get_page_footer()

  page_name=directory.name

  if len(page_name)==0:
    page_name="RelMonSummary"
  if hashing:
      if page_name != "RelMonSummary":
          #print " ## oFile path: %s"%(join(directory.full_path))
          #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
          ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
      else:
          ofilename = "RelMonSummary.html"
  else:
      ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
      ofilename=ofilename.strip("_")
  #print "Writing on %s" %ofilename
  ofile=open(ofilename,"w")
  ofile.write(page_html)
  ofile.close()

  #chdir(old_cwd)

def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):

  color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
  font_size = int(w/9)
  gauge_max = 1. - rate

  name = random.getrandbits(64) # just a random has for the canvas
  html = "" 
  html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)

  # "gauge" chart
  html += '<script> new Chart("%s",'%(name) 
  html += '{ type: "doughnut",'

  # data
  html += 'data: {'
  html += 'labels: ["Success", "Failure"],'
  html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
  html += 'data: [%f,%f]}] },'%(rate,gauge_max)

  #options
  html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' 
  html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
  html += '}); </script>'
  if add_rate:
    html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
    html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)

  return html
#-------------------------------------------------------------------------------

def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
  # Let's make a summary for an overview in categories act on the global dir
  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)

  list_of_names=[]
  if the_aggr_pairs==[]:
    for samplename,sampledir in dir_dict.items():
      for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
        if not subsysdirname in list_of_names:
          list_of_names.append(subsysdirname)
          the_aggr_pairs.append((subsysdirname,[subsysdirname]))  
          
  #print the_aggr_pairs
  for cat_name, subdir_list in the_aggr_pairs:
    total_successes=0.
    total_directory_successes=0
    total_weight=0.    
    present_subdirs={}
    total_ndirs=0
    # Loop on samples
    for dirname, sampledir in dir_dict.items():
      # Loop on directories
      for subdirname,subdir in sampledir.get_subdirs_dict().items():        
        if subdirname in subdir_list:          
          nsucc=subdir.n_successes
          total_successes+=nsucc
          weight=subdir.weight
          total_weight+=weight
          total_ndirs+=1
          
          total_directory_successes+= float(nsucc)/weight
          if subdirname in present_subdirs:
            this_dir_dict=present_subdirs[subdirname]
            this_dir_dict["nsucc"]+=nsucc
            this_dir_dict["weight"]+=weight
          else:
            present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
        # Make it usable also for subdirectories
        for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():          
          for pathname in [name for name in subdir_list if "/" in name]:           
            selected_subdirname,selected_subsubdirname = pathname.split("/")
            if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
              #print "Studying directory ",subsubdirname," in directory ",subdirname
              nsucc=subsubdir.n_successes
              total_successes+=nsucc
              weight=subsubdir.weight
              total_weight+=weight
              total_ndirs+=1              
              total_directory_successes+= float(nsucc)/weight
              
              if subsubdirname in present_subdirs:
                this_dir_dict=present_subdirs[subsubdirname]
                this_dir_dict["nsucc"]+=nsucc
                this_dir_dict["weight"]+=weight
              else:
                present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}      

    if total_ndirs == 0:
      print("No directory of the category %s is present in the samples: skipping." %cat_name)
      continue
    
    average_success_rate=total_directory_successes/(total_ndirs)
    aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
    
  return aggr_pairs_info

#-------------------------------------------------------------------------------

def make_categories_summary(dir_dict,aggregation_rules):
    
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
  
  #print aggr_pairs_info
  
  # Now Let's build the HTML
  
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'

  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    #print cat_name,present_subdirs,total_weight,average_success_rate
    html+='<div class="span-3 prepend-0 colborder">'
    html+='<h3>%s</h3>'%cat_name
    html+='<div><span class="alt">Avg. Success rate:</span></div>'
    html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
    html+='</div>'    
    html+='<div class="span-9">'

    html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
    html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
    html+='<ul>'    
    for subdirname in sorted(present_subdirs.keys()):
      this_dir_dict=present_subdirs[subdirname]
      nsucc=this_dir_dict["nsucc"]
      weight=this_dir_dict["weight"]
      html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
    html+='</ul>'    
    html+='</div>'
    
    html+='<div class="span-6 last">'
    html+=build_gauge_js(average_success_rate,add_rate=True)
    html+='</div>'
    
    html+='<hr>'
  return html+'<br><a href="#top">Top...</a> </div><hr>'  
    
 #-------------------------------------------------------------------------------

def make_twiki_table(dir_dict,aggregation_rules):
  
  # decide the release
  meta= list(dir_dict.items())[0][1].meta
  releases=sorted([meta.release1,meta.release2])
  latest_release=releases[1].split("-")[0]
  
  
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
  
  # Now Let's build the HTML
  
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
  html+='<div>| Release | Comparison |'
  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    html+=cat_name
    html+=" | "
  html+='</div>'
  
  html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release

  # Now add all the line with small gauges

  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    #print cat_name,present_subdirs,total_weight,average_success_rate
    html+=build_gauge_js(average_success_rate,w=40)
    html+=" | "    
  
  html+='</div> <a href="#top">Top...</a>'
  html+='<hr>'
  return html+'</div>'
  
#-------------------------------------------------------------------------------

def get_pie_tooltip(directory):
  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
  return tooltip

#-------------------------------------------------------------------------------

def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):  
  
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)       

  script="""
    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
    <script type="text/javascript">
      google.load("visualization", "1", {packages:["corechart"]});
      google.setOnLoadCallback(drawChart);
      function drawChart() {
        var data = new google.visualization.DataTable();
        data.addColumn('string', 'DQM Directory');
        data.addColumn('number', 'Success Rate');
        """
  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
  counter=0
  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
    #print subsystname,present_directories
    script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
    script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
    counter+=1
  script+="""
        var chart = new google.visualization.BarChart(document.getElementById('%s'));
        chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
                          vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
                         });
      }
    </script>
    """%(name,35*counter,title)
  return script


#-------------------------------------------------------------------------------

def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
  """Create a table, with as rows the directories and as columns the samples.
  Each box in the table will contain a pie chart linking to the directory.
  """  
  #aggregation_rules={}
  #aggregation_rules_twiki={}

  chdir(indir)
  if os.path.isabs(indir):
      title = basename(indir)
  else:
      title=indir
  title=title.strip(".")
  title=title.strip("/")
  
  
  # Get the list of pickles
  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
  
  # Load directories, build a list of all first level subdirs  
  dir_unpicklers=[]
  n_unpicklers=0
  for sample_pkl in sample_pkls:
    dir_unpickler=unpickler(sample_pkl)
    dir_unpickler.start()
    n_unpicklers+=1
    dir_unpicklers.append(dir_unpickler)
    if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
      n_unpicklers=0
      for dir_unpickler in dir_unpicklers:
        dir_unpickler.join()
  
  dir_dict={}
  
  # create a fake global directory
  global_dir=Directory("global","")  
  for dir_unpickler in dir_unpicklers:
    dir_unpickler.join()
    directory=dir_unpickler.directory
    #directory.prune("Run summary")    
    #directory.calcStats()
    global_dir.meta=directory.meta
    dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
    global_dir.subdirs.append(directory)
  
  global_dir.calcStats()
  
  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
  
  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
  rel1=""
  rel2=""
  try:
    rel1,rel2=title.split("VS")
  except:
    rel1=global_dir.meta.release1.split("-")[0]
    rel2=global_dir.meta.release2.split("-")[0] 
    global_dir.meta.release1=rel1
    global_dir.meta.release2=rel2
    
  # union of all subdirs names
  all_subdirs=[]
  for directory in dir_dict.values():
    for subdir_name in directory.get_subdirs_names():
      all_subdirs.append(subdir_name)
  all_subdirs=sorted(list(set(all_subdirs)))
  
  #print " $$ all_subdirs: %s" %(all_subdirs)
  
  # Get The title
  page_html+= '<div class="span-20">'+\
              '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
              '</div>'+\
              '<div class="span-1">'+\
              '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
              '</div>'+\
              '<hr>'
  page_html+='<div class="span-24"><p></p></div>\n'*3
  
  # Get The summary section
  page_html+= get_summary_section(global_dir,False)  

  # Make the anchor sections
  page_html+= '<div class="span-24">'
  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
              '<ul>'+\
              '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
              '<li><a href="#categories">Categories</a></li>'+\
              '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
          '<li><a href="#summary_table">Summary Table</a></li>'+\
              '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
              '<li><a href="#twiki_table">Twiki Table</a></li>'+\
              '</ul>'+\
              '</div><hr>'


  # Make the CategoriesBar chart
  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'

  # Make the gauges per categories
  page_html+=make_categories_summary(dir_dict,aggregation_rules)

  # Make the Directories chart
  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
  
  # Barbarian vertical space. Suggestions are welcome
  for i in range(2):
    page_html+='<div class="span-24"><p></p></div>\n'

 
  # Prepare the table
  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'

  div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
  page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
  page_html+="""
        <table>
        <thead>
          <tr>
          <th> <p class = "vertical_head center_head"></p> </th>        
  """
  
  # First row with samples
  page_html+="""
          <th> <p class = "vertical_head center_head">Summary</p></th>"""

  sorted_samples=sorted(dir_dict.keys())
  for sample in sorted_samples:
    if "_" in sample and "Data" not in sample:
      sample_nick="_".join(sample.split("X_")[0].split("_")[:-1]) 
      # Cleaning for MC: just the fragment
    elif "Data" in sample and "RelVal" in sample:
      sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
      # Cleaning for Data: PD + Era + Run
    else:
      sample_nick = sample
      
    page_html+="""
          <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
  page_html+="</tr> \n </thead> \n </tbody> \n"


 # FIRST ROW
 # Now the summaries  at the beginning of the table
  page_html+="<tr>"
  page_html+='<td class="sticky-col">'
  
  page_html+='<b>Summary</b></td>'
  page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
  for sample in sorted_samples:
    col=dir_dict[sample]
    # check if the directory was a top one or not
    summary_page_name="RelMonSummary.html"
    if col.name!="":
      summary_page_name=hash_name(col.name, hashing_flag)+".html"
    title = get_pie_tooltip(col)
    chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
    page_html+='<td>'
    page_html+='%s </a></td>' %(chart)
  page_html+="</tr>"

  # Now the content
  for subdir_name in all_subdirs:  

    page_html+='          <tr>\n'
    page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name  

    row_summary=Directory("row_summary","")
    sample_counter=0
    n_samples=len(sorted_samples)    

    for sample in sorted_samples:
      subdirs_dict=directory.get_subdirs_dict()
      directory=dir_dict[sample]
      dir_is_there=subdir_name in subdirs_dict
      if dir_is_there:
        row_summary.subdirs.append(subdirs_dict[subdir_name])

    # one first row for the summary!
    row_summary.calcStats()
    title = get_pie_tooltip(col)
    chart = row_summary.get_piechart_js(50)
    page_html+='<td><div>'#<div class="span-1">'
    page_html+= chart + '</div></td>'

    for sample in sorted_samples:
      sample_counter+=1      

      directory=dir_dict[sample]
      subdirs_dict=directory.get_subdirs_dict()

      # Check if the directory is the top one
      if directory.name!="":
        # We did not run on the topdir
        #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
        #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
        if hashing_flag:
          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
        else:
          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
      else:
        #print "   ## summary_page: %s"%(directory.name+subdir_name)
        #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
        summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
      dir_is_there=subdir_name in subdirs_dict

      img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
      page_html+='<td><div class="span-1">'

      if dir_is_there:
        #row_summary.subdirs.append(subdirs_dict[subdir_name])
        chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
        page_html+='%s'%chart
      else:
        page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
      page_html+='</div></td>' 

    page_html+="          </tr>\n"        



  page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'

  page_html+=get_rank_section(global_dir)

  #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) 
  # ^ commenting out for the moment, not really useful nor used

  page_html+=get_page_footer()
  return page_html  


#-----------UPDATES------
def hash_name(file_name, flag):
    if flag: #if hashing flag is ON then return
        if (3,0,0) <= sys.version_info:
            return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
        return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
    else:
        return file_name #return standart name