directories2html Namespace Reference

Functions
def	build_gauge_js (rate, w=100, minrate=.80, add_rate=False)
def	build_obj (run, sample, version, plot_path, tier)
def	build_obj_addr (run, sample, version, plot_path, tier)
def	directory2html (directory, hashing, standalone, depth=0)
def	encode_obj_url (url)
def	fairy_url (run1, run2, sample1, sample2, version1, version2, plot_path, tier1, tier2, draw_opts="", h=250, w=200)
def	fairy_url_single (run, sample, version, plot_path, tier, draw_opts="", h=250, w=200)
def	get_aggr_pairs_info (dir_dict, the_aggr_pairs=[])
def	get_comparisons (category, directory)
def	get_dir_stats (directory)
def	get_missing_objs_section (directory)
def	get_page_footer ()
def	get_page_header (directory=None, standalone=False, additional_header="")
def	get_pie_tooltip (directory)
def	get_rank_section (directory)
def	get_subdirs_section (directory, hashing_flag)
def	get_summary_section (directory, matrix_page=True)
def	get_title_section (directory, hashing_flag, standalone, depth=2)
def	hash_name (file_name, flag)
def	make_barchart_summary (dir_dict, name="the_chart", title="DQM directory", the_aggr_pairs=[])
def	make_categories_summary (dir_dict, aggregation_rules)
def	make_summary_table (indir, aggregation_rules, aggregation_rules_twiki, hashing_flag, standalone_flag)
def	make_twiki_table (dir_dict, aggregation_rules)
def	plot_size (h=250, w=200)

Variables
	argv
	theargv

Function Documentation

build_gauge_js()

def directories2html.build_gauge_js	(		rate,
			w = 100,
			minrate = .80,
			add_rate = False
	)

Definition at line 512 of file directories2html.py.

References definitions.gauge_cmap, createfilelist.int, and to_hex().

Referenced by make_categories_summary(), and make_twiki_table().

def build_gauge_js(rate,w=100,minrate=.80,add_rate=False):

  color = to_hex(gauge_cmap((rate-minrate)/(1.0-minrate)))
  font_size = int(w/9)
  gauge_max = 1. - rate

  name = random.getrandbits(64) # just a random has for the canvas
  html = "" 
  html += '<canvas id="%s" style="width:100%%;max-width:%d"></canvas>'%(name,w)

  # "gauge" chart
  html += '<script> new Chart("%s",'%(name) 
  html += '{ type: "doughnut",'

  # data
  html += 'data: {'
  html += 'labels: ["Success", "Failure"],'
  html += 'datasets: [{ backgroundColor: ["%s", "#C7C7C7"],'%(color)
  html += 'data: [%f,%f]}] },'%(rate,gauge_max)

  #options
  html += 'options: { responsive: true, rotation: -3.1415926536, circumference: 3.1415926536 ,' 
  html += ' legend: { display: false }, tooltips: {enabled: false}, hover: {mode: null}},'
  html += '}); </script>'
  if add_rate:
    html += '<div style="width: 100%%; position: relative; left: %d; margin-top: -%dpx; font-align: center">'%(int(2*w/7),font_size)
    html += '<span style="color: %s; font-family: courier; font-size:  %dpx;">%.2f%%</span></div>'%(color,font_size,rate*100.)

  return html
#-------------------------------------------------------------------------------

build_obj()

def directories2html.build_obj	(		run,
			sample,
			version,
			plot_path,
			tier
	)

Definition at line 59 of file directories2html.py.

References build_obj_addr(), and encode_obj_url().

Referenced by fairy_url().

def build_obj(run,sample,version,plot_path,tier):
  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
  return encode_obj_url(obj_url)

build_obj_addr()

def directories2html.build_obj_addr	(		run,
			sample,
			version,
			plot_path,
			tier
	)

Definition at line 49 of file directories2html.py.

Referenced by build_obj(), and fairy_url_single().

def build_obj_addr(run,sample,version,plot_path,tier):
  slash="/"
  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
  obj_url+=plot_path
  while obj_url.endswith(slash):
    obj_url=obj_url[:-1]  
  while slash*2 in obj_url:
    obj_url=obj_url.replace(slash*2,slash)
  return obj_url
  

directory2html()

def directories2html.directory2html	(		directory,
			hashing,
			standalone,
			depth = 0
	)

Converts a directory tree into html pages, very nice ones.

Definition at line 452 of file directories2html.py.

References get_comparisons(), get_missing_objs_section(), get_page_footer(), get_page_header(), get_rank_section(), get_subdirs_section(), get_summary_section(), get_title_section(), hash_name(), and join().

def directory2html(directory, hashing, standalone, depth=0):
  """Converts a directory tree into html pages, very nice ones.
  """
  #print "d2html: depth", str(depth)," dir ",directory.name
  depth+=1
  #old_cwd=getcwd()
  #if not exists(directory.name) and len(directory.name)>0:
    #makedirs(directory.name)
  
  #if len(directory.name)>0:
    #chdir(directory.name)
  
  for subdir in directory.subdirs:
    directory2html(subdir,hashing,standalone, depth)
  
  page_html=get_page_header(directory, standalone)+\
            get_title_section(directory,hashing, standalone, depth)+\
            get_summary_section(directory)+\
            get_subdirs_section(directory, hashing)

  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
                     (directory.n_comp_nulls >0,NULL ),
                     
                     (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
                     (directory.n_comp_skiped >0,SKIPED)):
    if do_cat:
      page_html+=get_comparisons(cat,directory)
    
  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
    page_html += get_missing_objs_section(directory)

  # Distribution of ranks

  if depth==1:
    page_html+=get_rank_section(directory)


  page_html+=get_page_footer()

  page_name=directory.name

  if len(page_name)==0:
    page_name="RelMonSummary"
  if hashing:
      if page_name != "RelMonSummary":
          #print " ## oFile path: %s"%(join(directory.full_path))
          #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
          ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
      else:
          ofilename = "RelMonSummary.html"
  else:
      ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
      ofilename=ofilename.strip("_")
  #print "Writing on %s" %ofilename
  ofile=open(ofilename,"w")
  ofile.write(page_html)
  ofile.close()

  #chdir(old_cwd)

encode_obj_url()

def directories2html.encode_obj_url

(

url

)

Definition at line 41 of file directories2html.py.

Referenced by build_obj().

def encode_obj_url(url):
  for old,new in url_encode_dict.items():
    url=url.replace(old,new)
  return url

fairy_url()

def directories2html.fairy_url	(		run1,
			run2,
			sample1,
			sample2,
			version1,
			version2,
			plot_path,
			tier1,
			tier2,
			draw_opts = "",
			h = 250,
			w = 200
	)

Definition at line 63 of file directories2html.py.

References build_obj(), and plot_size().

Referenced by get_comparisons().

def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
  if len(draw_opts)>0:
    fairy_url+="drawopts=%s;" %draw_opts
  fairy_url+= plot_size(h,w)
  fairy_url += ";ref=ratiooverlay"
  return fairy_url

fairy_url_single()

def directories2html.fairy_url_single	(		run,
			sample,
			version,
			plot_path,
			tier,
			draw_opts = "",
			h = 250,
			w = 200
	)

Definition at line 73 of file directories2html.py.

References build_obj_addr(), and plot_size().

Referenced by get_comparisons().

def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
  fairy_url+= "?%s"%plot_size(h,w)  
  if len(draw_opts)>0:
    fairy_url+="drawopts=%s;" %draw_opts  
  return fairy_url
 
#-------------------------------------------------------------------------------
style_location="/cms-service-reldqm"

get_aggr_pairs_info()

def directories2html.get_aggr_pairs_info	(		dir_dict,
			the_aggr_pairs = []
	)

Definition at line 543 of file directories2html.py.

References nano_mu_digi_cff.float, mps_monitormerge.items, relativeConstraints.keys, and print().

Referenced by make_barchart_summary(), make_categories_summary(), and make_twiki_table().

def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
  # Let's make a summary for an overview in categories act on the global dir
  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)

  list_of_names=[]
  if the_aggr_pairs==[]:
    for samplename,sampledir in dir_dict.items():
      for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
        if not subsysdirname in list_of_names:
          list_of_names.append(subsysdirname)
          the_aggr_pairs.append((subsysdirname,[subsysdirname]))  
          
  #print the_aggr_pairs
  for cat_name, subdir_list in the_aggr_pairs:
    total_successes=0.
    total_directory_successes=0
    total_weight=0.    
    present_subdirs={}
    total_ndirs=0
    # Loop on samples
    for dirname, sampledir in dir_dict.items():
      # Loop on directories
      for subdirname,subdir in sampledir.get_subdirs_dict().items():        
        if subdirname in subdir_list:          
          nsucc=subdir.n_successes
          total_successes+=nsucc
          weight=subdir.weight
          total_weight+=weight
          total_ndirs+=1
          
          total_directory_successes+= float(nsucc)/weight
          if subdirname in present_subdirs:
            this_dir_dict=present_subdirs[subdirname]
            this_dir_dict["nsucc"]+=nsucc
            this_dir_dict["weight"]+=weight
          else:
            present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
        # Make it usable also for subdirectories
        for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():          
          for pathname in [name for name in subdir_list if "/" in name]:           
            selected_subdirname,selected_subsubdirname = pathname.split("/")
            if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
              #print "Studying directory ",subsubdirname," in directory ",subdirname
              nsucc=subsubdir.n_successes
              total_successes+=nsucc
              weight=subsubdir.weight
              total_weight+=weight
              total_ndirs+=1              
              total_directory_successes+= float(nsucc)/weight
              
              if subsubdirname in present_subdirs:
                this_dir_dict=present_subdirs[subsubdirname]
                this_dir_dict["nsucc"]+=nsucc
                this_dir_dict["weight"]+=weight
              else:
                present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}      

    if total_ndirs == 0:
      print("No directory of the category %s is present in the samples: skipping." %cat_name)
      continue
    
    average_success_rate=total_directory_successes/(total_ndirs)
    aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
    
  return aggr_pairs_info

#-------------------------------------------------------------------------------

get_comparisons()

def directories2html.get_comparisons	(		category,
			directory
	)

Prepare the comparisons between histograms and organise them in the page.
Moreover create separate pages with the overlay and the single plots.

Definition at line 302 of file directories2html.py.

References fairy_url(), and fairy_url_single().

Referenced by directory2html().

def get_comparisons(category,directory):
  """Prepare the comparisons between histograms and organise them in the page.
  Moreover create separate pages with the overlay and the single plots.
  """
  counter=1
  tot_counter=1
  
  # get the right ones
  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]] 
  n_comparisons=len(comparisons)    

  is_reverse=True
  if category == FAIL:
    is_reverse=False
  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)

  
  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
  html_comparisons=""
  for comparison in comparisons:
    plot_name=comparison.img_name
    if "http://" not in plot_name:
      plot_name= basename(comparison.img_name)
    class_type="colborder"    
    if counter==3 or tot_counter==n_comparisons:
      class_type=" colborder last"
    comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)


    if directory.do_pngs:
      png_link=comparison.img_name
      html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
                      '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
                      '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
    else:
      big_fairy=fairy_url(directory.meta.run1,
                        directory.meta.run2,
                        directory.meta.sample1,
                        directory.meta.sample2,
                        directory.meta.release1,
                        directory.meta.release2,
                        comp_abs_path,
                        directory.meta.tier1,
                        directory.meta.tier2,
                        "",600,600)
      small_fairy=fairy_url(directory.meta.run1,
                        directory.meta.run2,
                        directory.meta.sample1,
                        directory.meta.sample2,
                        directory.meta.release1,
                        directory.meta.release2,
                        comp_abs_path,
                        directory.meta.tier1,
                        directory.meta.tier2)    

      single_fairy1=fairy_url_single(directory.meta.run1,
                                   directory.meta.sample1,
                                   directory.meta.release1,
                                   comp_abs_path,
                                   directory.meta.tier1,
                                   "",500,500)
      single_fairy2=fairy_url_single(directory.meta.run2,
                                   directory.meta.sample2,
                                   directory.meta.release2,
                                   comp_abs_path,
                                   directory.meta.tier2,
                                   "",500,500)

      html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
                      '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
                      '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
                      '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
                      '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])

    if counter==3:                    
      html_comparisons+="<hr>"
      counter=0
    counter+=1
    tot_counter+=1

  if len(html_comparisons)!=0:
    html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
    html+=html_comparisons
    html+='<hr>'
    return html
  return ""

#-------------------------------------------------------------------------------

get_dir_stats()

def directories2html.get_dir_stats

(

directory

)

Definition at line 189 of file directories2html.py.

Referenced by get_subdirs_section(), and get_summary_section().

def get_dir_stats(directory):
  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
  html+='<ul>'
  if directory.n_successes>0:
    html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
  if directory.n_nulls>0:
    html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
  if directory.n_fails>0:
    html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
  if directory.n_skiped>0:
    html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
  if directory.n_missing_objs>0:
    html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
  html+='</ul>'
  return html

#-------------------------------------------------------------------------------

get_missing_objs_section()

def directories2html.get_missing_objs_section

(

directory

)

Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other

Definition at line 439 of file directories2html.py.

Referenced by directory2html().

def get_missing_objs_section(directory):
    """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
    """
    page_html = "Unpaired in %s</br>"%(directory.filename1)
    for elem in directory.different_histograms['file1']:
        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
    page_html +="</br>"
    page_html += "Unpaired in %s</br>"%(directory.filename2)
    for elem in directory.different_histograms['file2']:
        page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
    return page_html
#-------------------------------------------------------------------------------

get_page_footer()

def directories2html.get_page_footer

(

)

Definition at line 141 of file directories2html.py.

Referenced by directory2html(), and make_summary_table().

def get_page_footer():
    return '</div></body></html>'

#-------------------------------------------------------------------------------

get_page_header()

def directories2html.get_page_header	(		directory = None,
			standalone = False,
			additional_header = ""
	)

Definition at line 83 of file directories2html.py.

References SiStripPI.min.

Referenced by directory2html(), and make_summary_table().

def get_page_header(directory=None, standalone=False, additional_header=""):
  style_location="/cms-service-reldqm"
  if standalone:
    style_location = "https://raw.githubusercontent.com/cms-PdmV/RelMonService2/77c534ec93401ca5de222ac62a6422f02389dafc/report_website/" #RelMonService2 
  javascripts=''
  style=''
  tablestyle=''
  thead_h = 400 
  wrapper_h = 1500
  if directory!=None and len(directory.comparisons)>0:
    meta=directory.meta
    style='img.fail {border:1px solid #ff0000;}\n'+\
          'img.succes {border:1px solid #00ff00;}\n'+\
          'img.null {border:1px solid #ffff00;}\n'+\
          'img.skiped {border:1px solid #7a7a7a;}\n'+\
          'a.black_link:link {color: #333333}\n'+\
          'a.black_link:hover {color: #737373}\n'+\
          'a.black_link:visited {color: #333333}\n'+\
          'a.black_link:active {color: #333333}\n'
    
    
    wrapper_h = min(thead_h + (70 * len(directory.comparisons)),1800)

  tablestyle = '\n.wrapper { overflow: auto; height: %dpx;} \n'%(wrapper_h)
  tablestyle += 'table {  position: relative; border-collapse: separate; border-spacing: 0;} \n'
  tablestyle += 'table { position: relative; border-collapse: separate; border-spacing: 0;} \n'
  tablestyle += 'table th, table td { width: 50px; padding: 5px; background-color: white;} \n'
  tablestyle += 'table th { position: sticky; top: 0; z-index: 2; height: %dpx;} \n'%(thead_h)
  tablestyle += 'table th:nth-child(1) { left:0; z-index:3;} \n'
  tablestyle += '.sticky-col { position: sticky; background-color: #C9FFD1  ; width: 200px; left:0}\n'
  tablestyle += '.center_head { position: absolute; top: 50%; left: 50%;} \n'
  tablestyle += '.vertical_head {top: 60%; -webkit-transform:  translateX(-50%) translateY(-50%) rotate(-90deg); -moz-transform: translateX(-50%) translateY(-50%) rotate(-90deg);} \n'
  javascripts=""

  

  html='<html>'+\
       '<head>'+\
       '<title>RelMon Summary</title>'+\
       '<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.9.4/Chart.js"></script>' + \
       '<link rel="stylesheet" href="%s/screen.css" type="text/css" media="screen, projection">'%style_location+\
       '<link rel="stylesheet" href="%s/print.css" type="text/css" media="print">'%style_location+\
       '<link rel="stylesheet" href="%s/fancy-type-screen.css" type="text/css" media="screen, projection">'%style_location+\
       '<style type="text/css">'+\
       '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
       '%s'%style+\
       '%s'%tablestyle +\
       '</style>'+\
       '%s'%javascripts+\
       '%s'%additional_header+\
       '</head>'+\
       '<body>'+\
       '<div class="container">'
  
  return html
  
#-------------------------------------------------------------------------------

get_pie_tooltip()

def directories2html.get_pie_tooltip

(

directory

)

Definition at line 686 of file directories2html.py.

Referenced by make_summary_table().

def get_pie_tooltip(directory):
  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
  return tooltip

#-------------------------------------------------------------------------------

get_rank_section()

def directories2html.get_rank_section

(

directory

)

Definition at line 391 of file directories2html.py.

References FastTimerService_cff.range.

Referenced by directory2html(), and make_summary_table().

def get_rank_section(directory):
# do Rank histo png
    imgname="RankSummary.png"
    gStyle.SetPadTickY(0)
    c=TCanvas("ranks","ranks",500,400)
    #gStyle.SetOptStat(0)
    c.cd()

    h=directory.rank_histo
    rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
    rank_histof.SetLineWidth(2)
    for i in range(0,h.GetNbinsX()+1):
      rank_histof.SetBinContent(i,h.GetBinContent(i))
    h.SetTitle("Ranks Summary;Rank;Frequency")
    h.Draw("Hist")
    c.Update()
    rank_histof.ComputeIntegral()
    integral = rank_histof.GetIntegral()
    rank_histof.SetContent(integral)

    rightmax = 1.1*rank_histof.GetMaximum()
    scale = gPad.GetUymax()/rightmax
    rank_histof.SetLineColor(kRed)
    rank_histof.Scale(scale)
    rank_histof.Draw("same")

    #draw an axis on the right side
    axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
    axis.SetTitle("Cumulative")
    axis.SetTitleColor(kRed)
    axis.SetLineColor(kRed)
    axis.SetLabelColor(kRed)
    axis.Draw()

    rank_histof.Draw("Same");
    

    c.Print(imgname)

    page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
    page_html+='<div class="span-19"><img src="%s"></div>' %imgname
    page_html+='</div> <a href="#top">Top...</a><hr>'

    return page_html
    
#-------------------------------------------------------------------------------

#-------------------------------------------------------------------------------

get_subdirs_section()

def directories2html.get_subdirs_section	(		directory,
			hashing_flag
	)

Definition at line 207 of file directories2html.py.

References get_dir_stats(), hash_name(), and join().

Referenced by directory2html().

def get_subdirs_section(directory, hashing_flag): 
  if len(directory.subdirs)==0:
    return ""
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt">Sub-Directories</h2>'
  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
  # so reverse :)
  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
  for subdir in sorted_subdirs:
    name=subdir.name
    if hashing_flag:
        link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
    else:
        link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
        link=link.strip("_")
    html+='<div class="span-4 prepend-2 colborder">'
    html+='<h3>%s</h3>'%name
    html+='</div>'
    
    html+='<div class="span-7">'
    html+=get_dir_stats(subdir)
    html+='</div>'
    
    html+='<div class="span-6 last">'
    html+= subdir.get_piechart_js(120,link)
    html+='</div>'
    
    html+='<hr>'
  return html+'</div>'

 
#-------------------------------------------------------------------------------

get_summary_section()

def directories2html.get_summary_section	(		directory,
			matrix_page = True
	)

Definition at line 242 of file directories2html.py.

References get_dir_stats().

Referenced by directory2html(), and make_summary_table().

def get_summary_section(directory,matrix_page=True):
  
  # Hack find the first comparison and fill test and threshold
  # shall this be put in meta?
  test_name=""
  test_threshold=0
  for comparison in directory.comparisons:
      test_name=comparison.test_name
      test_threshold=comparison.test_thr
      break
  if len(test_name)==0:
    for subdir in directory.subdirs:  
      for comparison in subdir.comparisons:
        test_name=comparison.test_name
        test_threshold=comparison.test_thr
        break
      if len(test_name)!=0:break      
      if len(test_name)==0:  
        for subsubdir in subdir.subdirs:          
          for comparison in subsubdir.comparisons:
            test_name=comparison.test_name
            test_threshold=comparison.test_thr
            break
          if len(test_name)!=0:break
        if len(test_name)==0:      
          for subsubsubdir in subsubdir.subdirs:
            for comparison in subsubsubdir.comparisons:
              test_name=comparison.test_name
              test_threshold=comparison.test_thr
              break
            if len(test_name)!=0:break


  meta=directory.meta
  
  html= '<div class="span-6">'+\
        '<h3>Summary</h3>'
  html+=get_dir_stats(directory)
  html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
  html+='</div>'
        
  html+='<div class="span-7 colborder">'+\
        '%s'%(directory.get_piechart_js(150)) +\
        '</div>'+\
        '<div class="span-9 last">'
  if matrix_page:
    html+='<h3>Sample:</h3>'+\
          '<p class="caps">%s</p>'%meta.sample1+\
          '<h3>Run1 and Run2:</h3>'+\
          '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
  html+='<h3>Releases:</h3>'+\
        '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
        '<h3>Statistical Test (Pvalue threshold):</h3>'+\
        '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
        '</div>'+\
        '<hr>'
  return html

#-------------------------------------------------------------------------------

get_title_section()

def directories2html.get_title_section	(		directory,
			hashing_flag,
			standalone,
			depth = 2
	)

Definition at line 146 of file directories2html.py.

References hash_name().

Referenced by directory2html().

def get_title_section(directory, hashing_flag, standalone, depth=2):
  
  mother_name=basename(directory.mother_dir)
  mother_file_name=""
  if depth==1:
    mother_file_name="../RelMonSummary.html"
    if mother_name!="":
      mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
  elif depth==2:
    mother_file_name="RelMonSummary.html"
    #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
    if mother_name!="":
      mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
  else:
      if hashing_flag:
          files = directory.mother_dir.split("/")
          if len(files) != 1:
              dir_name = files[-2]+files[-1] 
          else:
              dir_name = files[-1]
          dir_name = directory.mother_dir
          mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
      else:
          mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
          mother_file_name=mother_file_name.strip("_")
      
  link_to_mother='<a href="%s">..</a>' %mother_file_name
  html= '<div class="span-20">'+\
        '<h1>%s</h1>'%directory.name+\
        '</div>'+\
        '<div class="span-1">'+\
        '<h1>%s</h1>'%link_to_mother+\
        '</div>'+\
        '<div class="span-3 last">'+\
        '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
        '</div>'+\
        '<hr>' 
  if len(mother_name)>0:
    html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
          '<hr>' 
  return html
 
#-------------------------------------------------------------------------------

hash_name()

def directories2html.hash_name	(		file_name,
			flag
	)

Definition at line 961 of file directories2html.py.

Referenced by directory2html(), get_subdirs_section(), get_title_section(), and make_summary_table().

def hash_name(file_name, flag):
    if flag: #if hashing flag is ON then return
        if (3,0,0) <= sys.version_info:
            return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
        return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
    else:
        return file_name #return standart name

make_barchart_summary()

def directories2html.make_barchart_summary	(		dir_dict,
			name = "the_chart",
			title = "DQM directory",
			the_aggr_pairs = []
	)

Definition at line 692 of file directories2html.py.

References get_aggr_pairs_info().

Referenced by make_summary_table().

def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):  
  
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)       

  script="""
    <script type="text/javascript" src="https://www.google.com/jsapi"></script>
    <script type="text/javascript">
      google.load("visualization", "1", {packages:["corechart"]});
      google.setOnLoadCallback(drawChart);
      function drawChart() {
        var data = new google.visualization.DataTable();
        data.addColumn('string', 'DQM Directory');
        data.addColumn('number', 'Success Rate');
        """
  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
  counter=0
  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
    #print subsystname,present_directories
    script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
    script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
    counter+=1
  script+="""
        var chart = new google.visualization.BarChart(document.getElementById('%s'));
        chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
                          vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
                         });
      }
    </script>
    """%(name,35*counter,title)
  return script


#-------------------------------------------------------------------------------

make_categories_summary()

def directories2html.make_categories_summary	(		dir_dict,
			aggregation_rules
	)

Definition at line 611 of file directories2html.py.

References build_gauge_js(), nano_mu_digi_cff.float, and get_aggr_pairs_info().

Referenced by make_summary_table().

def make_categories_summary(dir_dict,aggregation_rules):
    
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
  
  #print aggr_pairs_info
  
  # Now Let's build the HTML
  
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'

  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    #print cat_name,present_subdirs,total_weight,average_success_rate
    html+='<div class="span-3 prepend-0 colborder">'
    html+='<h3>%s</h3>'%cat_name
    html+='<div><span class="alt">Avg. Success rate:</span></div>'
    html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
    html+='</div>'    
    html+='<div class="span-9">'

    html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
    html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
    html+='<ul>'    
    for subdirname in sorted(present_subdirs.keys()):
      this_dir_dict=present_subdirs[subdirname]
      nsucc=this_dir_dict["nsucc"]
      weight=this_dir_dict["weight"]
      html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
    html+='</ul>'    
    html+='</div>'
    
    html+='<div class="span-6 last">'
    html+=build_gauge_js(average_success_rate,add_rate=True)
    html+='</div>'
    
    html+='<hr>'
  return html+'<br><a href="#top">Top...</a> </div><hr>'  
    
 #-------------------------------------------------------------------------------

make_summary_table()

def directories2html.make_summary_table	(		indir,
			aggregation_rules,
			aggregation_rules_twiki,
			hashing_flag,
			standalone_flag
	)

Create a table, with as rows the directories and as columns the samples.
Each box in the table will contain a pie chart linking to the directory.

Definition at line 726 of file directories2html.py.

References get_page_footer(), get_page_header(), get_pie_tooltip(), get_rank_section(), get_summary_section(), hash_name(), join(), make_barchart_summary(), make_categories_summary(), SiStripPI.min, FastTimerService_cff.range, and submitPVValidationJobs.split().

Referenced by ValidationMatrix.do_html().

def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
  """Create a table, with as rows the directories and as columns the samples.
  Each box in the table will contain a pie chart linking to the directory.
  """  
  #aggregation_rules={}
  #aggregation_rules_twiki={}

  chdir(indir)
  if os.path.isabs(indir):
      title = basename(indir)
  else:
      title=indir
  title=title.strip(".")
  title=title.strip("/")
  
  
  # Get the list of pickles
  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
  
  # Load directories, build a list of all first level subdirs  
  dir_unpicklers=[]
  n_unpicklers=0
  for sample_pkl in sample_pkls:
    dir_unpickler=unpickler(sample_pkl)
    dir_unpickler.start()
    n_unpicklers+=1
    dir_unpicklers.append(dir_unpickler)
    if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
      n_unpicklers=0
      for dir_unpickler in dir_unpicklers:
        dir_unpickler.join()
  
  dir_dict={}
  
  # create a fake global directory
  global_dir=Directory("global","")  
  for dir_unpickler in dir_unpicklers:
    dir_unpickler.join()
    directory=dir_unpickler.directory
    #directory.prune("Run summary")    
    #directory.calcStats()
    global_dir.meta=directory.meta
    dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
    global_dir.subdirs.append(directory)
  
  global_dir.calcStats()
  
  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
  
  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
  rel1=""
  rel2=""
  try:
    rel1,rel2=title.split("VS")
  except:
    rel1=global_dir.meta.release1.split("-")[0]
    rel2=global_dir.meta.release2.split("-")[0] 
    global_dir.meta.release1=rel1
    global_dir.meta.release2=rel2
    
  # union of all subdirs names
  all_subdirs=[]
  for directory in dir_dict.values():
    for subdir_name in directory.get_subdirs_names():
      all_subdirs.append(subdir_name)
  all_subdirs=sorted(list(set(all_subdirs)))
  
  #print " $$ all_subdirs: %s" %(all_subdirs)
  
  # Get The title
  page_html+= '<div class="span-20">'+\
              '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
              '</div>'+\
              '<div class="span-1">'+\
              '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
              '</div>'+\
              '<hr>'
  page_html+='<div class="span-24"><p></p></div>\n'*3
  
  # Get The summary section
  page_html+= get_summary_section(global_dir,False)  

  # Make the anchor sections
  page_html+= '<div class="span-24">'
  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
              '<ul>'+\
              '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
              '<li><a href="#categories">Categories</a></li>'+\
              '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
          '<li><a href="#summary_table">Summary Table</a></li>'+\
              '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
              '<li><a href="#twiki_table">Twiki Table</a></li>'+\
              '</ul>'+\
              '</div><hr>'


  # Make the CategoriesBar chart
  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'

  # Make the gauges per categories
  page_html+=make_categories_summary(dir_dict,aggregation_rules)

  # Make the Directories chart
  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
  
  # Barbarian vertical space. Suggestions are welcome
  for i in range(2):
    page_html+='<div class="span-24"><p></p></div>\n'

 
  # Prepare the table
  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2> <h4> <span class="alt"> (scrollable) </span> </h4> </div>'

  div_width= min(len(dir_dict.keys()) * 70 + 500,1500) #80 px per column + 200 for the first column
  page_html+='<div class="wrapper" style = "width: %dpx;">'%(div_width)
  page_html+="""
        <table>
        <thead>
          <tr>
          <th> <p class = "vertical_head center_head"></p> </th>        
  """
  
  # First row with samples
  page_html+="""
          <th> <p class = "vertical_head center_head">Summary</p></th>"""

  sorted_samples=sorted(dir_dict.keys())
  for sample in sorted_samples:
    if "_" in sample and "Data" not in sample:
      sample_nick="_".join(sample.split("X_")[0].split("_")[:-1]) 
      # Cleaning for MC: just the fragment
    elif "Data" in sample and "RelVal" in sample:
      sample_nick = "".join([sample.split("_")[0],sample.split("RelVal")[-1]])
      # Cleaning for Data: PD + Era + Run
    else:
      sample_nick = sample
      
    page_html+="""
          <th> <p class = "vertical_head center_head">%s</th></p>"""%sample_nick
  page_html+="</tr> \n </thead> \n </tbody> \n"


 # FIRST ROW
 # Now the summaries  at the beginning of the table
  page_html+="<tr>"
  page_html+='<td class="sticky-col">'
  
  page_html+='<b>Summary</b></td>'
  page_html+='<td><div class="span-1"> %s </div></td>'%(global_dir.get_piechart_js(50))
  for sample in sorted_samples:
    col=dir_dict[sample]
    # check if the directory was a top one or not
    summary_page_name="RelMonSummary.html"
    if col.name!="":
      summary_page_name=hash_name(col.name, hashing_flag)+".html"
    title = get_pie_tooltip(col)
    chart = col.get_piechart_js(50,sample+"/"+summary_page_name)
    page_html+='<td>'
    page_html+='%s </a></td>' %(chart)
  page_html+="</tr>"

  # Now the content
  for subdir_name in all_subdirs:  

    page_html+='          <tr>\n'
    page_html+='          <td class="sticky-col" style="font-weight:bold;">%s</td>\n' %subdir_name  

    row_summary=Directory("row_summary","")
    sample_counter=0
    n_samples=len(sorted_samples)    

    for sample in sorted_samples:
      subdirs_dict=directory.get_subdirs_dict()
      directory=dir_dict[sample]
      dir_is_there=subdir_name in subdirs_dict
      if dir_is_there:
        row_summary.subdirs.append(subdirs_dict[subdir_name])

    # one first row for the summary!
    row_summary.calcStats()
    title = get_pie_tooltip(col)
    chart = row_summary.get_piechart_js(50)
    page_html+='<td><div>'#<div class="span-1">'
    page_html+= chart + '</div></td>'

    for sample in sorted_samples:
      sample_counter+=1      

      directory=dir_dict[sample]
      subdirs_dict=directory.get_subdirs_dict()

      # Check if the directory is the top one
      if directory.name!="":
        # We did not run on the topdir
        #print "   ## summary_page: %s"%(directory.name+"_"+subdir_name)
        #print "   ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
        if hashing_flag:
          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
        else:
          summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
      else:
        #print "   ## summary_page: %s"%(directory.name+subdir_name)
        #print "   ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
        summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
      dir_is_there=subdir_name in subdirs_dict

      img_link="https://upload.wikimedia.org/wikipedia/commons/a/a8/Circle_Davys-Grey_Solid.svg"
      page_html+='<td><div class="span-1">'

      if dir_is_there:
        #row_summary.subdirs.append(subdirs_dict[subdir_name])
        chart=subdirs_dict[subdir_name].get_piechart_js(50,summary_page)
        page_html+='%s'%chart
      else:
        page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,"Unavailable")
      page_html+='</div></td>' 

    page_html+="          </tr>\n"        



  page_html+='</tbody> </table> </div> <a href="#top">Top...</a><hr>'

  page_html+=get_rank_section(global_dir)

  #page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki) 
  # ^ commenting out for the moment, not really useful nor used

  page_html+=get_page_footer()
  return page_html  


#-----------UPDATES------

make_twiki_table()

def directories2html.make_twiki_table	(		dir_dict,
			aggregation_rules
	)

Definition at line 651 of file directories2html.py.

References build_gauge_js(), get_aggr_pairs_info(), and submitPVValidationJobs.split().

def make_twiki_table(dir_dict,aggregation_rules):
  
  # decide the release
  meta= list(dir_dict.items())[0][1].meta
  releases=sorted([meta.release1,meta.release2])
  latest_release=releases[1].split("-")[0]
  
  
  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
  
  # Now Let's build the HTML
  
  html= '<div class="span-20 colborder">'
  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
  html+='<div>| Release | Comparison |'
  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    html+=cat_name
    html+=" | "
  html+='</div>'
  
  html+='<div>| %s |  %%ICON{arrowdot}%%  | '%latest_release

  # Now add all the line with small gauges

  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
    #print cat_name,present_subdirs,total_weight,average_success_rate
    html+=build_gauge_js(average_success_rate,w=40)
    html+=" | "    
  
  html+='</div> <a href="#top">Top...</a>'
  html+='<hr>'
  return html+'</div>'
  
#-------------------------------------------------------------------------------

plot_size()

def directories2html.plot_size	(		h = 250,
			w = 200
	)

Definition at line 46 of file directories2html.py.

Referenced by fairy_url(), and fairy_url_single().

def plot_size(h=250,w=200):
  return "w=%s;h=%s" %(h,w)

Variable Documentation

argv

directories2html.argv

Definition at line 21 of file directories2html.py.

theargv

directories2html.theargv

Definition at line 20 of file directories2html.py.

---

Generated for CMSSW Reference Manual by  1.8.14