CMS 3D CMS Logo
directories2html.py
Go to the documentation of this file.
1 from __future__ import print_function
2 from __future__ import absolute_import
3 
7 
8 #
9 #
10 # Danilo Piparo CERN - danilo.piparo@cern.ch
11 #
12 
13 
14 from builtins import range
15 from os import chdir,getcwd,listdir,makedirs
16 from os.path import basename,join,exists
17 import html
18 
19 import sys
20 theargv=sys.argv
21 sys.argv=[]
22 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
23 sys.argv=theargv
24 
25 import os
26 import hashlib
27 
28 if "RELMON_SA" in os.environ:
29  from .dirstructure import Comparison,Directory
30  from .definitions import *
31  from .utils import unpickler
32 else:
33  from Utilities.RelMon.dirstructure import Comparison,Directory
34  from Utilities.RelMon.definitions import *
35  from Utilities.RelMon.utils import unpickler
36 
37 #-------------------------------------------------------------------------------
38 
39 def encode_obj_url(url):
40  for old,new in url_encode_dict.items():
41  url=url.replace(old,new)
42  return url
43 
44 def plot_size(h=250,w=200):
45  return "w=%s;h=%s" %(h,w)
46 
47 def build_obj_addr(run,sample,version,plot_path,tier):
48  slash="/"
49  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
50  obj_url+=plot_path
51  while obj_url.endswith(slash):
52  obj_url=obj_url[:-1]
53  while slash*2 in obj_url:
54  obj_url=obj_url.replace(slash*2,slash)
55  return obj_url
56 
57 def build_obj(run,sample,version,plot_path,tier):
58  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
59  return encode_obj_url(obj_url)
60 
61 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
62  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
63  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
64  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
65  if len(draw_opts)>0:
66  fairy_url+="drawopts=%s;" %draw_opts
67  fairy_url+= plot_size(h,w)
68  fairy_url += ";ref=ratiooverlay"
69  return fairy_url
70 
71 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
72  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
73  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
74  fairy_url+= "?%s"%plot_size(h,w)
75  if len(draw_opts)>0:
76  fairy_url+="drawopts=%s;" %draw_opts
77  return fairy_url
78 
79 #-------------------------------------------------------------------------------
80 style_location="/cms-service-reldqm"
81 def get_page_header(directory=None, standalone=False, additional_header=""):
82  style_location="/cms-service-reldqm"
83  if standalone:
84  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
85  javascripts=''
86  style=''
87  if directory!=None and len(directory.comparisons)>0:
88  meta=directory.meta
89  style='img.fail {border:1px solid #ff0000;}\n'+\
90  'img.succes {border:1px solid #00ff00;}\n'+\
91  'img.null {border:1px solid #ffff00;}\n'+\
92  'img.skiped {border:1px solid #7a7a7a;}\n'+\
93  'a.black_link:link {color: #333333}\n'+\
94  'a.black_link:hover {color: #737373}\n'+\
95  'a.black_link:visited {color: #333333}\n'+\
96  'a.black_link:active {color: #333333}\n'
97  javascripts=""
98 
99 
100  html='<html>'+\
101  '<head>'+\
102  '<title>RelMon Summary</title>'+\
103  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
104  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
105  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
106  '<style type="text/css">'+\
107  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
108  '%s'%style+\
109  '</style>'+\
110  '%s'%javascripts+\
111  '%s'%additional_header+\
112  '</head>'+\
113  '<body>'+\
114  '<div class="container">'
115 
116  return html
117 
118 #-------------------------------------------------------------------------------
119 
120 def get_page_footer():
121  return '</div></body></html>'
122 
123 #-------------------------------------------------------------------------------
124 
125 def get_title_section(directory, hashing_flag, standalone, depth=2):
126  if standalone:
127  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
128  else:
129  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
130  mother_name=basename(directory.mother_dir)
131  mother_file_name=""
132  if depth==1:
133  mother_file_name="../RelMonSummary.html"
134  if mother_name!="":
135  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
136  elif depth==2:
137  mother_file_name="RelMonSummary.html"
138  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
139  if mother_name!="":
140  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
141  else:
142  if hashing_flag:
143  files = directory.mother_dir.split("/")
144  if len(files) != 1:
145  dir_name = files[-2]+files[-1]
146  else:
147  dir_name = files[-1]
148  dir_name = directory.mother_dir
149  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
150  else:
151  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
152  mother_file_name=mother_file_name.strip("_")
153 
154  link_to_mother='<a href="%s">..</a>' %mother_file_name
155  html= '<div class="span-20">'+\
156  '<h1>%s</h1>'%directory.name+\
157  '</div>'+\
158  '<div class="span-1">'+\
159  '<h1>%s</h1>'%link_to_mother+\
160  '</div>'+\
161  '<div class="span-3 last">'+\
162  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
163  '</div>'+\
164  '<hr>'
165  if len(mother_name)>0:
166  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
167  '<hr>'
168  return html
169 
170 #-------------------------------------------------------------------------------
171 def get_dir_stats(directory):
172  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
173  html+='<ul>'
174  if directory.n_successes>0:
175  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
176  if directory.n_nulls>0:
177  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
178  if directory.n_fails>0:
179  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
180  if directory.n_skiped>0:
181  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
182  if directory.n_missing_objs>0:
183  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
184  html+='</ul>'
185  return html
186 
187 #-------------------------------------------------------------------------------
188 
189 def get_subdirs_section(directory, hashing_flag):
190  if len(directory.subdirs)==0:
191  return ""
192  html= '<div class="span-20 colborder">'
193  html+='<h2 class="alt">Sub-Directories</h2>'
194  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
195  # so reverse :)
196  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
197  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
198  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
199  for subdir in sorted_subdirs:
200  name=subdir.name
201  if hashing_flag:
202  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
203  else:
204  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
205  link=link.strip("_")
206  html+='<div class="span-4 prepend-2 colborder">'
207  html+='<h3>%s</h3>'%name
208  html+='</div>'
209 
210  html+='<div class="span-7">'
211  html+=get_dir_stats(subdir)
212  html+='</div>'
213 
214  html+='<div class="span-6 last">'
215  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
216  html+='</div>'
217 
218  html+='<hr>'
219  return html+'</div>'
220 
221 
222 #-------------------------------------------------------------------------------
223 
224 def get_summary_section(directory,matrix_page=True):
225 
226  # Hack find the first comparison and fill test and threshold
227  # shall this be put in meta?
228  test_name=""
229  test_threshold=0
230  for comparison in directory.comparisons:
231  test_name=comparison.test_name
232  test_threshold=comparison.test_thr
233  break
234  if len(test_name)==0:
235  for subdir in directory.subdirs:
236  for comparison in subdir.comparisons:
237  test_name=comparison.test_name
238  test_threshold=comparison.test_thr
239  break
240  if len(test_name)!=0:break
241  if len(test_name)==0:
242  for subsubdir in subdir.subdirs:
243  for comparison in subsubdir.comparisons:
244  test_name=comparison.test_name
245  test_threshold=comparison.test_thr
246  break
247  if len(test_name)!=0:break
248  if len(test_name)==0:
249  for subsubsubdir in subsubdir.subdirs:
250  for comparison in subsubsubdir.comparisons:
251  test_name=comparison.test_name
252  test_threshold=comparison.test_thr
253  break
254  if len(test_name)!=0:break
255 
256 
257  meta=directory.meta
258 
259  html= '<div class="span-6">'+\
260  '<h3>Summary</h3>'
261  html+=get_dir_stats(directory)
262  html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
263  html+='</div>'
264 
265  html+='<div class="span-7 colborder">'+\
266  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
267  '</div>'+\
268  '<div class="span-9 last">'
269  if matrix_page:
270  html+='<h3>Sample:</h3>'+\
271  '<p class="caps">%s</p>'%meta.sample1+\
272  '<h3>Run1 and Run2:</h3>'+\
273  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
274  html+='<h3>Releases:</h3>'+\
275  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
276  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
277  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
278  '</div>'+\
279  '<hr>'
280  return html
281 
282 #-------------------------------------------------------------------------------
283 
284 def get_comparisons(category,directory):
285  """Prepare the comparisons between histograms and organise them in the page.
286  Moreover create separate pages with the overlay and the single plots.
287  """
288  counter=1
289  tot_counter=1
290 
291  # get the right ones
292  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
293  n_comparisons=len(comparisons)
294 
295  is_reverse=True
296  if category == FAIL:
297  is_reverse=False
298  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
299 
300 
301  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
302  html_comparisons=""
303  for comparison in comparisons:
304  plot_name=comparison.img_name
305  if "http://" not in plot_name:
306  plot_name= basename(comparison.img_name)
307  class_type="colborder"
308  if counter==3 or tot_counter==n_comparisons:
309  class_type=" colborder last"
310  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
311 
312 
313  if directory.do_pngs:
314  png_link=comparison.img_name
315  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
316  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
317  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
318  else:
319  big_fairy=fairy_url(directory.meta.run1,
320  directory.meta.run2,
321  directory.meta.sample1,
322  directory.meta.sample2,
323  directory.meta.release1,
324  directory.meta.release2,
325  comp_abs_path,
326  directory.meta.tier1,
327  directory.meta.tier2,
328  "",600,600)
329  small_fairy=fairy_url(directory.meta.run1,
330  directory.meta.run2,
331  directory.meta.sample1,
332  directory.meta.sample2,
333  directory.meta.release1,
334  directory.meta.release2,
335  comp_abs_path,
336  directory.meta.tier1,
337  directory.meta.tier2)
338 
339  single_fairy1=fairy_url_single(directory.meta.run1,
340  directory.meta.sample1,
341  directory.meta.release1,
342  comp_abs_path,
343  directory.meta.tier1,
344  "",500,500)
345  single_fairy2=fairy_url_single(directory.meta.run2,
346  directory.meta.sample2,
347  directory.meta.release2,
348  comp_abs_path,
349  directory.meta.tier2,
350  "",500,500)
351 
352  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
353  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
354  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
355  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
356  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
357 
358  if counter==3:
359  html_comparisons+="<hr>"
360  counter=0
361  counter+=1
362  tot_counter+=1
363 
364  if len(html_comparisons)!=0:
365  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
366  html+=html_comparisons
367  html+='<hr>'
368  return html
369  return ""
370 
371 #-------------------------------------------------------------------------------
372 
373 def get_rank_section(directory):
374 # do Rank histo png
375  imgname="RankSummary.png"
376  gStyle.SetPadTickY(0)
377  c=TCanvas("ranks","ranks",500,400)
378  #gStyle.SetOptStat(0)
379  c.cd()
380 
381  h=directory.rank_histo
382  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
383  rank_histof.SetLineWidth(2)
384  for i in range(0,h.GetNbinsX()+1):
385  rank_histof.SetBinContent(i,h.GetBinContent(i))
386  h.SetTitle("Ranks Summary;Rank;Frequency")
387  h.Draw("Hist")
388  c.Update()
389  rank_histof.ComputeIntegral()
390  integral = rank_histof.GetIntegral()
391  rank_histof.SetContent(integral)
392 
393  rightmax = 1.1*rank_histof.GetMaximum()
394  scale = gPad.GetUymax()/rightmax
395  rank_histof.SetLineColor(kRed)
396  rank_histof.Scale(scale)
397  rank_histof.Draw("same")
398 
399  #draw an axis on the right side
400  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
401  axis.SetTitle("Cumulative")
402  axis.SetTitleColor(kRed)
403  axis.SetLineColor(kRed)
404  axis.SetLabelColor(kRed)
405  axis.Draw()
406 
407  rank_histof.Draw("Same");
408 
409 
410  c.Print(imgname)
411 
412  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
413  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
414  page_html+='</div> <a href="#top">Top...</a><hr>'
415 
416  return page_html
417 
418 #-------------------------------------------------------------------------------
419 
420 #-------------------------------------------------------------------------------
421 def get_missing_objs_section(directory):
422  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
423  """
424  page_html = "Unpaired in %s</br>"%(directory.filename1)
425  for elem in directory.different_histograms['file1']:
426  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
427  page_html +="</br>"
428  page_html += "Unpaired in %s</br>"%(directory.filename2)
429  for elem in directory.different_histograms['file2']:
430  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
431  return page_html
432 #-------------------------------------------------------------------------------
433 
434 def directory2html(directory, hashing, standalone, depth=0):
435  """Converts a directory tree into html pages, very nice ones.
436  """
437  #print "d2html: depth", str(depth)," dir ",directory.name
438  depth+=1
439  #old_cwd=getcwd()
440  #if not exists(directory.name) and len(directory.name)>0:
441  #makedirs(directory.name)
442 
443  #if len(directory.name)>0:
444  #chdir(directory.name)
445 
446  for subdir in directory.subdirs:
447  directory2html(subdir,hashing,standalone, depth)
448 
449  page_html=get_page_header(directory, standalone)+\
450  get_title_section(directory,hashing, standalone, depth)+\
451  get_summary_section(directory)+\
452  get_subdirs_section(directory, hashing)
453 
454  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
455  (directory.n_comp_nulls >0,NULL ),
456 
457  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
458  (directory.n_comp_skiped >0,SKIPED)):
459  if do_cat:
460  page_html+=get_comparisons(cat,directory)
461 
462  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
463  page_html += get_missing_objs_section(directory)
464 
465  # Distribution of ranks
466 
467  if depth==1:
468  page_html+=get_rank_section(directory)
469 
470 
471  page_html+=get_page_footer()
472 
473  page_name=directory.name
474 
475  if len(page_name)==0:
476  page_name="RelMonSummary"
477  if hashing:
478  if page_name != "RelMonSummary":
479  #print " ## oFile path: %s"%(join(directory.full_path))
480  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
481  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
482  else:
483  ofilename = "RelMonSummary.html"
484  else:
485  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
486  ofilename=ofilename.strip("_")
487  #print "Writing on %s" %ofilename
488  ofile=open(ofilename,"w")
489  ofile.write(page_html)
490  ofile.close()
491 
492  #chdir(old_cwd)
493 
494 #-------------------------------------------------------------------------------
495 
496 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
497  total_success_rate_scaled=(total_success_rate-minrate)
498  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
499  if total_success_rate_scaled_repr<0:
500  total_success_rate_scaled_repr=0
501  size_s="200x100"
502  if small:
503  size_s="40x30"
504  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
505  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
506  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
507  if not small:
508  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
509  gauge_link+="&chma=10,10,10,0"
510  img_tag= '<img src="%s">'%gauge_link
511  if escaped:
512  img_tag=html.escape(img_tag)
513  return img_tag
514 
515 #-------------------------------------------------------------------------------
516 
517 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
518  # Let's make a summary for an overview in categories act on the global dir
519  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
520 
521  list_of_names=[]
522  if the_aggr_pairs==[]:
523  for samplename,sampledir in dir_dict.items():
524  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
525  if not subsysdirname in list_of_names:
526  list_of_names.append(subsysdirname)
527  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
528 
529  #print the_aggr_pairs
530  for cat_name, subdir_list in the_aggr_pairs:
531  total_successes=0.
532  total_directory_successes=0
533  total_weight=0.
534  present_subdirs={}
535  total_ndirs=0
536  # Loop on samples
537  for dirname, sampledir in dir_dict.items():
538  # Loop on directories
539  for subdirname,subdir in sampledir.get_subdirs_dict().items():
540  if subdirname in subdir_list:
541  nsucc=subdir.n_successes
542  total_successes+=nsucc
543  weight=subdir.weight
544  total_weight+=weight
545  total_ndirs+=1
546 
547  total_directory_successes+= float(nsucc)/weight
548  if subdirname in present_subdirs:
549  this_dir_dict=present_subdirs[subdirname]
550  this_dir_dict["nsucc"]+=nsucc
551  this_dir_dict["weight"]+=weight
552  else:
553  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
554  # Make it usable also for subdirectories
555  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
556  for pathname in [name for name in subdir_list if "/" in name]:
557  selected_subdirname,selected_subsubdirname = pathname.split("/")
558  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
559  #print "Studying directory ",subsubdirname," in directory ",subdirname
560  nsucc=subsubdir.n_successes
561  total_successes+=nsucc
562  weight=subsubdir.weight
563  total_weight+=weight
564  total_ndirs+=1
565  total_directory_successes+= float(nsucc)/weight
566 
567  if subsubdirname in present_subdirs:
568  this_dir_dict=present_subdirs[subsubdirname]
569  this_dir_dict["nsucc"]+=nsucc
570  this_dir_dict["weight"]+=weight
571  else:
572  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
573 
574  if total_ndirs == 0:
575  print("No directory of the category %s is present in the samples: skipping." %cat_name)
576  continue
577 
578  average_success_rate=total_directory_successes/(total_ndirs)
579  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
580 
581  return aggr_pairs_info
582 
583 #-------------------------------------------------------------------------------
584 
585 def make_categories_summary(dir_dict,aggregation_rules):
586 
587  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
588 
589  #print aggr_pairs_info
590 
591  # Now Let's build the HTML
592 
593  html= '<div class="span-20 colborder">'
594  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
595 
596  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
597  #print cat_name,present_subdirs,total_weight,average_success_rate
598  html+='<div class="span-3 prepend-0 colborder">'
599  html+='<h3>%s</h3>'%cat_name
600  html+='<div><span class="alt">Avg. Success rate:</span></div>'
601  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
602  html+='</div>'
603  html+='<div class="span-9">'
604 
605  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
606  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
607  html+='<ul>'
608  for subdirname in sorted(present_subdirs.keys()):
609  this_dir_dict=present_subdirs[subdirname]
610  nsucc=this_dir_dict["nsucc"]
611  weight=this_dir_dict["weight"]
612  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
613  html+='</ul>'
614  html+='</div>'
615 
616  html+='<div class="span-6 last">'
617  html+=build_gauge(average_success_rate)
618  html+='</div>'
619 
620  html+='<hr>'
621  return html+'<br><a href="#top">Top...</a> </div><hr>'
622 
623  #-------------------------------------------------------------------------------
624 
625 def make_twiki_table(dir_dict,aggregation_rules):
626 
627  # decide the release
628  meta= list(dir_dict.items())[0][1].meta
629  releases=sorted([meta.release1,meta.release2])
630  latest_release=releases[1].split("-")[0]
631 
632 
633  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
634 
635  # Now Let's build the HTML
636 
637  html= '<div class="span-20 colborder">'
638  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
639  html+='<div>| Release | Comparison |'
640  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
641  html+=cat_name
642  html+=" | "
643  html+='</div>'
644 
645  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
646 
647  # Now add all the line with small gauges
648 
649  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
650  #print cat_name,present_subdirs,total_weight,average_success_rate
651  html+=build_gauge(average_success_rate,small=True,escaped=True)
652  html+=" | "
653 
654  html+='</div> <a href="#top">Top...</a>'
655  html+='<hr>'
656  return html+'</div>'
657 
658 #-------------------------------------------------------------------------------
659 
660 def get_pie_tooltip(directory):
661  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
662  return tooltip
663 
664 #-------------------------------------------------------------------------------
665 
666 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
667 
668  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
669 
670  script="""
671  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
672  <script type="text/javascript">
673  google.load("visualization", "1", {packages:["corechart"]});
674  google.setOnLoadCallback(drawChart);
675  function drawChart() {
676  var data = new google.visualization.DataTable();
677  data.addColumn('string', 'DQM Directory');
678  data.addColumn('number', 'Success Rate');
679  """
680  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
681  counter=0
682  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
683  #print subsystname,present_directories
684  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
685  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
686  counter+=1
687  script+="""
688  var chart = new google.visualization.BarChart(document.getElementById('%s'));
689  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
690  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
691  });
692  }
693  </script>
694  """%(name,40*counter,title)
695  return script
696 
697 
698 #-------------------------------------------------------------------------------
699 
700 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
701  """Create a table, with as rows the directories and as columns the samples.
702  Each box in the table will contain a pie chart linking to the directory.
703  """
704  #aggregation_rules={}
705  #aggregation_rules_twiki={}
706 
707  chdir(indir)
708  if os.path.isabs(indir):
709  title = basename(indir)
710  else:
711  title=indir
712  title=title.strip(".")
713  title=title.strip("/")
714 
715 
716  # Get the list of pickles
717  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
718 
719  # Load directories, build a list of all first level subdirs
720  dir_unpicklers=[]
721  n_unpicklers=0
722  for sample_pkl in sample_pkls:
723  dir_unpickler=unpickler(sample_pkl)
724  dir_unpickler.start()
725  n_unpicklers+=1
726  dir_unpicklers.append(dir_unpickler)
727  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
728  n_unpicklers=0
729  for dir_unpickler in dir_unpicklers:
730  dir_unpickler.join()
731 
732  dir_dict={}
733 
734  # create a fake global directory
735  global_dir=Directory("global","")
736  for dir_unpickler in dir_unpicklers:
737  dir_unpickler.join()
738  directory=dir_unpickler.directory
739  #directory.prune("Run summary")
740  #directory.calcStats()
741  global_dir.meta=directory.meta
742  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
743  global_dir.subdirs.append(directory)
744 
745  global_dir.calcStats()
746 
747  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
748  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
749 
750  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
751  rel1=""
752  rel2=""
753  try:
754  rel1,rel2=title.split("VS")
755  except:
756  rel1=global_dir.meta.release1.split("-")[0]
757  rel2=global_dir.meta.release2.split("-")[0]
758  global_dir.meta.release1=rel1
759  global_dir.meta.release2=rel2
760 
761  # union of all subdirs names
762  all_subdirs=[]
763  for directory in dir_dict.values():
764  for subdir_name in directory.get_subdirs_names():
765  all_subdirs.append(subdir_name)
766  all_subdirs=sorted(list(set(all_subdirs)))
767 
768  #print " $$ all_subdirs: %s" %(all_subdirs)
769 
770  # Get The title
771  page_html+= '<div class="span-20">'+\
772  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
773  '</div>'+\
774  '<div class="span-1">'+\
775  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
776  '</div>'+\
777  '<hr>'
778  page_html+='<div class="span-24"><p></p></div>\n'*3
779 
780  # Get The summary section
781  page_html+= get_summary_section(global_dir,False)
782 
783  # Make the anchor sections
784  page_html+= '<div class="span-24">'
785  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
786  '<ul>'+\
787  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
788  '<li><a href="#categories">Categories</a></li>'+\
789  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
790  '<li><a href="#summary_table">Summary Table</a></li>'+\
791  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
792  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
793  '</ul>'+\
794  '</div><hr>'
795 
796 
797  # Make the CategoriesBar chart
798  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
799  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
800 
801  # Make the gauges per categories
802  page_html+=make_categories_summary(dir_dict,aggregation_rules)
803 
804  # Make the Directories chart
805  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
806  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
807 
808  # Barbarian vertical space. Suggestions are welcome
809  for i in range(2):
810  page_html+='<div class="span-24"><p></p></div>\n'
811 
812 
813  # Prepare the table
814  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
815 
816  for i in range(5):
817  page_html+='<div class="span-24"><p></p></div>\n'
818 
819  page_html+="""
820  <table border="1" >
821  <tr>
822  <td> </td>
823  """
824 
825  # First row with samples
826  page_html+="""
827  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
828 
829  sorted_samples=sorted(dir_dict.keys())
830  for sample in sorted_samples:
831  sample_nick=sample
832 
837 
838 
839  page_html+="""
840  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
841  page_html+=" </tr>\n"
842 
843 
844  # FIRST ROW
845  # Now the summaries at the beginning of the table
846  page_html+="<tr>"
847  page_html+='<td style="background-color:white;"><div class="span-1">'
848 
849  page_html+='<b>Summary</b></div></td>'
850  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
851  for sample in sorted_samples:
852  col=dir_dict[sample]
853  # check if the directory was a top one or not
854  summary_page_name="RelMonSummary.html"
855  if col.name!="":
856  summary_page_name=hash_name(col.name, hashing_flag)+".html"
857  img_link=col.get_summary_chart_ajax(55,55)
858  page_html+='<td style="background-color:white;"><div class="span-1">'
859  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
860  page_html+="</tr>"
861 
862  # Now the content
863  for subdir_name in all_subdirs:
864 
865  page_html+=' <tr>\n'
866  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
867 
868  row_summary=Directory("row_summary","")
869  sample_counter=0
870  n_samples=len(sorted_samples)
871 
872  for sample in sorted_samples:
873  subdirs_dict=directory.get_subdirs_dict()
874  directory=dir_dict[sample]
875  dir_is_there=subdir_name in subdirs_dict
876  if dir_is_there:
877  row_summary.subdirs.append(subdirs_dict[subdir_name])
878 
879  # one first row for the summary!
880  row_summary.calcStats()
881  img_link=row_summary.get_summary_chart_ajax(55,55)
882  page_html+='<td style="background-color:white;"><div class="span-1">'
883  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
884 
885  for sample in sorted_samples:
886  sample_counter+=1
887 
888  directory=dir_dict[sample]
889  subdirs_dict=directory.get_subdirs_dict()
890 
891  # Check if the directory is the top one
892  if directory.name!="":
893  # We did not run on the topdir
894  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
895  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
896  if hashing_flag:
897  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
898  else:
899  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
900  else:
901  #print " ## summary_page: %s"%(directory.name+subdir_name)
902  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
903  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
904  dir_is_there=subdir_name in subdirs_dict
905 
906  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
907  img_tooltip="N/A"
908  if dir_is_there:
909  #row_summary.subdirs.append(subdirs_dict[subdir_name])
910  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
911  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
912 
913  page_html+='<td style="background-color:white;"><div class="span-1">'
914  if dir_is_there:
915  page_html+='<a href="%s">'%(summary_page)
916  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
917  if dir_is_there:
918  page_html+='</a>'
919  page_html+='</div></td>'
920 
921  page_html+=" </tr>\n"
922 
923 
924 
925  page_html+='</table> <a href="#top">Top...</a><hr>'
926 
927  page_html+=get_rank_section(global_dir)
928 
929  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
930 
931  page_html+=get_page_footer()
932  return page_html
933 
934 
935 #-----------UPDATES------
936 def hash_name(file_name, flag):
937  if flag: #if hashing flag is ON then return
938  if (3,0,0) <= sys.version_info:
939  return hashlib.md5(file_name.encode('utf-8')).hexdigest()[:10]
940  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
941  else:
942  return file_name #return standart name
directories2html.build_gauge
def build_gauge(total_success_rate, minrate=.80, small=False, escaped=False)
Definition: directories2html.py:496
directories2html.hash_name
def hash_name(file_name, flag)
Definition: directories2html.py:936
directories2html.get_comparisons
def get_comparisons(category, directory)
Definition: directories2html.py:284
directories2html.get_subdirs_section
def get_subdirs_section(directory, hashing_flag)
Definition: directories2html.py:189
directories2html.get_pie_tooltip
def get_pie_tooltip(directory)
Definition: directories2html.py:660
directories2html.build_obj
def build_obj(run, sample, version, plot_path, tier)
Definition: directories2html.py:57
directories2html.get_aggr_pairs_info
def get_aggr_pairs_info(dir_dict, the_aggr_pairs=[])
Definition: directories2html.py:517
directories2html.make_summary_table
def make_summary_table(indir, aggregation_rules, aggregation_rules_twiki, hashing_flag, standalone_flag)
Definition: directories2html.py:700
nano_mu_digi_cff.float
float
Definition: nano_mu_digi_cff.py:13
directories2html.plot_size
def plot_size(h=250, w=200)
Definition: directories2html.py:44
directories2html.get_page_header
def get_page_header(directory=None, standalone=False, additional_header="")
Definition: directories2html.py:81
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
directories2html.get_title_section
def get_title_section(directory, hashing_flag, standalone, depth=2)
Definition: directories2html.py:125
directories2html.get_dir_stats
def get_dir_stats(directory)
Definition: directories2html.py:171
directories2html.make_barchart_summary
def make_barchart_summary(dir_dict, name="the_chart", title="DQM directory", the_aggr_pairs=[])
Definition: directories2html.py:666
directories2html.get_summary_section
def get_summary_section(directory, matrix_page=True)
Definition: directories2html.py:224
directories2html.directory2html
def directory2html(directory, hashing, standalone, depth=0)
Definition: directories2html.py:434
directories2html.make_twiki_table
def make_twiki_table(dir_dict, aggregation_rules)
Definition: directories2html.py:625
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
directories2html.make_categories_summary
def make_categories_summary(dir_dict, aggregation_rules)
Definition: directories2html.py:585
FastTimerService_cff.range
range
Definition: FastTimerService_cff.py:34
directories2html.fairy_url_single
def fairy_url_single(run, sample, version, plot_path, tier, draw_opts="", h=250, w=200)
Definition: directories2html.py:71
directories2html.fairy_url
def fairy_url(run1, run2, sample1, sample2, version1, version2, plot_path, tier1, tier2, draw_opts="", h=250, w=200)
Definition: directories2html.py:61
directories2html.build_obj_addr
def build_obj_addr(run, sample, version, plot_path, tier)
Definition: directories2html.py:47
directories2html.get_missing_objs_section
def get_missing_objs_section(directory)
Definition: directories2html.py:421
directories2html.get_rank_section
def get_rank_section(directory)
Definition: directories2html.py:373
Generated for CMSSW Reference Manual by doxygen 1.8.14