CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
directories2html.py
Go to the documentation of this file.
1 from __future__ import print_function
2 from __future__ import absolute_import
3 ################################################################################
4 # RelMon: a tool for automatic Release Comparison
5 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
6 #
7 
8 #
9 #
10 # Danilo Piparo CERN - danilo.piparo@cern.ch
11 #
12 ################################################################################
13 
14 from builtins import range
15 from os import chdir,getcwd,listdir,makedirs
16 from os.path import basename,join,exists
17 import html
18 
19 import sys
20 theargv=sys.argv
21 sys.argv=[]
22 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
23 sys.argv=theargv
24 
25 import os
26 if "RELMON_SA" in os.environ:
27  from .dirstructure import Comparison,Directory
28  from .definitions import *
29  from .utils import unpickler
30 else:
31  from Utilities.RelMon.dirstructure import Comparison,Directory
32  from Utilities.RelMon.definitions import *
33  from Utilities.RelMon.utils import unpickler
34 
35  import hashlib
36 #-------------------------------------------------------------------------------
37 
38 def encode_obj_url(url):
39  for old,new in url_encode_dict.items():
40  url=url.replace(old,new)
41  return url
42 
43 def plot_size(h=250,w=200):
44  return "w=%s;h=%s" %(h,w)
45 
46 def build_obj_addr(run,sample,version,plot_path,tier):
47  slash="/"
48  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
49  obj_url+=plot_path
50  while obj_url.endswith(slash):
51  obj_url=obj_url[:-1]
52  while slash*2 in obj_url:
53  obj_url=obj_url.replace(slash*2,slash)
54  return obj_url
55 
56 def build_obj(run,sample,version,plot_path,tier):
57  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
58  return encode_obj_url(obj_url)
59 
60 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
61  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
62  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
63  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
64  if len(draw_opts)>0:
65  fairy_url+="drawopts=%s;" %draw_opts
66  fairy_url+= plot_size(h,w)
67  fairy_url += ";ref=ratiooverlay"
68  return fairy_url
69 
70 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
71  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
72  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
73  fairy_url+= "?%s"%plot_size(h,w)
74  if len(draw_opts)>0:
75  fairy_url+="drawopts=%s;" %draw_opts
76  return fairy_url
77 
78 #-------------------------------------------------------------------------------
79 style_location="/cms-service-reldqm"
80 def get_page_header(directory=None, standalone=False, additional_header=""):
81  style_location="/cms-service-reldqm"
82  if standalone:
83  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
84  javascripts=''
85  style=''
86  if directory!=None and len(directory.comparisons)>0:
87  meta=directory.meta
88  style='img.fail {border:1px solid #ff0000;}\n'+\
89  'img.succes {border:1px solid #00ff00;}\n'+\
90  'img.null {border:1px solid #ffff00;}\n'+\
91  'img.skiped {border:1px solid #7a7a7a;}\n'+\
92  'a.black_link:link {color: #333333}\n'+\
93  'a.black_link:hover {color: #737373}\n'+\
94  'a.black_link:visited {color: #333333}\n'+\
95  'a.black_link:active {color: #333333}\n'
96  javascripts=""
97 
98 
99  html='<html>'+\
100  '<head>'+\
101  '<title>RelMon Summary</title>'+\
102  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
103  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
104  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
105  '<style type="text/css">'+\
106  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
107  '%s'%style+\
108  '</style>'+\
109  '%s'%javascripts+\
110  '%s'%additional_header+\
111  '</head>'+\
112  '<body>'+\
113  '<div class="container">'
114 
115  return html
116 
117 #-------------------------------------------------------------------------------
118 
119 def get_page_footer():
120  return '</div></body></html>'
121 
122 #-------------------------------------------------------------------------------
123 
124 def get_title_section(directory, hashing_flag, standalone, depth=2):
125  if standalone:
126  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
127  else:
128  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
129  mother_name=basename(directory.mother_dir)
130  mother_file_name=""
131  if depth==1:
132  mother_file_name="../RelMonSummary.html"
133  if mother_name!="":
134  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
135  elif depth==2:
136  mother_file_name="RelMonSummary.html"
137  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
138  if mother_name!="":
139  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
140  else:
141  if hashing_flag:
142  files = directory.mother_dir.split("/")
143  if len(files) != 1:
144  dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
145  else:
146  dir_name = files[-1]
147  dir_name = directory.mother_dir
148  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
149  else:
150  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
151  mother_file_name=mother_file_name.strip("_")
152 
153  link_to_mother='<a href="%s">..</a>' %mother_file_name
154  html= '<div class="span-20">'+\
155  '<h1>%s</h1>'%directory.name+\
156  '</div>'+\
157  '<div class="span-1">'+\
158  '<h1>%s</h1>'%link_to_mother+\
159  '</div>'+\
160  '<div class="span-3 last">'+\
161  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
162  '</div>'+\
163  '<hr>'
164  if len(mother_name)>0:
165  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
166  '<hr>'
167  return html
168 
169 #-------------------------------------------------------------------------------
170 def get_dir_stats(directory):
171  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
172  html+='<ul>'
173  if directory.n_successes>0:
174  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
175  if directory.n_nulls>0:
176  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
177  if directory.n_fails>0:
178  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
179  if directory.n_skiped>0:
180  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
181  if directory.n_missing_objs>0:
182  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
183  html+='</ul>'
184  return html
185 
186 #-------------------------------------------------------------------------------
187 
188 def get_subdirs_section(directory, hashing_flag):
189  if len(directory.subdirs)==0:
190  return ""
191  html= '<div class="span-20 colborder">'
192  html+='<h2 class="alt">Sub-Directories</h2>'
193  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
194  # so reverse :)
195  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
196  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
197  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
198  for subdir in sorted_subdirs:
199  name=subdir.name
200  if hashing_flag:
201  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
202  else:
203  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
204  link=link.strip("_")
205  html+='<div class="span-4 prepend-2 colborder">'
206  html+='<h3>%s</h3>'%name
207  html+='</div>'
208 
209  html+='<div class="span-7">'
210  html+=get_dir_stats(subdir)
211  html+='</div>'
212 
213  html+='<div class="span-6 last">'
214  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
215  html+='</div>'
216 
217  html+='<hr>'
218  return html+'</div>'
219 
220 
221 #-------------------------------------------------------------------------------
222 
223 def get_summary_section(directory,matrix_page=True):
224 
225  # Hack find the first comparison and fill test and threshold
226  # shall this be put in meta?
227  test_name=""
228  test_threshold=0
229  for comparison in directory.comparisons:
230  test_name=comparison.test_name
231  test_threshold=comparison.test_thr
232  break
233  if len(test_name)==0:
234  for subdir in directory.subdirs:
235  for comparison in subdir.comparisons:
236  test_name=comparison.test_name
237  test_threshold=comparison.test_thr
238  break
239  if len(test_name)!=0:break
240  if len(test_name)==0:
241  for subsubdir in subdir.subdirs:
242  for comparison in subsubdir.comparisons:
243  test_name=comparison.test_name
244  test_threshold=comparison.test_thr
245  break
246  if len(test_name)!=0:break
247  if len(test_name)==0:
248  for subsubsubdir in subsubdir.subdirs:
249  for comparison in subsubsubdir.comparisons:
250  test_name=comparison.test_name
251  test_threshold=comparison.test_thr
252  break
253  if len(test_name)!=0:break
254 
255 
256  meta=directory.meta
257 
258  html= '<div class="span-6">'+\
259  '<h3>Summary</h3>'
260  html+=get_dir_stats(directory)
261  html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
262  html+='</div>'
263 
264  html+='<div class="span-7 colborder">'+\
265  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
266  '</div>'+\
267  '<div class="span-9 last">'
268  if matrix_page:
269  html+='<h3>Sample:</h3>'+\
270  '<p class="caps">%s</p>'%meta.sample1+\
271  '<h3>Run1 and Run2:</h3>'+\
272  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
273  html+='<h3>Releases:</h3>'+\
274  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
275  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
276  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
277  '</div>'+\
278  '<hr>'
279  return html
280 
281 #-------------------------------------------------------------------------------
282 
283 def get_comparisons(category,directory):
284  """Prepare the comparisons between histograms and organise them in the page.
285  Moreover create separate pages with the overlay and the single plots.
286  """
287  counter=1
288  tot_counter=1
289 
290  # get the right ones
291  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
292  n_comparisons=len(comparisons)
293 
294  is_reverse=True
295  if category == FAIL:
296  is_reverse=False
297  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
298 
299 
300  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
301  html_comparisons=""
302  for comparison in comparisons:
303  plot_name=comparison.img_name
304  if "http://" not in plot_name:
305  plot_name= basename(comparison.img_name)
306  class_type="colborder"
307  if counter==3 or tot_counter==n_comparisons:
308  class_type=" colborder last"
309  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
310 
311 
312  if directory.do_pngs:
313  png_link=comparison.img_name
314  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
315  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
316  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
317  else:
318  big_fairy=fairy_url(directory.meta.run1,
319  directory.meta.run2,
320  directory.meta.sample1,
321  directory.meta.sample2,
322  directory.meta.release1,
323  directory.meta.release2,
324  comp_abs_path,
325  directory.meta.tier1,
326  directory.meta.tier2,
327  "",600,600)
328  small_fairy=fairy_url(directory.meta.run1,
329  directory.meta.run2,
330  directory.meta.sample1,
331  directory.meta.sample2,
332  directory.meta.release1,
333  directory.meta.release2,
334  comp_abs_path,
335  directory.meta.tier1,
336  directory.meta.tier2)
337 
338  single_fairy1=fairy_url_single(directory.meta.run1,
339  directory.meta.sample1,
340  directory.meta.release1,
341  comp_abs_path,
342  directory.meta.tier1,
343  "",500,500)
344  single_fairy2=fairy_url_single(directory.meta.run2,
345  directory.meta.sample2,
346  directory.meta.release2,
347  comp_abs_path,
348  directory.meta.tier2,
349  "",500,500)
350 
351  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
352  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
353  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
354  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
355  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
356 
357  if counter==3:
358  html_comparisons+="<hr>"
359  counter=0
360  counter+=1
361  tot_counter+=1
362 
363  if len(html_comparisons)!=0:
364  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
365  html+=html_comparisons
366  html+='<hr>'
367  return html
368  return ""
369 
370 #-------------------------------------------------------------------------------
371 
372 def get_rank_section(directory):
373 # do Rank histo png
374  imgname="RankSummary.png"
375  gStyle.SetPadTickY(0)
376  c=TCanvas("ranks","ranks",500,400)
377  #gStyle.SetOptStat(0)
378  c.cd()
379 
380  h=directory.rank_histo
381  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
382  rank_histof.SetLineWidth(2)
383  for i in range(0,h.GetNbinsX()+1):
384  rank_histof.SetBinContent(i,h.GetBinContent(i))
385  h.SetTitle("Ranks Summary;Rank;Frequency")
386  h.Draw("Hist")
387  c.Update()
388  rank_histof.ComputeIntegral()
389  integral = rank_histof.GetIntegral()
390  rank_histof.SetContent(integral)
391 
392  rightmax = 1.1*rank_histof.GetMaximum()
393  scale = gPad.GetUymax()/rightmax
394  rank_histof.SetLineColor(kRed)
395  rank_histof.Scale(scale)
396  rank_histof.Draw("same")
397 
398  #draw an axis on the right side
399  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
400  axis.SetTitle("Cumulative")
401  axis.SetTitleColor(kRed)
402  axis.SetLineColor(kRed)
403  axis.SetLabelColor(kRed)
404  axis.Draw()
405 
406  rank_histof.Draw("Same");
407 
408 
409  c.Print(imgname)
410 
411  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
412  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
413  page_html+='</div> <a href="#top">Top...</a><hr>'
414 
415  return page_html
416 
417 #-------------------------------------------------------------------------------
418 
419 #-------------------------------------------------------------------------------
420 def get_missing_objs_section(directory):
421  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
422  """
423  page_html = "Unpaired in %s</br>"%(directory.filename1)
424  for elem in directory.different_histograms['file1']:
425  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
426  page_html +="</br>"
427  page_html += "Unpaired in %s</br>"%(directory.filename2)
428  for elem in directory.different_histograms['file2']:
429  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
430  return page_html
431 #-------------------------------------------------------------------------------
432 
433 def directory2html(directory, hashing, standalone, depth=0):
434  """Converts a directory tree into html pages, very nice ones.
435  """
436  #print "d2html: depth", str(depth)," dir ",directory.name
437  depth+=1
438  #old_cwd=getcwd()
439  #if not exists(directory.name) and len(directory.name)>0:
440  #makedirs(directory.name)
441 
442  #if len(directory.name)>0:
443  #chdir(directory.name)
444 
445  for subdir in directory.subdirs:
446  directory2html(subdir,hashing,standalone, depth)
447 
448  page_html=get_page_header(directory, standalone)+\
449  get_title_section(directory,hashing, standalone, depth)+\
450  get_summary_section(directory)+\
451  get_subdirs_section(directory, hashing)
452 
453  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
454  (directory.n_comp_nulls >0,NULL ),
455 
456  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
457  (directory.n_comp_skiped >0,SKIPED)):
458  if do_cat:
459  page_html+=get_comparisons(cat,directory)
460 
461  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
462  page_html += get_missing_objs_section(directory)
463 
464  # Distribution of ranks
465 
466  if depth==1:
467  page_html+=get_rank_section(directory)
468 
469 
470  page_html+=get_page_footer()
471 
472  page_name=directory.name
473 
474  if len(page_name)==0:
475  page_name="RelMonSummary"
476  if hashing:
477  if page_name != "RelMonSummary":
478  #print " ## oFile path: %s"%(join(directory.full_path))
479  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
480  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
481  else:
482  ofilename = "RelMonSummary.html"
483  else:
484  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
485  ofilename=ofilename.strip("_")
486  #print "Writing on %s" %ofilename
487  ofile=open(ofilename,"w")
488  ofile.write(page_html)
489  ofile.close()
490 
491  #chdir(old_cwd)
492 
493 #-------------------------------------------------------------------------------
494 
495 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
496  total_success_rate_scaled=(total_success_rate-minrate)
497  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
498  if total_success_rate_scaled_repr<0:
499  total_success_rate_scaled_repr=0
500  size_s="200x100"
501  if small:
502  size_s="40x30"
503  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
504  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
505  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
506  if not small:
507  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
508  gauge_link+="&chma=10,10,10,0"
509  img_tag= '<img src="%s">'%gauge_link
510  if escaped:
511  img_tag=html.escape(img_tag)
512  return img_tag
513 
514 #-------------------------------------------------------------------------------
515 
516 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
517  # Let's make a summary for an overview in categories act on the global dir
518  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
519 
520  list_of_names=[]
521  if the_aggr_pairs==[]:
522  for samplename,sampledir in dir_dict.items():
523  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
524  if not subsysdirname in list_of_names:
525  list_of_names.append(subsysdirname)
526  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
527 
528  #print the_aggr_pairs
529  for cat_name, subdir_list in the_aggr_pairs:
530  total_successes=0.
531  total_directory_successes=0
532  total_weight=0.
533  present_subdirs={}
534  total_ndirs=0
535  # Loop on samples
536  for dirname, sampledir in dir_dict.items():
537  # Loop on directories
538  for subdirname,subdir in sampledir.get_subdirs_dict().items():
539  if subdirname in subdir_list:
540  nsucc=subdir.n_successes
541  total_successes+=nsucc
542  weight=subdir.weight
543  total_weight+=weight
544  total_ndirs+=1
545 
546  total_directory_successes+= float(nsucc)/weight
547  if subdirname in present_subdirs:
548  this_dir_dict=present_subdirs[subdirname]
549  this_dir_dict["nsucc"]+=nsucc
550  this_dir_dict["weight"]+=weight
551  else:
552  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
553  # Make it usable also for subdirectories
554  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
555  for pathname in [name for name in subdir_list if "/" in name]:
556  selected_subdirname,selected_subsubdirname = pathname.split("/")
557  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
558  #print "Studying directory ",subsubdirname," in directory ",subdirname
559  nsucc=subsubdir.n_successes
560  total_successes+=nsucc
561  weight=subsubdir.weight
562  total_weight+=weight
563  total_ndirs+=1
564  total_directory_successes+= float(nsucc)/weight
565 
566  if subsubdirname in present_subdirs:
567  this_dir_dict=present_subdirs[subsubdirname]
568  this_dir_dict["nsucc"]+=nsucc
569  this_dir_dict["weight"]+=weight
570  else:
571  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
572 
573  if total_ndirs == 0:
574  print("No directory of the category %s is present in the samples: skipping." %cat_name)
575  continue
576 
577  average_success_rate=total_directory_successes/(total_ndirs)
578  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
579 
580  return aggr_pairs_info
581 
582 #-------------------------------------------------------------------------------
583 
584 def make_categories_summary(dir_dict,aggregation_rules):
585 
586  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
587 
588  #print aggr_pairs_info
589 
590  # Now Let's build the HTML
591 
592  html= '<div class="span-20 colborder">'
593  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
594 
595  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
596  #print cat_name,present_subdirs,total_weight,average_success_rate
597  html+='<div class="span-3 prepend-0 colborder">'
598  html+='<h3>%s</h3>'%cat_name
599  html+='<div><span class="alt">Avg. Success rate:</span></div>'
600  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
601  html+='</div>'
602  html+='<div class="span-9">'
603 
604  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
605  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
606  html+='<ul>'
607  for subdirname in sorted(present_subdirs.keys()):
608  this_dir_dict=present_subdirs[subdirname]
609  nsucc=this_dir_dict["nsucc"]
610  weight=this_dir_dict["weight"]
611  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
612  html+='</ul>'
613  html+='</div>'
614 
615  html+='<div class="span-6 last">'
616  html+=build_gauge(average_success_rate)
617  html+='</div>'
618 
619  html+='<hr>'
620  return html+'<br><a href="#top">Top...</a> </div><hr>'
621 
622  #-------------------------------------------------------------------------------
623 
624 def make_twiki_table(dir_dict,aggregation_rules):
625 
626  # decide the release
627  meta= list(dir_dict.items())[0][1].meta
628  releases=sorted([meta.release1,meta.release2])
629  latest_release=releases[1].split("-")[0]
630 
631 
632  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
633 
634  # Now Let's build the HTML
635 
636  html= '<div class="span-20 colborder">'
637  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
638  html+='<div>| Release | Comparison |'
639  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
640  html+=cat_name
641  html+=" | "
642  html+='</div>'
643 
644  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
645 
646  # Now add all the line with small gauges
647 
648  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
649  #print cat_name,present_subdirs,total_weight,average_success_rate
650  html+=build_gauge(average_success_rate,small=True,escaped=True)
651  html+=" | "
652 
653  html+='</div> <a href="#top">Top...</a>'
654  html+='<hr>'
655  return html+'</div>'
656 
657 #-------------------------------------------------------------------------------
658 
659 def get_pie_tooltip(directory):
660  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
661  return tooltip
662 
663 #-------------------------------------------------------------------------------
664 
665 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
666 
667  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
668 
669  script="""
670  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
671  <script type="text/javascript">
672  google.load("visualization", "1", {packages:["corechart"]});
673  google.setOnLoadCallback(drawChart);
674  function drawChart() {
675  var data = new google.visualization.DataTable();
676  data.addColumn('string', 'DQM Directory');
677  data.addColumn('number', 'Success Rate');
678  """
679  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
680  counter=0
681  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
682  #print subsystname,present_directories
683  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
684  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
685  counter+=1
686  script+="""
687  var chart = new google.visualization.BarChart(document.getElementById('%s'));
688  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
689  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
690  });
691  }
692  </script>
693  """%(name,40*counter,title)
694  return script
695 
696 
697 #-------------------------------------------------------------------------------
698 
699 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
700  """Create a table, with as rows the directories and as columns the samples.
701  Each box in the table will contain a pie chart linking to the directory.
702  """
703  #aggregation_rules={}
704  #aggregation_rules_twiki={}
705 
706  chdir(indir)
707  if os.path.isabs(indir):
708  title = basename(indir)
709  else:
710  title=indir
711  title=title.strip(".")
712  title=title.strip("/")
713 
714 
715  # Get the list of pickles
716  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
717 
718  # Load directories, build a list of all first level subdirs
719  dir_unpicklers=[]
720  n_unpicklers=0
721  for sample_pkl in sample_pkls:
722  dir_unpickler=unpickler(sample_pkl)
723  dir_unpickler.start()
724  n_unpicklers+=1
725  dir_unpicklers.append(dir_unpickler)
726  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
727  n_unpicklers=0
728  for dir_unpickler in dir_unpicklers:
729  dir_unpickler.join()
730 
731  dir_dict={}
732 
733  # create a fake global directory
734  global_dir=Directory("global","")
735  for dir_unpickler in dir_unpicklers:
736  dir_unpickler.join()
737  directory=dir_unpickler.directory
738  #directory.prune("Run summary")
739  #directory.calcStats()
740  global_dir.meta=directory.meta
741  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
742  global_dir.subdirs.append(directory)
743 
744  global_dir.calcStats()
745 
746  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
747  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
748 
749  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
750  rel1=""
751  rel2=""
752  try:
753  rel1,rel2=title.split("VS")
754  except:
755  rel1=global_dir.meta.release1.split("-")[0]
756  rel2=global_dir.meta.release2.split("-")[0]
757  global_dir.meta.release1=rel1
758  global_dir.meta.release2=rel2
759 
760  # union of all subdirs names
761  all_subdirs=[]
762  for directory in dir_dict.values():
763  for subdir_name in directory.get_subdirs_names():
764  all_subdirs.append(subdir_name)
765  all_subdirs=sorted(list(set(all_subdirs)))
766 
767  #print " $$ all_subdirs: %s" %(all_subdirs)
768 
769  # Get The title
770  page_html+= '<div class="span-20">'+\
771  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
772  '</div>'+\
773  '<div class="span-1">'+\
774  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
775  '</div>'+\
776  '<hr>'
777  page_html+='<div class="span-24"><p></p></div>\n'*3
778 
779  # Get The summary section
780  page_html+= get_summary_section(global_dir,False)
781 
782  # Make the anchor sections
783  page_html+= '<div class="span-24">'
784  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
785  '<ul>'+\
786  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
787  '<li><a href="#categories">Categories</a></li>'+\
788  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
789  '<li><a href="#summary_table">Summary Table</a></li>'+\
790  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
791  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
792  '</ul>'+\
793  '</div><hr>'
794 
795 
796  # Make the CategoriesBar chart
797  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
798  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
799 
800  # Make the gauges per categories
801  page_html+=make_categories_summary(dir_dict,aggregation_rules)
802 
803  # Make the Directories chart
804  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
805  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
806 
807  # Barbarian vertical space. Suggestions are welcome
808  for i in range(2):
809  page_html+='<div class="span-24"><p></p></div>\n'
810 
811 
812  # Prepare the table
813  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
814 
815  for i in range(5):
816  page_html+='<div class="span-24"><p></p></div>\n'
817 
818  page_html+="""
819  <table border="1" >
820  <tr>
821  <td> </td>
822  """
823 
824  # First row with samples
825  page_html+="""
826  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
827 
828  sorted_samples=sorted(dir_dict.keys())
829  for sample in sorted_samples:
830  sample_nick=sample
831  ## For runs: put only the number after the _
832  #if "_" in sample:
833  #run_number=sample.split("_")[-1]
834  #if (not run_number.isalpha()) and len(run_number)>=6:
835  #sample_nick=run_number
836 
837 
838  page_html+="""
839  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
840  page_html+=" </tr>\n"
841 
842 
843  # FIRST ROW
844  # Now the summaries at the beginning of the table
845  page_html+="<tr>"
846  page_html+='<td style="background-color:white;"><div class="span-1">'
847 
848  page_html+='<b>Summary</b></div></td>'
849  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
850  for sample in sorted_samples:
851  col=dir_dict[sample]
852  # check if the directory was a top one or not
853  summary_page_name="RelMonSummary.html"
854  if col.name!="":
855  summary_page_name=hash_name(col.name, hashing_flag)+".html"
856  img_link=col.get_summary_chart_ajax(55,55)
857  page_html+='<td style="background-color:white;"><div class="span-1">'
858  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
859  page_html+="</tr>"
860 
861  # Now the content
862  for subdir_name in all_subdirs:
863 
864  page_html+=' <tr>\n'
865  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
866 
867  row_summary=Directory("row_summary","")
868  sample_counter=0
869  n_samples=len(sorted_samples)
870 
871  for sample in sorted_samples:
872  subdirs_dict=directory.get_subdirs_dict()
873  directory=dir_dict[sample]
874  dir_is_there=subdir_name in subdirs_dict
875  if dir_is_there:
876  row_summary.subdirs.append(subdirs_dict[subdir_name])
877 
878  # one first row for the summary!
879  row_summary.calcStats()
880  img_link=row_summary.get_summary_chart_ajax(55,55)
881  page_html+='<td style="background-color:white;"><div class="span-1">'
882  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
883 
884  for sample in sorted_samples:
885  sample_counter+=1
886 
887  directory=dir_dict[sample]
888  subdirs_dict=directory.get_subdirs_dict()
889 
890  # Check if the directory is the top one
891  if directory.name!="":
892  # We did not run on the topdir
893  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
894  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
895  if hashing_flag:
896  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
897  else:
898  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
899  else:
900  #print " ## summary_page: %s"%(directory.name+subdir_name)
901  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
902  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
903  dir_is_there=subdir_name in subdirs_dict
904 
905  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
906  img_tooltip="N/A"
907  if dir_is_there:
908  #row_summary.subdirs.append(subdirs_dict[subdir_name])
909  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
910  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
911 
912  page_html+='<td style="background-color:white;"><div class="span-1">'
913  if dir_is_there:
914  page_html+='<a href="%s">'%(summary_page)
915  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
916  if dir_is_there:
917  page_html+='</a>'
918  page_html+='</div></td>'
919 
920  page_html+=" </tr>\n"
921 
922 
923 
924  page_html+='</table> <a href="#top">Top...</a><hr>'
925 
926  page_html+=get_rank_section(global_dir)
927 
928  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
929 
930  page_html+=get_page_footer()
931  return page_html
932 
933 
934 #-----------UPDATES------
935 def hash_name(file_name, flag):
936  #print " HashFILE name: "+file_name
937  if flag: #if hashing flag is ON then return
938  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
939  else:
940  return file_name #return standart name
sistrip::SpyUtilities::range
const uint16_t range(const Frame &aFrame)
Definition: SiStripSpyUtilities.cc:76
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:19
Generated for CMSSW Reference Manual by doxygen 1.8.5