CMS 3D CMS Logo
directories2html.py
Go to the documentation of this file.
1 from __future__ import print_function
2 from __future__ import absolute_import
3 ################################################################################
4 # RelMon: a tool for automatic Release Comparison
5 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
6 #
7 
8 #
9 #
10 # Danilo Piparo CERN - danilo.piparo@cern.ch
11 #
12 ################################################################################
13 
14 from builtins import range
15 from os import chdir,getcwd,listdir,makedirs
16 from os.path import basename,join,exists
17 import cgi
18 
19 import sys
20 theargv=sys.argv
21 sys.argv=[]
22 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
23 sys.argv=theargv
24 
25 import os
26 if "RELMON_SA" in os.environ:
27  from .dirstructure import Comparison,Directory
28  from .definitions import *
29  from .utils import unpickler
30 else:
31  from Utilities.RelMon.dirstructure import Comparison,Directory
32  from Utilities.RelMon.definitions import *
33  from Utilities.RelMon.utils import unpickler
34 
35  import hashlib
36 #-------------------------------------------------------------------------------
37 
38 def encode_obj_url(url):
39  for old,new in url_encode_dict.items():
40  url=url.replace(old,new)
41  return url
42 
43 def plot_size(h=250,w=200):
44  return "w=%s;h=%s" %(h,w)
45 
46 def build_obj_addr(run,sample,version,plot_path,tier):
47  slash="/"
48  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
49  obj_url+=plot_path
50  while obj_url.endswith(slash):
51  obj_url=obj_url[:-1]
52  while slash*2 in obj_url:
53  obj_url=obj_url.replace(slash*2,slash)
54  return obj_url
55 
56 def build_obj(run,sample,version,plot_path,tier):
57  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
58  return encode_obj_url(obj_url)
59 
60 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
61  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
62  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
63  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
64  if len(draw_opts)>0:
65  fairy_url+="drawopts=%s;" %draw_opts
66  fairy_url+= plot_size(h,w)
67  fairy_url += ";ref=ratiooverlay"
68  return fairy_url
69 
70 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
71  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
72  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
73  fairy_url+= "?%s"%plot_size(h,w)
74  if len(draw_opts)>0:
75  fairy_url+="drawopts=%s;" %draw_opts
76  return fairy_url
77 
78 #-------------------------------------------------------------------------------
79 style_location="/cms-service-reldqm"
80 def get_page_header(directory=None, standalone=False, additional_header=""):
81  style_location="/cms-service-reldqm"
82  if standalone:
83  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
84  javascripts=''
85  style=''
86  if directory!=None and len(directory.comparisons)>0:
87  meta=directory.meta
88  style='img.fail {border:1px solid #ff0000;}\n'+\
89  'img.succes {border:1px solid #00ff00;}\n'+\
90  'img.null {border:1px solid #ffff00;}\n'+\
91  'img.skiped {border:1px solid #7a7a7a;}\n'+\
92  'a.black_link:link {color: #333333}\n'+\
93  'a.black_link:hover {color: #737373}\n'+\
94  'a.black_link:visited {color: #333333}\n'+\
95  'a.black_link:active {color: #333333}\n'
96  javascripts=""
97 
98 
99  html='<html>'+\
100  '<head>'+\
101  '<title>RelMon Summary</title>'+\
102  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
103  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
104  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
105  '<style type="text/css">'+\
106  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
107  '%s'%style+\
108  '</style>'+\
109  '%s'%javascripts+\
110  '%s'%additional_header+\
111  '</head>'+\
112  '<body>'+\
113  '<div class="container">'
114 
115  return html
116 
117 #-------------------------------------------------------------------------------
118 
119 def get_page_footer():
120  return '</div></body></html>'
121 
122 #-------------------------------------------------------------------------------
123 
124 def get_title_section(directory, hashing_flag, standalone, depth=2):
125  if standalone:
126  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
127  else:
128  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
129  mother_name=basename(directory.mother_dir)
130  mother_file_name=""
131  if depth==1:
132  mother_file_name="../RelMonSummary.html"
133  if mother_name!="":
134  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
135  elif depth==2:
136  mother_file_name="RelMonSummary.html"
137  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
138  if mother_name!="":
139  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
140  else:
141  if hashing_flag:
142  files = directory.mother_dir.split("/")
143  if len(files) != 1:
144  dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
145  else:
146  dir_name = files[-1]
147  dir_name = directory.mother_dir
148  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
149  else:
150  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
151  mother_file_name=mother_file_name.strip("_")
152 
153  link_to_mother='<a href="%s">..</a>' %mother_file_name
154  html= '<div class="span-20">'+\
155  '<h1>%s</h1>'%directory.name+\
156  '</div>'+\
157  '<div class="span-1">'+\
158  '<h1>%s</h1>'%link_to_mother+\
159  '</div>'+\
160  '<div class="span-3 last">'+\
161  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
162  '</div>'+\
163  '<hr>'
164  if len(mother_name)>0:
165  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
166  '<hr>'
167  return html
168 
169 #-------------------------------------------------------------------------------
170 def get_dir_stats(directory):
171  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
172  html+='<ul>'
173  if directory.n_successes>0:
174  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
175  if directory.n_nulls>0:
176  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
177  if directory.n_fails>0:
178  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
179  if directory.n_skiped>0:
180  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
181  if directory.n_missing_objs>0:
182  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
183  html+='</ul>'
184  return html
185 
186 #-------------------------------------------------------------------------------
187 
188 def get_subdirs_section(directory, hashing_flag):
189  if len(directory.subdirs)==0:
190  return ""
191  html= '<div class="span-20 colborder">'
192  html+='<h2 class="alt">Sub-Directories</h2>'
193  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
194  # so reverse :)
195  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
196  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
197  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
198  for subdir in sorted_subdirs:
199  name=subdir.name
200  if hashing_flag:
201  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
202  else:
203  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
204  link=link.strip("_")
205  html+='<div class="span-4 prepend-2 colborder">'
206  html+='<h3>%s</h3>'%name
207  html+='</div>'
208 
209  html+='<div class="span-7">'
210  html+=get_dir_stats(subdir)
211  html+='</div>'
212 
213  html+='<div class="span-6 last">'
214  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
215  html+='</div>'
216 
217  html+='<hr>'
218  return html+'</div>'
219 
220 
221 #-------------------------------------------------------------------------------
222 
223 def get_summary_section(directory,matrix_page=True):
224 
225  # Hack find the first comparison and fill test and threshold
226  # shall this be put in meta?
227  test_name=""
228  test_threshold=0
229  for comparison in directory.comparisons:
230  test_name=comparison.test_name
231  test_threshold=comparison.test_thr
232  break
233  if len(test_name)==0:
234  for subdir in directory.subdirs:
235  for comparison in subdir.comparisons:
236  test_name=comparison.test_name
237  test_threshold=comparison.test_thr
238  break
239  if len(test_name)!=0:break
240  if len(test_name)==0:
241  for subsubdir in subdir.subdirs:
242  for comparison in subsubdir.comparisons:
243  test_name=comparison.test_name
244  test_threshold=comparison.test_thr
245  break
246  if len(test_name)!=0:break
247  if len(test_name)==0:
248  for subsubsubdir in subsubdir.subdirs:
249  for comparison in subsubsubdir.comparisons:
250  test_name=comparison.test_name
251  test_threshold=comparison.test_thr
252  break
253  if len(test_name)!=0:break
254 
255 
256  meta=directory.meta
257 
258  html= '<div class="span-6">'+\
259  '<h3>Summary</h3>'
260  html+=get_dir_stats(directory)
261  html+='<a href="%s/%s/start?runnr=%s;dataset=/%s/%s/DQMIO;sampletype=offline_data;filter=all;referencepos=on-side;referenceshow=all;referencenorm=True;referenceobj1=other:%s:/%s/%s/DQMIO:;referenceobj2=none;referenceobj3=none;referenceobj4=none;search=;striptype=object;stripruns=;stripaxis=run;stripomit=none;workspace=Everything;size=M;focus=;zoom=no;">To the DQM GUI...</a>' %(server,base_url,meta.run1,meta.sample1,meta.release1,meta.run2,meta.sample2,meta.release2)
262  html+='</div>'
263 
264  html+='<div class="span-7 colborder">'+\
265  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
266  '</div>'+\
267  '<div class="span-9 last">'
268  if matrix_page:
269  html+='<h3>Sample:</h3>'+\
270  '<p class="caps">%s</p>'%meta.sample1+\
271  '<h3>Run1 and Run2:</h3>'+\
272  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
273  html+='<h3>Releases:</h3>'+\
274  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
275  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
276  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
277  '</div>'+\
278  '<hr>'
279  return html
280 
281 #-------------------------------------------------------------------------------
282 
283 def get_comparisons(category,directory):
284  """Prepare the comparisons between histograms and organise them in the page.
285  Moreover create separate pages with the overlay and the single plots.
286  """
287  counter=1
288  tot_counter=1
289 
290  # get the right ones
291  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
292  n_comparisons=len(comparisons)
293 
294  is_reverse=True
295  if category == FAIL:
296  is_reverse=False
297  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
298 
299 
300  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
301  html_comparisons=""
302  for comparison in comparisons:
303  plot_name=comparison.img_name
304  if "http://" not in plot_name:
305  plot_name= basename(comparison.img_name)
306  class_type="colborder"
307  if counter==3 or tot_counter==n_comparisons:
308  class_type=" colborder last"
309  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
310 
311 
312  if directory.do_pngs:
313  png_link=comparison.img_name
314  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
315  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
316  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
317  else:
318  big_fairy=fairy_url(directory.meta.run1,
319  directory.meta.run2,
320  directory.meta.sample1,
321  directory.meta.sample2,
322  directory.meta.release1,
323  directory.meta.release2,
324  comp_abs_path,
325  directory.meta.tier1,
326  directory.meta.tier2,
327  "",600,600)
328  small_fairy=fairy_url(directory.meta.run1,
329  directory.meta.run2,
330  directory.meta.sample1,
331  directory.meta.sample2,
332  directory.meta.release1,
333  directory.meta.release2,
334  comp_abs_path,
335  directory.meta.tier1,
336  directory.meta.tier2)
337 
338  single_fairy1=fairy_url_single(directory.meta.run1,
339  directory.meta.sample1,
340  directory.meta.release1,
341  comp_abs_path,
342  directory.meta.tier1,
343  "",500,500)
344  single_fairy2=fairy_url_single(directory.meta.run2,
345  directory.meta.sample2,
346  directory.meta.release2,
347  comp_abs_path,
348  directory.meta.tier2,
349  "",500,500)
350 
351  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
352  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
353  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
354  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
355  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
356 
357  if counter==3:
358  html_comparisons+="<hr>"
359  counter=0
360  counter+=1
361  tot_counter+=1
362 
363  if len(html_comparisons)!=0:
364  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
365  html+=html_comparisons
366  html+='<hr>'
367  return html
368  return ""
369 
370 #-------------------------------------------------------------------------------
371 
372 def get_rank_section(directory):
373 # do Rank histo png
374  imgname="RankSummary.png"
375  gStyle.SetPadTickY(0)
376  c=TCanvas("ranks","ranks",500,400)
377  #gStyle.SetOptStat(0)
378  c.cd()
379 
380  h=directory.rank_histo
381  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
382  rank_histof.SetLineWidth(2)
383  for i in range(0,h.GetNbinsX()+1):
384  rank_histof.SetBinContent(i,h.GetBinContent(i))
385  h.SetTitle("Ranks Summary;Rank;Frequency")
386  h.Draw("Hist")
387  c.Update()
388  rank_histof.ComputeIntegral()
389  integral = rank_histof.GetIntegral()
390  rank_histof.SetContent(integral)
391 
392  rightmax = 1.1*rank_histof.GetMaximum()
393  scale = gPad.GetUymax()/rightmax
394  rank_histof.SetLineColor(kRed)
395  rank_histof.Scale(scale)
396  rank_histof.Draw("same")
397 
398  #draw an axis on the right side
399  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
400  axis.SetTitle("Cumulative")
401  axis.SetTitleColor(kRed)
402  axis.SetLineColor(kRed)
403  axis.SetLabelColor(kRed)
404  axis.Draw()
405 
406  rank_histof.Draw("Same");
407 
408 
409  c.Print(imgname)
410 
411  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
412  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
413  page_html+='</div> <a href="#top">Top...</a><hr>'
414 
415  return page_html
416 
417 #-------------------------------------------------------------------------------
418 
419 #-------------------------------------------------------------------------------
420 def get_missing_objs_section(directory):
421  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
422  """
423  page_html = "Unpaired in %s</br>"%(directory.filename1)
424  for elem in directory.different_histograms['file1']:
425  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
426  page_html +="</br>"
427  page_html += "Unpaired in %s</br>"%(directory.filename2)
428  for elem in directory.different_histograms['file2']:
429  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
430  return page_html
431 #-------------------------------------------------------------------------------
432 
433 def directory2html(directory, hashing, standalone, depth=0):
434  """Converts a directory tree into html pages, very nice ones.
435  """
436  #print "d2html: depth", str(depth)," dir ",directory.name
437  depth+=1
438  #old_cwd=getcwd()
439  #if not exists(directory.name) and len(directory.name)>0:
440  #makedirs(directory.name)
441 
442  #if len(directory.name)>0:
443  #chdir(directory.name)
444 
445  for subdir in directory.subdirs:
446  directory2html(subdir,hashing,standalone, depth)
447 
448  page_html=get_page_header(directory, standalone)+\
449  get_title_section(directory,hashing, standalone, depth)+\
450  get_summary_section(directory)+\
451  get_subdirs_section(directory, hashing)
452 
453  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
454  (directory.n_comp_nulls >0,NULL ),
455 
456  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
457  (directory.n_comp_skiped >0,SKIPED)):
458  if do_cat:
459  page_html+=get_comparisons(cat,directory)
460 
461  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
462  page_html += get_missing_objs_section(directory)
463 
464  # Distribution of ranks
465 
466  if depth==1:
467  page_html+=get_rank_section(directory)
468 
469 
470  page_html+=get_page_footer()
471 
472  page_name=directory.name
473 
474  if len(page_name)==0:
475  page_name="RelMonSummary"
476  if hashing:
477  if page_name != "RelMonSummary":
478  #print " ## oFile path: %s"%(join(directory.full_path))
479  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
480  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
481  else:
482  ofilename = "RelMonSummary.html"
483  else:
484  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
485  ofilename=ofilename.strip("_")
486  #print "Writing on %s" %ofilename
487  ofile=open(ofilename,"w")
488  ofile.write(page_html)
489  ofile.close()
490 
491  #chdir(old_cwd)
492 
493 #-------------------------------------------------------------------------------
494 
495 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
496  total_success_rate_scaled=(total_success_rate-minrate)
497  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
498  if total_success_rate_scaled_repr<0:
499  total_success_rate_scaled_repr=0
500  size_s="200x100"
501  if small:
502  size_s="40x30"
503  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
504  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
505  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
506  if not small:
507  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
508  gauge_link+="&chma=10,10,10,0"
509  img_tag= '<img src="%s">'%gauge_link
510  if escaped:
511  img_tag=cgi.escape(img_tag)
512  return img_tag
513 
514 #-------------------------------------------------------------------------------
515 
516 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
517  # Let's make a summary for an overview in categories act on the global dir
518  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
519 
520  list_of_names=[]
521  if the_aggr_pairs==[]:
522  for samplename,sampledir in dir_dict.items():
523  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
524  if not subsysdirname in list_of_names:
525  list_of_names.append(subsysdirname)
526  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
527 
528  #print the_aggr_pairs
529  for cat_name, subdir_list in the_aggr_pairs:
530  total_successes=0.
531  total_directory_successes=0
532  total_weight=0.
533  present_subdirs={}
534  total_ndirs=0
535  # Loop on samples
536  for dirname, sampledir in dir_dict.items():
537  # Loop on directories
538  for subdirname,subdir in sampledir.get_subdirs_dict().items():
539  if subdirname in subdir_list:
540  nsucc=subdir.n_successes
541  total_successes+=nsucc
542  weight=subdir.weight
543  total_weight+=weight
544  total_ndirs+=1
545 
546  total_directory_successes+= float(nsucc)/weight
547  if subdirname in present_subdirs:
548  this_dir_dict=present_subdirs[subdirname]
549  this_dir_dict["nsucc"]+=nsucc
550  this_dir_dict["weight"]+=weight
551  else:
552  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
553  # Make it usable also for subdirectories
554  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
555  for pathname in [name for name in subdir_list if "/" in name]:
556  selected_subdirname,selected_subsubdirname = pathname.split("/")
557  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
558  #print "Studying directory ",subsubdirname," in directory ",subdirname
559  nsucc=subsubdir.n_successes
560  total_successes+=nsucc
561  weight=subsubdir.weight
562  total_weight+=weight
563  total_ndirs+=1
564  total_directory_successes+= float(nsucc)/weight
565 
566  if subsubdirname in present_subdirs:
567  this_dir_dict=present_subdirs[subsubdirname]
568  this_dir_dict["nsucc"]+=nsucc
569  this_dir_dict["weight"]+=weight
570  else:
571  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
572 
573  if total_ndirs == 0:
574  print("No directory of the category %s is present in the samples: skipping." %cat_name)
575  continue
576 
577  average_success_rate=total_directory_successes/(total_ndirs)
578  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
579 
580  return aggr_pairs_info
581 
582 #-------------------------------------------------------------------------------
583 
584 def make_categories_summary(dir_dict,aggregation_rules):
585 
586  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
587 
588  #print aggr_pairs_info
589 
590  # Now Let's build the HTML
591 
592  html= '<div class="span-20 colborder">'
593  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
594 
595  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
596  #print cat_name,present_subdirs,total_weight,average_success_rate
597  html+='<div class="span-3 prepend-0 colborder">'
598  html+='<h3>%s</h3>'%cat_name
599  html+='<div><span class="alt">Avg. Success rate:</span></div>'
600  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
601  html+='</div>'
602  html+='<div class="span-9">'
603 
604  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
605  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
606  html+='<ul>'
607  for subdirname in sorted(present_subdirs.keys()):
608  this_dir_dict=present_subdirs[subdirname]
609  nsucc=this_dir_dict["nsucc"]
610  weight=this_dir_dict["weight"]
611  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
612  html+='</ul>'
613  html+='</div>'
614 
615  html+='<div class="span-6 last">'
616  html+=build_gauge(average_success_rate)
617  html+='</div>'
618 
619  html+='<hr>'
620  return html+'<br><a href="#top">Top...</a> </div><hr>'
621 
622  #-------------------------------------------------------------------------------
623 
624 def make_twiki_table(dir_dict,aggregation_rules):
625 
626  # decide the release
627  meta= dir_dict.items()[0][1].meta
628  releases=sorted([meta.release1,meta.release2])
629  latest_release=releases[1].split("-")[0]
630 
631 
632  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
633 
634  # Now Let's build the HTML
635 
636  html= '<div class="span-20 colborder">'
637  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
638  html+='<div>| Release | Comparison |'
639  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
640  html+=cat_name
641  html+=" | "
642  html+='</div>'
643 
644  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
645 
646  # Now add all the line with small gauges
647 
648  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
649  #print cat_name,present_subdirs,total_weight,average_success_rate
650  html+=build_gauge(average_success_rate,small=True,escaped=True)
651  html+=" | "
652 
653  html+='</div> <a href="#top">Top...</a>'
654  html+='<hr>'
655  return html+'</div>'
656 
657 #-------------------------------------------------------------------------------
658 
659 def get_pie_tooltip(directory):
660  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
661  return tooltip
662 
663 #-------------------------------------------------------------------------------
664 
665 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
666 
667  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
668 
669  script="""
670  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
671  <script type="text/javascript">
672  google.load("visualization", "1", {packages:["corechart"]});
673  google.setOnLoadCallback(drawChart);
674  function drawChart() {
675  var data = new google.visualization.DataTable();
676  data.addColumn('string', 'DQM Directory');
677  data.addColumn('number', 'Success Rate');
678  """
679  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
680  counter=0
681  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
682  #print subsystname,present_directories
683  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
684  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
685  counter+=1
686  script+="""
687  var chart = new google.visualization.BarChart(document.getElementById('%s'));
688  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
689  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
690  });
691  }
692  </script>
693  """%(name,40*counter,title)
694  return script
695 
696 
697 #-------------------------------------------------------------------------------
698 
699 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
700  """Create a table, with as rows the directories and as columns the samples.
701  Each box in the table will contain a pie chart linking to the directory.
702  """
703  #aggregation_rules={}
704  #aggregation_rules_twiki={}
705 
706  chdir(indir)
707  if os.path.isabs(indir):
708  title = basename(indir)
709  else:
710  title=indir
711  title=title.strip(".")
712  title=title.strip("/")
713 
714 
715  # Get the list of pickles
716  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
717 
718  # Load directories, build a list of all first level subdirs
719  dir_unpicklers=[]
720  n_unpicklers=0
721  for sample_pkl in sample_pkls:
722  dir_unpickler=unpickler(sample_pkl)
723  dir_unpickler.start()
724  n_unpicklers+=1
725  dir_unpicklers.append(dir_unpickler)
726  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
727  n_unpicklers=0
728  for dir_unpickler in dir_unpicklers:
729  dir_unpickler.join()
730 
731  dir_dict={}
732 
733  # create a fake global directory
734  global_dir=Directory("global","")
735  for dir_unpickler in dir_unpicklers:
736  dir_unpickler.join()
737  directory=dir_unpickler.directory
738  #directory.prune("Run summary")
739  #directory.calcStats()
740  global_dir.meta=directory.meta
741  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
742  global_dir.subdirs.append(directory)
743 
744  global_dir.calcStats()
745 
746  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
747  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
748 
749  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
750  rel1=""
751  rel2=""
752  try:
753  rel1,rel2=title.split("VS")
754  except:
755  rel1=global_dir.meta.release1.split("-")[0]
756  rel2=global_dir.meta.release2.split("-")[0]
757  global_dir.meta.release1=rel1
758  global_dir.meta.release2=rel2
759 
760  # union of all subdirs names
761  all_subdirs=[]
762  for directory in dir_dict.values():
763  for subdir_name in directory.get_subdirs_names():
764  all_subdirs.append(subdir_name)
765  all_subdirs=sorted(list(set(all_subdirs)))
766 
767  #print " $$ all_subdirs: %s" %(all_subdirs)
768 
769  # Get The title
770  page_html+= '<div class="span-20">'+\
771  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
772  '</div>'+\
773  '<div class="span-1">'+\
774  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
775  '</div>'+\
776  '<hr>'
777  page_html+='<div class="span-24"><p></p></div>\n'*3
778 
779  # Get The summary section
780  page_html+= get_summary_section(global_dir,False)
781 
782  # Make the anchor sections
783  page_html+= '<div class="span-24">'
784  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
785  '<ul>'+\
786  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
787  '<li><a href="#categories">Categories</a></li>'+\
788  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
789  '<li><a href="#summary_table">Summary Table</a></li>'+\
790  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
791  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
792  '</ul>'+\
793  '</div><hr>'
794 
795 
796  # Make the CategoriesBar chart
797  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
798  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
799 
800  # Make the gauges per categories
801  page_html+=make_categories_summary(dir_dict,aggregation_rules)
802 
803  # Make the Directories chart
804  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
805  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
806 
807  # Barbarian vertical space. Suggestions are welcome
808  for i in range(2):
809  page_html+='<div class="span-24"><p></p></div>\n'
810 
811 
812  # Prepare the table
813  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
814 
815  for i in range(5):
816  page_html+='<div class="span-24"><p></p></div>\n'
817 
818  page_html+="""
819  <table border="1" >
820  <tr>
821  <td> </td>
822  """
823 
824  # First row with samples
825  page_html+="""
826  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
827 
828  sorted_samples=sorted(dir_dict.keys())
829  for sample in sorted_samples:
830  sample_nick=sample
831  ## For runs: put only the number after the _
832  #if "_" in sample:
833  #run_number=sample.split("_")[-1]
834  #if (not run_number.isalpha()) and len(run_number)>=6:
835  #sample_nick=run_number
836 
837 
838  page_html+="""
839  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
840  page_html+=" </tr>\n"
841 
842 
843  # FIRST ROW
844  # Now the summaries at the beginning of the table
845  page_html+="<tr>"
846  page_html+='<td style="background-color:white;"><div class="span-1">'
847 
848  page_html+='<b>Summary</b></div></td>'
849  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
850  for sample in sorted_samples:
851  col=dir_dict[sample]
852  # check if the directory was a top one or not
853  summary_page_name="RelMonSummary.html"
854  if col.name!="":
855  summary_page_name=hash_name(col.name, hashing_flag)+".html"
856  img_link=col.get_summary_chart_ajax(55,55)
857  page_html+='<td style="background-color:white;"><div class="span-1">'
858  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
859  page_html+="</tr>"
860 
861  # Now the content
862  for subdir_name in all_subdirs:
863 
864  page_html+=' <tr>\n'
865  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
866 
867  row_summary=Directory("row_summary","")
868  sample_counter=0
869  n_samples=len(sorted_samples)
870 
871  for sample in sorted_samples:
872  subdirs_dict=directory.get_subdirs_dict()
873  directory=dir_dict[sample]
874  dir_is_there=subdir_name in subdirs_dict
875  if dir_is_there:
876  row_summary.subdirs.append(subdirs_dict[subdir_name])
877 
878  # one first row for the summary!
879  row_summary.calcStats()
880  img_link=row_summary.get_summary_chart_ajax(55,55)
881  page_html+='<td style="background-color:white;"><div class="span-1">'
882  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
883 
884  for sample in sorted_samples:
885  sample_counter+=1
886 
887  directory=dir_dict[sample]
888  subdirs_dict=directory.get_subdirs_dict()
889 
890  # Check if the directory is the top one
891  if directory.name!="":
892  # We did not run on the topdir
893  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
894  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
895  if hashing_flag:
896  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
897  else:
898  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
899  else:
900  #print " ## summary_page: %s"%(directory.name+subdir_name)
901  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
902  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
903  dir_is_there=subdir_name in subdirs_dict
904 
905  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
906  img_tooltip="N/A"
907  if dir_is_there:
908  #row_summary.subdirs.append(subdirs_dict[subdir_name])
909  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
910  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
911 
912  page_html+='<td style="background-color:white;"><div class="span-1">'
913  if dir_is_there:
914  page_html+='<a href="%s">'%(summary_page)
915  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
916  if dir_is_there:
917  page_html+='</a>'
918  page_html+='</div></td>'
919 
920  page_html+=" </tr>\n"
921 
922 
923 
924  page_html+='</table> <a href="#top">Top...</a><hr>'
925 
926  page_html+=get_rank_section(global_dir)
927 
928  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
929 
930  page_html+=get_page_footer()
931  return page_html
932 
933 
934 #-----------UPDATES------
935 def hash_name(file_name, flag):
936  #print " HashFILE name: "+file_name
937  if flag: #if hashing flag is ON then return
938  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
939  else:
940  return file_name #return standart name
cms::dd::split
std::vector< std::string_view > split(std::string_view, const char *)
directories2html.build_gauge
def build_gauge(total_success_rate, minrate=.80, small=False, escaped=False)
Definition: directories2html.py:495
directories2html.hash_name
def hash_name(file_name, flag)
Definition: directories2html.py:935
directories2html.get_comparisons
def get_comparisons(category, directory)
Definition: directories2html.py:283
directories2html.get_subdirs_section
def get_subdirs_section(directory, hashing_flag)
Definition: directories2html.py:188
directories2html.get_pie_tooltip
def get_pie_tooltip(directory)
Definition: directories2html.py:659
directories2html.build_obj
def build_obj(run, sample, version, plot_path, tier)
Definition: directories2html.py:56
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
directories2html.get_aggr_pairs_info
def get_aggr_pairs_info(dir_dict, the_aggr_pairs=[])
Definition: directories2html.py:516
directories2html.make_summary_table
def make_summary_table(indir, aggregation_rules, aggregation_rules_twiki, hashing_flag, standalone_flag)
Definition: directories2html.py:699
directories2html.plot_size
def plot_size(h=250, w=200)
Definition: directories2html.py:43
directories2html.get_page_header
def get_page_header(directory=None, standalone=False, additional_header="")
Definition: directories2html.py:80
directories2html.get_title_section
def get_title_section(directory, hashing_flag, standalone, depth=2)
Definition: directories2html.py:124
directories2html.get_dir_stats
def get_dir_stats(directory)
Definition: directories2html.py:170
directories2html.make_barchart_summary
def make_barchart_summary(dir_dict, name="the_chart", title="DQM directory", the_aggr_pairs=[])
Definition: directories2html.py:665
directories2html.get_summary_section
def get_summary_section(directory, matrix_page=True)
Definition: directories2html.py:223
directories2html.directory2html
def directory2html(directory, hashing, standalone, depth=0)
Definition: directories2html.py:433
directories2html.make_twiki_table
def make_twiki_table(dir_dict, aggregation_rules)
Definition: directories2html.py:624
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:29
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:17
directories2html.make_categories_summary
def make_categories_summary(dir_dict, aggregation_rules)
Definition: directories2html.py:584
FastTimerService_cff.range
range
Definition: FastTimerService_cff.py:34
directories2html.fairy_url_single
def fairy_url_single(run, sample, version, plot_path, tier, draw_opts="", h=250, w=200)
Definition: directories2html.py:70
directories2html.fairy_url
def fairy_url(run1, run2, sample1, sample2, version1, version2, plot_path, tier1, tier2, draw_opts="", h=250, w=200)
Definition: directories2html.py:60
directories2html.build_obj_addr
def build_obj_addr(run, sample, version, plot_path, tier)
Definition: directories2html.py:46
directories2html.get_missing_objs_section
def get_missing_objs_section(directory)
Definition: directories2html.py:420
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
directories2html.get_rank_section
def get_rank_section(directory)
Definition: directories2html.py:372
Generated for CMSSW Reference Manual by doxygen 1.8.11