CMS 3D CMS Logo
directories2html.py
Go to the documentation of this file.
1 from __future__ import print_function
2 ################################################################################
3 # RelMon: a tool for automatic Release Comparison
4 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
5 #
6 
7 #
8 #
9 # Danilo Piparo CERN - danilo.piparo@cern.ch
10 #
11 ################################################################################
12 
13 from os import chdir,getcwd,listdir,makedirs
14 from os.path import basename,join,exists
15 import cgi
16 
17 import sys
18 theargv=sys.argv
19 sys.argv=[]
20 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
21 sys.argv=theargv
22 
23 import os
24 if "RELMON_SA" in os.environ:
25  from dirstructure import Comparison,Directory
26  from definitions import *
27  from utils import unpickler
28 else:
29  from Utilities.RelMon.dirstructure import Comparison,Directory
30  from Utilities.RelMon.definitions import *
31  from Utilities.RelMon.utils import unpickler
32 
33  import hashlib
34 #-------------------------------------------------------------------------------
35 
36 def encode_obj_url(url):
37  for old,new in url_encode_dict.items():
38  url=url.replace(old,new)
39  return url
40 
41 def plot_size(h=250,w=200):
42  return "w=%s;h=%s" %(h,w)
43 
44 def build_obj_addr(run,sample,version,plot_path,tier):
45  slash="/"
46  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
47  obj_url+=plot_path
48  while obj_url.endswith(slash):
49  obj_url=obj_url[:-1]
50  while slash*2 in obj_url:
51  obj_url=obj_url.replace(slash*2,slash)
52  return obj_url
53 
54 def build_obj(run,sample,version,plot_path,tier):
55  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
56  return encode_obj_url(obj_url)
57 
58 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
59  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
60  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
61  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
62  if len(draw_opts)>0:
63  fairy_url+="drawopts=%s;" %draw_opts
64  fairy_url+= plot_size(h,w)
65  fairy_url += ";ref=ratiooverlay"
66  return fairy_url
67 
68 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
69  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
70  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
71  fairy_url+= "?%s"%plot_size(h,w)
72  if len(draw_opts)>0:
73  fairy_url+="drawopts=%s;" %draw_opts
74  return fairy_url
75 
76 #-------------------------------------------------------------------------------
77 style_location="/cms-service-reldqm"
78 def get_page_header(directory=None, standalone=False, additional_header=""):
79  style_location="/cms-service-reldqm"
80  if standalone:
81  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
82  javascripts=''
83  style=''
84  if directory!=None and len(directory.comparisons)>0:
85  meta=directory.meta
86  style='img.fail {border:1px solid #ff0000;}\n'+\
87  'img.succes {border:1px solid #00ff00;}\n'+\
88  'img.null {border:1px solid #ffff00;}\n'+\
89  'img.skiped {border:1px solid #7a7a7a;}\n'+\
90  'a.black_link:link {color: #333333}\n'+\
91  'a.black_link:hover {color: #737373}\n'+\
92  'a.black_link:visited {color: #333333}\n'+\
93  'a.black_link:active {color: #333333}\n'
94  javascripts=""
95 
96 
97  html='<html>'+\
98  '<head>'+\
99  '<title>RelMon Summary</title>'+\
100  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
101  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
102  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
103  '<style type="text/css">'+\
104  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
105  '%s'%style+\
106  '</style>'+\
107  '%s'%javascripts+\
108  '%s'%additional_header+\
109  '</head>'+\
110  '<body>'+\
111  '<div class="container">'
112 
113  return html
114 
115 #-------------------------------------------------------------------------------
116 
117 def get_page_footer():
118  return '</div></body></html>'
119 
120 #-------------------------------------------------------------------------------
121 
122 def get_title_section(directory, hashing_flag, standalone, depth=2):
123  if standalone:
124  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
125  else:
126  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
127  mother_name=basename(directory.mother_dir)
128  mother_file_name=""
129  if depth==1:
130  mother_file_name="../RelMonSummary.html"
131  if mother_name!="":
132  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
133  elif depth==2:
134  mother_file_name="RelMonSummary.html"
135  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
136  if mother_name!="":
137  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
138  else:
139  if hashing_flag:
140  files = directory.mother_dir.split("/")
141  if len(files) != 1:
142  dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
143  else:
144  dir_name = files[-1]
145  dir_name = directory.mother_dir
146  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
147  else:
148  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
149  mother_file_name=mother_file_name.strip("_")
150 
151  link_to_mother='<a href="%s">..</a>' %mother_file_name
152  html= '<div class="span-20">'+\
153  '<h1>%s</h1>'%directory.name+\
154  '</div>'+\
155  '<div class="span-1">'+\
156  '<h1>%s</h1>'%link_to_mother+\
157  '</div>'+\
158  '<div class="span-3 last">'+\
159  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
160  '</div>'+\
161  '<hr>'
162  if len(mother_name)>0:
163  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
164  '<hr>'
165  return html
166 
167 #-------------------------------------------------------------------------------
168 def get_dir_stats(directory):
169  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
170  html+='<ul>'
171  if directory.n_successes>0:
172  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
173  if directory.n_nulls>0:
174  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
175  if directory.n_fails>0:
176  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
177  if directory.n_skiped>0:
178  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
179  if directory.n_missing_objs>0:
180  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
181  html+='</ul>'
182  return html
183 
184 #-------------------------------------------------------------------------------
185 
186 def get_subdirs_section(directory, hashing_flag):
187  if len(directory.subdirs)==0:
188  return ""
189  html= '<div class="span-20 colborder">'
190  html+='<h2 class="alt">Sub-Directories</h2>'
191  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
192  # so reverse :)
193  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
194  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
195  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
196  for subdir in sorted_subdirs:
197  name=subdir.name
198  if hashing_flag:
199  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
200  else:
201  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
202  link=link.strip("_")
203  html+='<div class="span-4 prepend-2 colborder">'
204  html+='<h3>%s</h3>'%name
205  html+='</div>'
206 
207  html+='<div class="span-7">'
208  html+=get_dir_stats(subdir)
209  html+='</div>'
210 
211  html+='<div class="span-6 last">'
212  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
213  html+='</div>'
214 
215  html+='<hr>'
216  return html+'</div>'
217 
218 
219 #-------------------------------------------------------------------------------
220 
221 def get_summary_section(directory,matrix_page=True):
222 
223  # Hack find the first comparison and fill test and threshold
224  # shall this be put in meta?
225  test_name=""
226  test_threshold=0
227  for comparison in directory.comparisons:
228  test_name=comparison.test_name
229  test_threshold=comparison.test_thr
230  break
231  if len(test_name)==0:
232  for subdir in directory.subdirs:
233  for comparison in subdir.comparisons:
234  test_name=comparison.test_name
235  test_threshold=comparison.test_thr
236  break
237  if len(test_name)!=0:break
238  if len(test_name)==0:
239  for subsubdir in subdir.subdirs:
240  for comparison in subsubdir.comparisons:
241  test_name=comparison.test_name
242  test_threshold=comparison.test_thr
243  break
244  if len(test_name)!=0:break
245  if len(test_name)==0:
246  for subsubsubdir in subsubdir.subdirs:
247  for comparison in subsubsubdir.comparisons:
248  test_name=comparison.test_name
249  test_threshold=comparison.test_thr
250  break
251  if len(test_name)!=0:break
252 
253 
254  meta=directory.meta
255 
256  html= '<div class="span-6">'+\
257  '<h3>Summary</h3>'
258  html+=get_dir_stats(directory)
259  html+='<a href="%s/%s">To the DQM GUI...</a>' %(server,base_url)
260  html+='</div>'
261 
262  html+='<div class="span-7 colborder">'+\
263  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
264  '</div>'+\
265  '<div class="span-9 last">'
266  if matrix_page:
267  html+='<h3>Sample:</h3>'+\
268  '<p class="caps">%s</p>'%meta.sample1+\
269  '<h3>Run1 and Run2:</h3>'+\
270  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
271  html+='<h3>Releases:</h3>'+\
272  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
273  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
274  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
275  '</div>'+\
276  '<hr>'
277  return html
278 
279 #-------------------------------------------------------------------------------
280 
281 def get_comparisons(category,directory):
282  """Prepare the comparisons between histograms and organise them in the page.
283  Moreover create separate pages with the overlay and the single plots.
284  """
285  counter=1
286  tot_counter=1
287 
288  # get the right ones
289  comparisons= [comp for comp in directory.comparisons if comp.status == cat_states[category]]
290  n_comparisons=len(comparisons)
291 
292  is_reverse=True
293  if category == FAIL:
294  is_reverse=False
295  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
296 
297 
298  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
299  html_comparisons=""
300  for comparison in comparisons:
301  plot_name=comparison.img_name
302  if "http://" not in plot_name:
303  plot_name= basename(comparison.img_name)
304  class_type="colborder"
305  if counter==3 or tot_counter==n_comparisons:
306  class_type=" colborder last"
307  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
308 
309 
310  if directory.do_pngs:
311  png_link=comparison.img_name
312  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
313  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
314  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
315  else:
316  big_fairy=fairy_url(directory.meta.run1,
317  directory.meta.run2,
318  directory.meta.sample1,
319  directory.meta.sample2,
320  directory.meta.release1,
321  directory.meta.release2,
322  comp_abs_path,
323  directory.meta.tier1,
324  directory.meta.tier2,
325  "",600,600)
326  small_fairy=fairy_url(directory.meta.run1,
327  directory.meta.run2,
328  directory.meta.sample1,
329  directory.meta.sample2,
330  directory.meta.release1,
331  directory.meta.release2,
332  comp_abs_path,
333  directory.meta.tier1,
334  directory.meta.tier2)
335 
336  single_fairy1=fairy_url_single(directory.meta.run1,
337  directory.meta.sample1,
338  directory.meta.release1,
339  comp_abs_path,
340  directory.meta.tier1,
341  "",500,500)
342  single_fairy2=fairy_url_single(directory.meta.run2,
343  directory.meta.sample2,
344  directory.meta.release2,
345  comp_abs_path,
346  directory.meta.tier2,
347  "",500,500)
348 
349  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
350  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
351  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
352  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
353  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
354 
355  if counter==3:
356  html_comparisons+="<hr>"
357  counter=0
358  counter+=1
359  tot_counter+=1
360 
361  if len(html_comparisons)!=0:
362  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
363  html+=html_comparisons
364  html+='<hr>'
365  return html
366  return ""
367 
368 #-------------------------------------------------------------------------------
369 
370 def get_rank_section(directory):
371 # do Rank histo png
372  imgname="RankSummary.png"
373  gStyle.SetPadTickY(0)
374  c=TCanvas("ranks","ranks",500,400)
375  #gStyle.SetOptStat(0)
376  c.cd()
377 
378  h=directory.rank_histo
379  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
380  rank_histof.SetLineWidth(2)
381  for i in xrange(0,h.GetNbinsX()+1):
382  rank_histof.SetBinContent(i,h.GetBinContent(i))
383  h.SetTitle("Ranks Summary;Rank;Frequency")
384  h.Draw("Hist")
385  c.Update()
386  rank_histof.ComputeIntegral()
387  integral = rank_histof.GetIntegral()
388  rank_histof.SetContent(integral)
389 
390  rightmax = 1.1*rank_histof.GetMaximum()
391  scale = gPad.GetUymax()/rightmax
392  rank_histof.SetLineColor(kRed)
393  rank_histof.Scale(scale)
394  rank_histof.Draw("same")
395 
396  #draw an axis on the right side
397  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
398  axis.SetTitle("Cumulative")
399  axis.SetTitleColor(kRed)
400  axis.SetLineColor(kRed)
401  axis.SetLabelColor(kRed)
402  axis.Draw()
403 
404  rank_histof.Draw("Same");
405 
406 
407  c.Print(imgname)
408 
409  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
410  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
411  page_html+='</div> <a href="#top">Top...</a><hr>'
412 
413  return page_html
414 
415 #-------------------------------------------------------------------------------
416 
417 #-------------------------------------------------------------------------------
418 def get_missing_objs_section(directory):
419  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
420  """
421  page_html = "Unpaired in %s</br>"%(directory.filename1)
422  for elem in directory.different_histograms['file1']:
423  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
424  page_html +="</br>"
425  page_html += "Unpaired in %s</br>"%(directory.filename2)
426  for elem in directory.different_histograms['file2']:
427  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
428  return page_html
429 #-------------------------------------------------------------------------------
430 
431 def directory2html(directory, hashing, standalone, depth=0):
432  """Converts a directory tree into html pages, very nice ones.
433  """
434  #print "d2html: depth", str(depth)," dir ",directory.name
435  depth+=1
436  #old_cwd=getcwd()
437  #if not exists(directory.name) and len(directory.name)>0:
438  #makedirs(directory.name)
439 
440  #if len(directory.name)>0:
441  #chdir(directory.name)
442 
443  for subdir in directory.subdirs:
444  directory2html(subdir,hashing,standalone, depth)
445 
446  page_html=get_page_header(directory, standalone)+\
447  get_title_section(directory,hashing, standalone, depth)+\
448  get_summary_section(directory)+\
449  get_subdirs_section(directory, hashing)
450 
451  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
452  (directory.n_comp_nulls >0,NULL ),
453 
454  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
455  (directory.n_comp_skiped >0,SKIPED)):
456  if do_cat:
457  page_html+=get_comparisons(cat,directory)
458 
459  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
460  page_html += get_missing_objs_section(directory)
461 
462  # Distribution of ranks
463 
464  if depth==1:
465  page_html+=get_rank_section(directory)
466 
467 
468  page_html+=get_page_footer()
469 
470  page_name=directory.name
471 
472  if len(page_name)==0:
473  page_name="RelMonSummary"
474  if hashing:
475  if page_name != "RelMonSummary":
476  #print " ## oFile path: %s"%(join(directory.full_path))
477  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
478  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
479  else:
480  ofilename = "RelMonSummary.html"
481  else:
482  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
483  ofilename=ofilename.strip("_")
484  #print "Writing on %s" %ofilename
485  ofile=open(ofilename,"w")
486  ofile.write(page_html)
487  ofile.close()
488 
489  #chdir(old_cwd)
490 
491 #-------------------------------------------------------------------------------
492 
493 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
494  total_success_rate_scaled=(total_success_rate-minrate)
495  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
496  if total_success_rate_scaled_repr<0:
497  total_success_rate_scaled_repr=0
498  size_s="200x100"
499  if small:
500  size_s="40x30"
501  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
502  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
503  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
504  if not small:
505  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
506  gauge_link+="&chma=10,10,10,0"
507  img_tag= '<img src="%s">'%gauge_link
508  if escaped:
509  img_tag=cgi.escape(img_tag)
510  return img_tag
511 
512 #-------------------------------------------------------------------------------
513 
514 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
515  # Let's make a summary for an overview in categories act on the global dir
516  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
517 
518  list_of_names=[]
519  if the_aggr_pairs==[]:
520  for samplename,sampledir in dir_dict.items():
521  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
522  if not subsysdirname in list_of_names:
523  list_of_names.append(subsysdirname)
524  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
525 
526  #print the_aggr_pairs
527  for cat_name, subdir_list in the_aggr_pairs:
528  total_successes=0.
529  total_directory_successes=0
530  total_weight=0.
531  present_subdirs={}
532  total_ndirs=0
533  # Loop on samples
534  for dirname, sampledir in dir_dict.items():
535  # Loop on directories
536  for subdirname,subdir in sampledir.get_subdirs_dict().items():
537  if subdirname in subdir_list:
538  nsucc=subdir.n_successes
539  total_successes+=nsucc
540  weight=subdir.weight
541  total_weight+=weight
542  total_ndirs+=1
543 
544  total_directory_successes+= float(nsucc)/weight
545  if subdirname in present_subdirs:
546  this_dir_dict=present_subdirs[subdirname]
547  this_dir_dict["nsucc"]+=nsucc
548  this_dir_dict["weight"]+=weight
549  else:
550  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
551  # Make it usable also for subdirectories
552  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
553  for pathname in [name for name in subdir_list if "/" in name]:
554  selected_subdirname,selected_subsubdirname = pathname.split("/")
555  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
556  #print "Studying directory ",subsubdirname," in directory ",subdirname
557  nsucc=subsubdir.n_successes
558  total_successes+=nsucc
559  weight=subsubdir.weight
560  total_weight+=weight
561  total_ndirs+=1
562  total_directory_successes+= float(nsucc)/weight
563 
564  if subsubdirname in present_subdirs:
565  this_dir_dict=present_subdirs[subsubdirname]
566  this_dir_dict["nsucc"]+=nsucc
567  this_dir_dict["weight"]+=weight
568  else:
569  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
570 
571  if total_ndirs == 0:
572  print("No directory of the category %s is present in the samples: skipping." %cat_name)
573  continue
574 
575  average_success_rate=total_directory_successes/(total_ndirs)
576  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
577 
578  return aggr_pairs_info
579 
580 #-------------------------------------------------------------------------------
581 
582 def make_categories_summary(dir_dict,aggregation_rules):
583 
584  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
585 
586  #print aggr_pairs_info
587 
588  # Now Let's build the HTML
589 
590  html= '<div class="span-20 colborder">'
591  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
592 
593  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
594  #print cat_name,present_subdirs,total_weight,average_success_rate
595  html+='<div class="span-3 prepend-0 colborder">'
596  html+='<h3>%s</h3>'%cat_name
597  html+='<div><span class="alt">Avg. Success rate:</span></div>'
598  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
599  html+='</div>'
600  html+='<div class="span-9">'
601 
602  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
603  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
604  html+='<ul>'
605  for subdirname in sorted(present_subdirs.keys()):
606  this_dir_dict=present_subdirs[subdirname]
607  nsucc=this_dir_dict["nsucc"]
608  weight=this_dir_dict["weight"]
609  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
610  html+='</ul>'
611  html+='</div>'
612 
613  html+='<div class="span-6 last">'
614  html+=build_gauge(average_success_rate)
615  html+='</div>'
616 
617  html+='<hr>'
618  return html+'<br><a href="#top">Top...</a> </div><hr>'
619 
620  #-------------------------------------------------------------------------------
621 
622 def make_twiki_table(dir_dict,aggregation_rules):
623 
624  # decide the release
625  meta= dir_dict.items()[0][1].meta
626  releases=sorted([meta.release1,meta.release2])
627  latest_release=releases[1].split("-")[0]
628 
629 
630  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
631 
632  # Now Let's build the HTML
633 
634  html= '<div class="span-20 colborder">'
635  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
636  html+='<div>| Release | Comparison |'
637  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
638  html+=cat_name
639  html+=" | "
640  html+='</div>'
641 
642  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
643 
644  # Now add all the line with small gauges
645 
646  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
647  #print cat_name,present_subdirs,total_weight,average_success_rate
648  html+=build_gauge(average_success_rate,small=True,escaped=True)
649  html+=" | "
650 
651  html+='</div> <a href="#top">Top...</a>'
652  html+='<hr>'
653  return html+'</div>'
654 
655 #-------------------------------------------------------------------------------
656 
657 def get_pie_tooltip(directory):
658  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
659  return tooltip
660 
661 #-------------------------------------------------------------------------------
662 
663 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
664 
665  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
666 
667  script="""
668  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
669  <script type="text/javascript">
670  google.load("visualization", "1", {packages:["corechart"]});
671  google.setOnLoadCallback(drawChart);
672  function drawChart() {
673  var data = new google.visualization.DataTable();
674  data.addColumn('string', 'DQM Directory');
675  data.addColumn('number', 'Success Rate');
676  """
677  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
678  counter=0
679  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
680  #print subsystname,present_directories
681  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
682  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
683  counter+=1
684  script+="""
685  var chart = new google.visualization.BarChart(document.getElementById('%s'));
686  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
687  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
688  });
689  }
690  </script>
691  """%(name,40*counter,title)
692  return script
693 
694 
695 #-------------------------------------------------------------------------------
696 
697 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
698  """Create a table, with as rows the directories and as columns the samples.
699  Each box in the table will contain a pie chart linking to the directory.
700  """
701  #aggregation_rules={}
702  #aggregation_rules_twiki={}
703 
704  chdir(indir)
705  if os.path.isabs(indir):
706  title = basename(indir)
707  else:
708  title=indir
709  title=title.strip(".")
710  title=title.strip("/")
711 
712 
713  # Get the list of pickles
714  sample_pkls=[name for name in listdir("./") if name.endswith(".pkl")]
715 
716  # Load directories, build a list of all first level subdirs
717  dir_unpicklers=[]
718  n_unpicklers=0
719  for sample_pkl in sample_pkls:
720  dir_unpickler=unpickler(sample_pkl)
721  dir_unpickler.start()
722  n_unpicklers+=1
723  dir_unpicklers.append(dir_unpickler)
724  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
725  n_unpicklers=0
726  for dir_unpickler in dir_unpicklers:
727  dir_unpickler.join()
728 
729  dir_dict={}
730 
731  # create a fake global directory
732  global_dir=Directory("global","")
733  for dir_unpickler in dir_unpicklers:
734  dir_unpickler.join()
735  directory=dir_unpickler.directory
736  #directory.prune("Run summary")
737  #directory.calcStats()
738  global_dir.meta=directory.meta
739  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
740  global_dir.subdirs.append(directory)
741 
742  global_dir.calcStats()
743 
744  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
745  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
746 
747  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
748  rel1=""
749  rel2=""
750  try:
751  rel1,rel2=title.split("VS")
752  except:
753  rel1=global_dir.meta.release1.split("-")[0]
754  rel2=global_dir.meta.release2.split("-")[0]
755  global_dir.meta.release1=rel1
756  global_dir.meta.release2=rel2
757 
758  # union of all subdirs names
759  all_subdirs=[]
760  for directory in dir_dict.values():
761  for subdir_name in directory.get_subdirs_names():
762  all_subdirs.append(subdir_name)
763  all_subdirs=sorted(list(set(all_subdirs)))
764 
765  #print " $$ all_subdirs: %s" %(all_subdirs)
766 
767  # Get The title
768  page_html+= '<div class="span-20">'+\
769  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
770  '</div>'+\
771  '<div class="span-1">'+\
772  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
773  '</div>'+\
774  '<hr>'
775  page_html+='<div class="span-24"><p></p></div>\n'*3
776 
777  # Get The summary section
778  page_html+= get_summary_section(global_dir,False)
779 
780  # Make the anchor sections
781  page_html+= '<div class="span-24">'
782  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
783  '<ul>'+\
784  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
785  '<li><a href="#categories">Categories</a></li>'+\
786  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
787  '<li><a href="#summary_table">Summary Table</a></li>'+\
788  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
789  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
790  '</ul>'+\
791  '</div><hr>'
792 
793 
794  # Make the CategoriesBar chart
795  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
796  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
797 
798  # Make the gauges per categories
799  page_html+=make_categories_summary(dir_dict,aggregation_rules)
800 
801  # Make the Directories chart
802  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
803  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
804 
805  # Barbarian vertical space. Suggestions are welcome
806  for i in xrange(2):
807  page_html+='<div class="span-24"><p></p></div>\n'
808 
809 
810  # Prepare the table
811  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
812 
813  for i in xrange(5):
814  page_html+='<div class="span-24"><p></p></div>\n'
815 
816  page_html+="""
817  <table border="1" >
818  <tr>
819  <td> </td>
820  """
821 
822  # First row with samples
823  page_html+="""
824  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
825 
826  sorted_samples=sorted(dir_dict.keys())
827  for sample in sorted_samples:
828  sample_nick=sample
829  ## For runs: put only the number after the _
830  #if "_" in sample:
831  #run_number=sample.split("_")[-1]
832  #if (not run_number.isalpha()) and len(run_number)>=6:
833  #sample_nick=run_number
834 
835 
836  page_html+="""
837  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
838  page_html+=" </tr>\n"
839 
840 
841  # FIRST ROW
842  # Now the summaries at the beginning of the table
843  page_html+="<tr>"
844  page_html+='<td style="background-color:white;"><div class="span-1">'
845 
846  page_html+='<b>Summary</b></div></td>'
847  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
848  for sample in sorted_samples:
849  col=dir_dict[sample]
850  # check if the directory was a top one or not
851  summary_page_name="RelMonSummary.html"
852  if col.name!="":
853  summary_page_name=hash_name(col.name, hashing_flag)+".html"
854  img_link=col.get_summary_chart_ajax(55,55)
855  page_html+='<td style="background-color:white;"><div class="span-1">'
856  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
857  page_html+="</tr>"
858 
859  # Now the content
860  for subdir_name in all_subdirs:
861 
862  page_html+=' <tr>\n'
863  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
864 
865  row_summary=Directory("row_summary","")
866  sample_counter=0
867  n_samples=len(sorted_samples)
868 
869  for sample in sorted_samples:
870  subdirs_dict=directory.get_subdirs_dict()
871  directory=dir_dict[sample]
872  dir_is_there=subdir_name in subdirs_dict
873  if dir_is_there:
874  row_summary.subdirs.append(subdirs_dict[subdir_name])
875 
876  # one first row for the summary!
877  row_summary.calcStats()
878  img_link=row_summary.get_summary_chart_ajax(55,55)
879  page_html+='<td style="background-color:white;"><div class="span-1">'
880  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
881 
882  for sample in sorted_samples:
883  sample_counter+=1
884 
885  directory=dir_dict[sample]
886  subdirs_dict=directory.get_subdirs_dict()
887 
888  # Check if the directory is the top one
889  if directory.name!="":
890  # We did not run on the topdir
891  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
892  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
893  if hashing_flag:
894  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
895  else:
896  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
897  else:
898  #print " ## summary_page: %s"%(directory.name+subdir_name)
899  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
900  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
901  dir_is_there=subdir_name in subdirs_dict
902 
903  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
904  img_tooltip="N/A"
905  if dir_is_there:
906  #row_summary.subdirs.append(subdirs_dict[subdir_name])
907  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
908  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
909 
910  page_html+='<td style="background-color:white;"><div class="span-1">'
911  if dir_is_there:
912  page_html+='<a href="%s">'%(summary_page)
913  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
914  if dir_is_there:
915  page_html+='</a>'
916  page_html+='</div></td>'
917 
918  page_html+=" </tr>\n"
919 
920 
921 
922  page_html+='</table> <a href="#top">Top...</a><hr>'
923 
924  page_html+=get_rank_section(global_dir)
925 
926  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
927 
928  page_html+=get_page_footer()
929  return page_html
930 
931 
932 #-----------UPDATES------
933 def hash_name(file_name, flag):
934  #print " HashFILE name: "+file_name
935  if flag: #if hashing flag is ON then return
936  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
937  else:
938  return file_name #return standart name
directories2html.build_gauge
def build_gauge(total_success_rate, minrate=.80, small=False, escaped=False)
Definition: directories2html.py:493
directories2html.hash_name
def hash_name(file_name, flag)
Definition: directories2html.py:933
directories2html.get_comparisons
def get_comparisons(category, directory)
Definition: directories2html.py:281
directories2html.get_subdirs_section
def get_subdirs_section(directory, hashing_flag)
Definition: directories2html.py:186
directories2html.get_pie_tooltip
def get_pie_tooltip(directory)
Definition: directories2html.py:657
directories2html.build_obj
def build_obj(run, sample, version, plot_path, tier)
Definition: directories2html.py:54
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
directories2html.get_aggr_pairs_info
def get_aggr_pairs_info(dir_dict, the_aggr_pairs=[])
Definition: directories2html.py:514
directories2html.make_summary_table
def make_summary_table(indir, aggregation_rules, aggregation_rules_twiki, hashing_flag, standalone_flag)
Definition: directories2html.py:697
directories2html.plot_size
def plot_size(h=250, w=200)
Definition: directories2html.py:41
directories2html.get_page_header
def get_page_header(directory=None, standalone=False, additional_header="")
Definition: directories2html.py:78
directories2html.get_title_section
def get_title_section(directory, hashing_flag, standalone, depth=2)
Definition: directories2html.py:122
directories2html.get_dir_stats
def get_dir_stats(directory)
Definition: directories2html.py:168
directories2html.make_barchart_summary
def make_barchart_summary(dir_dict, name="the_chart", title="DQM directory", the_aggr_pairs=[])
Definition: directories2html.py:663
directories2html.get_summary_section
def get_summary_section(directory, matrix_page=True)
Definition: directories2html.py:221
directories2html.directory2html
def directory2html(directory, hashing, standalone, depth=0)
Definition: directories2html.py:431
directories2html.make_twiki_table
def make_twiki_table(dir_dict, aggregation_rules)
Definition: directories2html.py:622
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:28
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
directories2html.make_categories_summary
def make_categories_summary(dir_dict, aggregation_rules)
Definition: directories2html.py:582
directories2html.fairy_url_single
def fairy_url_single(run, sample, version, plot_path, tier, draw_opts="", h=250, w=200)
Definition: directories2html.py:68
directories2html.fairy_url
def fairy_url(run1, run2, sample1, sample2, version1, version2, plot_path, tier1, tier2, draw_opts="", h=250, w=200)
Definition: directories2html.py:58
directories2html.build_obj_addr
def build_obj_addr(run, sample, version, plot_path, tier)
Definition: directories2html.py:44
split
double split
Definition: MVATrainer.cc:139
directories2html.get_missing_objs_section
def get_missing_objs_section(directory)
Definition: directories2html.py:418
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
directories2html.get_rank_section
def get_rank_section(directory)
Definition: directories2html.py:370
Generated for CMSSW Reference Manual by doxygen 1.8.11