CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
directories2html.py
Go to the documentation of this file.
1 ################################################################################
2 # RelMon: a tool for automatic Release Comparison
3 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
4 #
5 # $Author: anorkus $
6 # $Date: 2013/03/07 11:46:56 $
7 # $Revision: 1.6 $
8 
9 #
10 #
11 # Danilo Piparo CERN - danilo.piparo@cern.ch
12 #
13 ################################################################################
14 
15 from os import chdir,getcwd,listdir,makedirs
16 from os.path import basename,join,exists
17 import cgi
18 
19 import sys
20 theargv=sys.argv
21 sys.argv=[]
22 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
23 sys.argv=theargv
24 
25 import os
26 if os.environ.has_key("RELMON_SA"):
27  from dirstructure import Comparison,Directory
28  from definitions import *
29  from utils import unpickler
30 else:
31  from Utilities.RelMon.dirstructure import Comparison,Directory
32  from Utilities.RelMon.definitions import *
33  from Utilities.RelMon.utils import unpickler
34 
35  import hashlib
36 #-------------------------------------------------------------------------------
37 
38 def encode_obj_url(url):
39  for old,new in url_encode_dict.items():
40  url=url.replace(old,new)
41  return url
42 
43 def plot_size(h=250,w=200):
44  return "w=%s;h=%s" %(h,w)
45 
46 def build_obj_addr(run,sample,version,plot_path,tier):
47  slash="/"
48  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
49  obj_url+=plot_path
50  while obj_url.endswith(slash):
51  obj_url=obj_url[:-1]
52  while slash*2 in obj_url:
53  obj_url=obj_url.replace(slash*2,slash)
54  return obj_url
55 
56 def build_obj(run,sample,version,plot_path,tier):
57  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
58  return encode_obj_url(obj_url)
59 
60 def fairy_url(run,sample,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
61  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
62  fairy_url+= build_obj(run,sample,version1,plot_path,tier1)
63  fairy_url+= build_obj(run,sample,version2,plot_path,tier2)
64  if len(draw_opts)>0:
65  fairy_url+="drawopts=%s;" %draw_opts
66  fairy_url+= plot_size(h,w)
67  return fairy_url
68 
69 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
70  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
71  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
72  fairy_url+= "?%s"%plot_size(h,w)
73  if len(draw_opts)>0:
74  fairy_url+="drawopts=%s;" %draw_opts
75  return fairy_url
76 
77 #-------------------------------------------------------------------------------
78 style_location="/cms-service-reldqm"
79 def get_page_header(directory=None,additional_header=""):
80  javascripts=''
81  style=''
82  if directory!=None and len(directory.comparisons)>0:
83  meta=directory.meta
84  style='img.fail {border:1px solid #ff0000;}\n'+\
85  'img.succes {border:1px solid #00ff00;}\n'+\
86  'img.null {border:1px solid #ffff00;}\n'+\
87  'img.skiped {border:1px solid #7a7a7a;}\n'+\
88  'a.black_link:link {color: #333333}\n'+\
89  'a.black_link:hover {color: #737373}\n'+\
90  'a.black_link:visited {color: #333333}\n'+\
91  'a.black_link:active {color: #333333}\n'
92  javascripts=""
93 
94 
95  html='<html>'+\
96  '<head>'+\
97  '<title>RelMon Summary</title>'+\
98  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
99  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
100  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
101  '<style type="text/css">'+\
102  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
103  '%s'%style+\
104  '</style>'+\
105  '%s'%javascripts+\
106  '%s'%additional_header+\
107  '</head>'+\
108  '<body>'+\
109  '<div class="container">'
110 
111  return html
112 
113 #-------------------------------------------------------------------------------
114 
115 def get_page_footer():
116  return '</div></body></html>'
117 
118 #-------------------------------------------------------------------------------
119 
120 def get_title_section(directory, hashing_flag, depth=2):
121  mother_name=basename(directory.mother_dir)
122  mother_file_name=""
123  if depth==1:
124  mother_file_name="../RelMonSummary.html"
125  if mother_name!="":
126  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
127  elif depth==2:
128  #mother_file_name="RelMonSummary.html"
129  mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
130  if mother_name!="":
131  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
132  else:
133  if hashing_flag:
134  files = directory.mother_dir.split("/")
135  if len(files) != 1:
136  dir_name = files[-2] ##return the mother directory name only as the html file name by it
137  else:
138  dir_name = files[-1]
139  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
140  else:
141  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
142  mother_file_name=mother_file_name.strip("_")
143 
144  link_to_mother='<a href="%s">..</a>' %mother_file_name
145  html= '<div class="span-20">'+\
146  '<h1>%s</h1>'%directory.name+\
147  '</div>'+\
148  '<div class="span-1">'+\
149  '<h1>%s</h1>'%link_to_mother+\
150  '</div>'+\
151  '<div class="span-3 last">'+\
152  '<img src="cms-service-reldqm/style/CMS.gif" class="top right" width="54" hight="54">'+\
153  '</div>'+\
154  '<hr>'
155  if len(mother_name)>0:
156  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
157  '<hr>'
158  return html
159 
160 #-------------------------------------------------------------------------------
161 def get_dir_stats(directory):
162  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
163  html+='<ul>'
164  if directory.n_successes>0:
165  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
166  if directory.n_nulls>0:
167  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
168  if directory.n_fails>0:
169  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
170  if directory.n_skiped>0:
171  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
172  html+='</ul>'
173  return html
174 
175 #-------------------------------------------------------------------------------
176 
177 def get_subdirs_section(directory, hashing_flag):
178  if len(directory.subdirs)==0:
179  return ""
180  html= '<div class="span-20 colborder">'
181  html+='<h2 class="alt">Sub-Directories</h2>'
182  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
183  # so reverse :)
184  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
185  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
186  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
187  for subdir in sorted_subdirs:
188  name=subdir.name
189  if hashing_flag:
190  link = "%s.html" %(hash_name(name, hashing_flag))
191  else:
192  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
193  link=link.strip("_")
194  html+='<div class="span-4 prepend-2 colborder">'
195  html+='<h3>%s</h3>'%name
196  html+='</div>'
197 
198  html+='<div class="span-7">'
199  html+=get_dir_stats(subdir)
200  html+='</div>'
201 
202  html+='<div class="span-6 last">'
203  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
204  html+='</div>'
205 
206  html+='<hr>'
207  return html+'</div>'
208 
209 
210 #-------------------------------------------------------------------------------
211 
212 def get_summary_section(directory,matrix_page=True):
213 
214  # Hack find the first comparison and fill test and threshold
215  # shall this be put in meta?
216  test_name=""
217  test_threshold=0
218  for comparison in directory.comparisons:
219  test_name=comparison.test_name
220  test_threshold=comparison.test_thr
221  break
222  if len(test_name)==0:
223  for subdir in directory.subdirs:
224  for comparison in subdir.comparisons:
225  test_name=comparison.test_name
226  test_threshold=comparison.test_thr
227  break
228  if len(test_name)!=0:break
229  if len(test_name)==0:
230  for subsubdir in subdir.subdirs:
231  for comparison in subsubdir.comparisons:
232  test_name=comparison.test_name
233  test_threshold=comparison.test_thr
234  break
235  if len(test_name)!=0:break
236  if len(test_name)==0:
237  for subsubsubdir in subsubdir.subdirs:
238  for comparison in subsubsubdir.comparisons:
239  test_name=comparison.test_name
240  test_threshold=comparison.test_thr
241  break
242  if len(test_name)!=0:break
243 
244 
245  meta=directory.meta
246 
247  html= '<div class="span-6">'+\
248  '<h3>Summary</h3>'
249  html+=get_dir_stats(directory)
250  html+='<a href="%s/%s">To the DQM GUI...</a>' %(server,base_url)
251  html+='</div>'
252 
253  html+='<div class="span-7 colborder">'+\
254  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
255  '</div>'+\
256  '<div class="span-9 last">'
257  if matrix_page:
258  html+='<h3>Sample:</h3>'+\
259  '<p class="caps">%s</p>'%meta.sample+\
260  '<h3>Run1 and Run2:</h3>'+\
261  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
262  html+='<h3>Releases:</h3>'+\
263  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
264  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
265  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
266  '</div>'+\
267  '<hr>'
268  return html
269 
270 #-------------------------------------------------------------------------------
271 
272 def get_comparisons(category,directory):
273  """Prepare the comparisons between histograms and organise them in the page.
274  Moreover create separate pages with the overlay and the single plots.
275  """
276  counter=1
277  tot_counter=1
278 
279  # get the right ones
280  comparisons= filter (lambda comp: comp.status == cat_states[category] , directory.comparisons)
281  n_comparisons=len(comparisons)
282 
283  is_reverse=True
284  if category == FAIL:
285  is_reverse=False
286  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
287 
288 
289  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
290  html_comparisons=""
291  for comparison in comparisons:
292  plot_name=comparison.img_name
293  if "http://" not in plot_name:
294  plot_name= basename(comparison.img_name)
295  class_type="colborder"
296  if counter==3 or tot_counter==n_comparisons:
297  class_type=" colborder last"
298  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
299 
300 
301  if directory.do_pngs:
302  png_link=comparison.img_name
303  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
304  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
305  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
306  else:
307  big_fairy=fairy_url(directory.meta.run1,
308  directory.meta.sample,
309  directory.meta.release1,
310  directory.meta.release2,
311  comp_abs_path,
312  directory.meta.tier1,
313  directory.meta.tier2,
314  "",600,600)
315  small_fairy=fairy_url(directory.meta.run1,
316  directory.meta.sample,
317  directory.meta.release1,
318  directory.meta.release2,
319  comp_abs_path,
320  directory.meta.tier1,
321  directory.meta.tier2)
322 
323  single_fairy1=fairy_url_single(directory.meta.run1,
324  directory.meta.sample,
325  directory.meta.release1,
326  comp_abs_path,
327  directory.meta.tier1,
328  "",500,500)
329  single_fairy2=fairy_url_single(directory.meta.run2,
330  directory.meta.sample,
331  directory.meta.release2,
332  comp_abs_path,
333  directory.meta.tier2,
334  "",500,500)
335 
336  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
337  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
338  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
339  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
340  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
341 
342  if counter==3:
343  html_comparisons+="<hr>"
344  counter=0
345  counter+=1
346  tot_counter+=1
347 
348  if len(html_comparisons)!=0:
349  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
350  html+=html_comparisons
351  html+='<hr>'
352  return html
353  return ""
354 
355 #-------------------------------------------------------------------------------
356 
357 def get_rank_section(directory):
358 # do Rank histo png
359  imgname="RankSummary.png"
360  gStyle.SetPadTickY(0)
361  c=TCanvas("ranks","ranks",500,400)
362  #gStyle.SetOptStat(0)
363  c.cd()
364 
365  h=directory.rank_histo
366  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
367  rank_histof.SetLineWidth(2)
368  for i in xrange(0,h.GetNbinsX()+1):
369  rank_histof.SetBinContent(i,h.GetBinContent(i))
370  h.SetTitle("Ranks Summary;Rank;Frequency")
371  h.Draw("Hist")
372  c.Update()
373  rank_histof.ComputeIntegral()
374  integral = rank_histof.GetIntegral()
375  rank_histof.SetContent(integral)
376 
377  rightmax = 1.1*rank_histof.GetMaximum()
378  scale = gPad.GetUymax()/rightmax
379  rank_histof.SetLineColor(kRed)
380  rank_histof.Scale(scale)
381  rank_histof.Draw("same")
382 
383  #draw an axis on the right side
384  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
385  axis.SetTitle("Cumulative")
386  axis.SetTitleColor(kRed)
387  axis.SetLineColor(kRed)
388  axis.SetLabelColor(kRed)
389  axis.Draw()
390 
391  rank_histof.Draw("Same");
392 
393 
394  c.Print(imgname)
395 
396  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
397  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
398  page_html+='</div> <a href="#top">Top...</a><hr>'
399 
400  return page_html
401 
402 #-------------------------------------------------------------------------------
403 
404 def directory2html(directory, hashing, depth=0):
405  """Converts a directory tree into html pages, very nice ones.
406  """
407  #print "d2html: depth", str(depth)," dir ",directory.name
408  depth+=1
409  #old_cwd=getcwd()
410  #if not exists(directory.name) and len(directory.name)>0:
411  #makedirs(directory.name)
412 
413  #if len(directory.name)>0:
414  #chdir(directory.name)
415 
416  for subdir in directory.subdirs:
417  directory2html(subdir,hashing, depth)
418 
419  page_html=get_page_header(directory)+\
420  get_title_section(directory,hashing, depth)+\
421  get_summary_section(directory)+\
422  get_subdirs_section(directory, hashing)
423 
424  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
425  (directory.n_comp_nulls >0,NULL ),
426 
427  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
428  (directory.n_comp_skiped >0,SKIPED)):
429  if do_cat:
430  page_html+=get_comparisons(cat,directory)
431 
432  # Distribution of ranks
433 
434  if depth==1:
435  page_html+=get_rank_section(directory)
436 
437 
438  page_html+=get_page_footer()
439 
440  page_name=directory.name
441 
442  if len(page_name)==0:
443  page_name="RelMonSummary"
444  if hashing:
445  ofilename = "%s.html" %(hash_name(page_name, hashing))
446  else:
447  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
448  ofilename=ofilename.strip("_")
449  #print "Writing on %s" %ofilename
450  ofile=open(ofilename,"w")
451  ofile.write(page_html)
452  ofile.close()
453 
454  #chdir(old_cwd)
455 
456 #-------------------------------------------------------------------------------
457 
458 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
459  total_success_rate_scaled=(total_success_rate-minrate)
460  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
461  if total_success_rate_scaled_repr<0:
462  total_success_rate_scaled_repr=0
463  size_s="200x100"
464  if small:
465  size_s="40x30"
466  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
467  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
468  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
469  if not small:
470  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
471  gauge_link+="&chma=10,10,10,0"
472  img_tag= '<img src="%s">'%gauge_link
473  if escaped:
474  img_tag=cgi.escape(img_tag)
475  return img_tag
476 
477 #-------------------------------------------------------------------------------
478 
479 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
480  # Let's make a summary for an overview in categories act on the global dir
481  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
482 
483  list_of_names=[]
484  if the_aggr_pairs==[]:
485  for samplename,sampledir in dir_dict.items():
486  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
487  if not subsysdirname in list_of_names:
488  list_of_names.append(subsysdirname)
489  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
490 
491  #print the_aggr_pairs
492  for cat_name, subdir_list in the_aggr_pairs:
493  total_successes=0.
494  total_directory_successes=0
495  total_weight=0.
496  present_subdirs={}
497  total_ndirs=0
498  # Loop on samples
499  for dirname, sampledir in dir_dict.items():
500  # Loop on directories
501  for subdirname,subdir in sampledir.get_subdirs_dict().items():
502  if subdirname in subdir_list:
503  nsucc=subdir.n_successes
504  total_successes+=nsucc
505  weight=subdir.weight
506  total_weight+=weight
507  total_ndirs+=1
508 
509  total_directory_successes+= float(nsucc)/weight
510  if present_subdirs.has_key(subdirname):
511  this_dir_dict=present_subdirs[subdirname]
512  this_dir_dict["nsucc"]+=nsucc
513  this_dir_dict["weight"]+=weight
514  else:
515  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
516  # Make it usable also for subdirectories
517  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
518  for pathname in filter(lambda name:"/" in name,subdir_list):
519  selected_subdirname,selected_subsubdirname = pathname.split("/")
520  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
521  #print "Studying directory ",subsubdirname," in directory ",subdirname
522  nsucc=subsubdir.n_successes
523  total_successes+=nsucc
524  weight=subsubdir.weight
525  total_weight+=weight
526  total_ndirs+=1
527  total_directory_successes+= float(nsucc)/weight
528 
529  if present_subdirs.has_key(subsubdirname):
530  this_dir_dict=present_subdirs[subsubdirname]
531  this_dir_dict["nsucc"]+=nsucc
532  this_dir_dict["weight"]+=weight
533  else:
534  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
535 
536  if total_ndirs == 0:
537  print "No directory of the category %s is present in the samples: skipping." %cat_name
538  continue
539 
540  average_success_rate=total_directory_successes/(total_ndirs)
541  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
542 
543  return aggr_pairs_info
544 
545 #-------------------------------------------------------------------------------
546 
547 def make_categories_summary(dir_dict,aggregation_rules):
548 
549  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
550 
551  #print aggr_pairs_info
552 
553  # Now Let's build the HTML
554 
555  html= '<div class="span-20 colborder">'
556  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
557 
558  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
559  #print cat_name,present_subdirs,total_weight,average_success_rate
560  html+='<div class="span-3 prepend-0 colborder">'
561  html+='<h3>%s</h3>'%cat_name
562  html+='<div><span class="alt">Avg. Success rate:</span></div>'
563  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
564  html+='</div>'
565  html+='<div class="span-9">'
566 
567  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
568  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
569  html+='<ul>'
570  for subdirname in sorted(present_subdirs.keys()):
571  this_dir_dict=present_subdirs[subdirname]
572  nsucc=this_dir_dict["nsucc"]
573  weight=this_dir_dict["weight"]
574  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
575  html+='</ul>'
576  html+='</div>'
577 
578  html+='<div class="span-6 last">'
579  html+=build_gauge(average_success_rate)
580  html+='</div>'
581 
582  html+='<hr>'
583  return html+'<br><a href="#top">Top...</a> </div><hr>'
584 
585  #-------------------------------------------------------------------------------
586 
587 def make_twiki_table(dir_dict,aggregation_rules):
588 
589  # decide the release
590  meta= dir_dict.items()[0][1].meta
591  releases=sorted([meta.release1,meta.release2])
592  latest_release=releases[1].split("-")[0]
593 
594 
595  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
596 
597  # Now Let's build the HTML
598 
599  html= '<div class="span-20 colborder">'
600  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
601  html+='<div>| Release | Comparison |'
602  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
603  html+=cat_name
604  html+=" | "
605  html+='</div>'
606 
607  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
608 
609  # Now add all the line with small gauges
610 
611  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
612  #print cat_name,present_subdirs,total_weight,average_success_rate
613  html+=build_gauge(average_success_rate,small=True,escaped=True)
614  html+=" | "
615 
616  html+='</div> <a href="#top">Top...</a>'
617  html+='<hr>'
618  return html+'</div>'
619 
620 #-------------------------------------------------------------------------------
621 
622 def get_pie_tooltip(directory):
623  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
624  return tooltip
625 
626 #-------------------------------------------------------------------------------
627 
628 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
629 
630  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
631 
632  script="""
633  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
634  <script type="text/javascript">
635  google.load("visualization", "1", {packages:["corechart"]});
636  google.setOnLoadCallback(drawChart);
637  function drawChart() {
638  var data = new google.visualization.DataTable();
639  data.addColumn('string', 'DQM Directory');
640  data.addColumn('number', 'Success Rate');
641  """
642  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
643  counter=0
644  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
645  #print subsystname,present_directories
646  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
647  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
648  counter+=1
649  script+="""
650  var chart = new google.visualization.BarChart(document.getElementById('%s'));
651  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
652  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
653  });
654  }
655  </script>
656  """%(name,40*counter,title)
657  return script
658 
659 
660 #-------------------------------------------------------------------------------
661 
662 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag):
663  """Create a table, with as rows the directories and as columns the samples.
664  Each box in the table will contain a pie chart linking to the directory.
665  """
666  #aggregation_rules={}
667  #aggregation_rules_twiki={}
668 
669  chdir(indir)
670  if os.path.isabs(indir):
671  title = basename(indir)
672  else:
673  title=indir
674  title=title.strip(".")
675  title=title.strip("/")
676 
677 
678  # Get the list of pickles
679  sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
680 
681  # Load directories, build a list of all first level subdirs
682  dir_unpicklers=[]
683  n_unpicklers=0
684  for sample_pkl in sample_pkls:
685  dir_unpickler=unpickler(sample_pkl)
686  dir_unpickler.start()
687  n_unpicklers+=1
688  dir_unpicklers.append(dir_unpickler)
689  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
690  n_unpicklers=0
691  for dir_unpickler in dir_unpicklers:
692  dir_unpickler.join()
693 
694  dir_dict={}
695 
696  # create a fake global directory
697  global_dir=Directory("global","")
698  for dir_unpickler in dir_unpicklers:
699  dir_unpickler.join()
700  directory=dir_unpickler.directory
701  #directory.prune("Run summary")
702  #directory.calcStats()
703  global_dir.meta=directory.meta
704  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
705  global_dir.subdirs.append(directory)
706 
707  global_dir.calcStats()
708 
709  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
710  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
711 
712  page_html = get_page_header(additional_header=directories_barchart+categories_barchart)
713  rel1=""
714  rel2=""
715  try:
716  rel1,rel2=title.split("VS")
717  except:
718  rel1=global_dir.meta.release1.split("-")[0]
719  rel2=global_dir.meta.release2.split("-")[0]
720  global_dir.meta.release1=rel1
721  global_dir.meta.release2=rel2
722 
723  # union of all subdirs names
724  all_subdirs=[]
725  for directory in dir_dict.values():
726  for subdir_name in directory.get_subdirs_names():
727  all_subdirs.append(subdir_name)
728  all_subdirs=sorted(list(set(all_subdirs)))
729 
730  # Get The title
731  page_html+= '<div class="span-20">'+\
732  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
733  '</div>'+\
734  '<div class="span-1">'+\
735  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
736  '</div>'+\
737  '<hr>'
738  page_html+='<div class="span-24"><p></p></div>\n'*3
739 
740  # Get The summary section
741  page_html+= get_summary_section(global_dir,False)
742 
743  # Make the anchor sections
744  page_html+= '<div class="span-24">'
745  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
746  '<ul>'+\
747  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
748  '<li><a href="#categories">Categories</a></li>'+\
749  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
750  '<li><a href="#summary_table">Summary Table</a></li>'+\
751  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
752  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
753  '</ul>'+\
754  '</div><hr>'
755 
756 
757  # Make the CategoriesBar chart
758  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
759  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
760 
761  # Make the gauges per categories
762  page_html+=make_categories_summary(dir_dict,aggregation_rules)
763 
764  # Make the Directories chart
765  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
766  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
767 
768  # Barbarian vertical space. Suggestions are welcome
769  for i in xrange(2):
770  page_html+='<div class="span-24"><p></p></div>\n'
771 
772 
773  # Prepare the table
774  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
775 
776  for i in xrange(5):
777  page_html+='<div class="span-24"><p></p></div>\n'
778 
779  page_html+="""
780  <table border="1" >
781  <tr>
782  <td> </td>
783  """
784 
785  # First row with samples
786  page_html+="""
787  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
788 
789  sorted_samples=sorted(dir_dict.keys())
790  for sample in sorted_samples:
791  sample_nick=sample
792  ## For runs: put only the number after the _
793  #if "_" in sample:
794  #run_number=sample.split("_")[-1]
795  #if (not run_number.isalpha()) and len(run_number)>=6:
796  #sample_nick=run_number
797 
798 
799  page_html+="""
800  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
801  page_html+=" </tr>\n"
802 
803 
804  # FIRST ROW
805  # Now the summaries at the beginning of the table
806  page_html+="<tr>"
807  page_html+='<td style="background-color:white;"><div class="span-1">'
808 
809  page_html+='<b>Summary</b></div></td>'
810  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
811  for sample in sorted_samples:
812  col=dir_dict[sample]
813  # check if the directory was a top one or not
814  summary_page_name=hash_name("RelMonSummary", hashing_flag)+".html"
815  if col.name!="":
816  summary_page_name=hash_name(col.name, hashing_flag)+".html"
817  img_link=col.get_summary_chart_ajax(55,55)
818  page_html+='<td style="background-color:white;"><div class="span-1">'
819  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
820  page_html+="</tr>"
821 
822  # Now the content
823  for subdir_name in all_subdirs:
824 
825  page_html+=' <tr>\n'
826  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
827 
828  row_summary=Directory("row_summary","")
829  sample_counter=0
830  n_samples=len(sorted_samples)
831 
832  for sample in sorted_samples:
833  subdirs_dict=directory.get_subdirs_dict()
834  directory=dir_dict[sample]
835  dir_is_there=subdirs_dict.has_key(subdir_name)
836  if dir_is_there:
837  row_summary.subdirs.append(subdirs_dict[subdir_name])
838 
839  # one first row for the summary!
840  row_summary.calcStats()
841  img_link=row_summary.get_summary_chart_ajax(55,55)
842  page_html+='<td style="background-color:white;"><div class="span-1">'
843  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
844 
845  for sample in sorted_samples:
846  sample_counter+=1
847 
848  directory=dir_dict[sample]
849  subdirs_dict=directory.get_subdirs_dict()
850 
851  # Check if the directory is the top one
852  summary_page=join(sample,"%s.html"%(hash_name(subdir_name, hashing_flag)))
853  if directory.name!="":
854  # We did not run on the topdir
855  summary_page=join(sample,"%s_%s.html"%(directory.name,hash_name(subdir_name,hashing_flag)))
856  dir_is_there=subdirs_dict.has_key(subdir_name)
857 
858  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
859  img_tooltip="N/A"
860  if dir_is_there:
861  #row_summary.subdirs.append(subdirs_dict[subdir_name])
862  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
863  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
864 
865  page_html+='<td style="background-color:white;"><div class="span-1">'
866  if dir_is_there:
867  page_html+='<a href="%s">'%(summary_page)
868  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
869  if dir_is_there:
870  page_html+='</a>'
871  page_html+='</div></td>'
872 
873  page_html+=" </tr>\n"
874 
875 
876 
877  page_html+='</table> <a href="#top">Top...</a><hr>'
878 
879  page_html+=get_rank_section(global_dir)
880 
881  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
882 
883  page_html+=get_page_footer()
884  return page_html
885 
886 
887 #-----------UPDATES------
888 def hash_name(file_name, flag):
889  if flag: #if hashing flag is ON then return
890  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
891  else:
892  return file_name #return standart name
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
split
double split
Definition: MVATrainer.cc:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
runtimedef::set
void set(const std::string &name, int value)
set the flag, with a run-time name
Definition: ProfilingTools.cc:67
Generated for CMSSW Reference Manual by doxygen 1.8.5