CMS 3D CMS Logo
directories2html.py
Go to the documentation of this file.
1 ################################################################################
2 # RelMon: a tool for automatic Release Comparison
3 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
4 #
5 
6 #
7 #
8 # Danilo Piparo CERN - danilo.piparo@cern.ch
9 #
10 ################################################################################
11 
12 from os import chdir,getcwd,listdir,makedirs
13 from os.path import basename,join,exists
14 import cgi
15 
16 import sys
17 theargv=sys.argv
18 sys.argv=[]
19 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
20 sys.argv=theargv
21 
22 import os
23 if "RELMON_SA" in os.environ:
24  from dirstructure import Comparison,Directory
25  from definitions import *
26  from utils import unpickler
27 else:
28  from Utilities.RelMon.dirstructure import Comparison,Directory
29  from Utilities.RelMon.definitions import *
30  from Utilities.RelMon.utils import unpickler
31 
32  import hashlib
33 #-------------------------------------------------------------------------------
34 
35 def encode_obj_url(url):
36  for old,new in url_encode_dict.items():
37  url=url.replace(old,new)
38  return url
39 
40 def plot_size(h=250,w=200):
41  return "w=%s;h=%s" %(h,w)
42 
43 def build_obj_addr(run,sample,version,plot_path,tier):
44  slash="/"
45  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
46  obj_url+=plot_path
47  while obj_url.endswith(slash):
48  obj_url=obj_url[:-1]
49  while slash*2 in obj_url:
50  obj_url=obj_url.replace(slash*2,slash)
51  return obj_url
52 
53 def build_obj(run,sample,version,plot_path,tier):
54  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
55  return encode_obj_url(obj_url)
56 
57 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
58  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
59  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
60  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
61  if len(draw_opts)>0:
62  fairy_url+="drawopts=%s;" %draw_opts
63  fairy_url+= plot_size(h,w)
64  fairy_url += ";ref=ratiooverlay"
65  return fairy_url
66 
67 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
68  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
69  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
70  fairy_url+= "?%s"%plot_size(h,w)
71  if len(draw_opts)>0:
72  fairy_url+="drawopts=%s;" %draw_opts
73  return fairy_url
74 
75 #-------------------------------------------------------------------------------
76 style_location="/cms-service-reldqm"
77 def get_page_header(directory=None, standalone=False, additional_header=""):
78  style_location="/cms-service-reldqm"
79  if standalone:
80  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
81  javascripts=''
82  style=''
83  if directory!=None and len(directory.comparisons)>0:
84  meta=directory.meta
85  style='img.fail {border:1px solid #ff0000;}\n'+\
86  'img.succes {border:1px solid #00ff00;}\n'+\
87  'img.null {border:1px solid #ffff00;}\n'+\
88  'img.skiped {border:1px solid #7a7a7a;}\n'+\
89  'a.black_link:link {color: #333333}\n'+\
90  'a.black_link:hover {color: #737373}\n'+\
91  'a.black_link:visited {color: #333333}\n'+\
92  'a.black_link:active {color: #333333}\n'
93  javascripts=""
94 
95 
96  html='<html>'+\
97  '<head>'+\
98  '<title>RelMon Summary</title>'+\
99  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
100  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
101  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
102  '<style type="text/css">'+\
103  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
104  '%s'%style+\
105  '</style>'+\
106  '%s'%javascripts+\
107  '%s'%additional_header+\
108  '</head>'+\
109  '<body>'+\
110  '<div class="container">'
111 
112  return html
113 
114 #-------------------------------------------------------------------------------
115 
116 def get_page_footer():
117  return '</div></body></html>'
118 
119 #-------------------------------------------------------------------------------
120 
121 def get_title_section(directory, hashing_flag, standalone, depth=2):
122  if standalone:
123  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
124  else:
125  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
126  mother_name=basename(directory.mother_dir)
127  mother_file_name=""
128  if depth==1:
129  mother_file_name="../RelMonSummary.html"
130  if mother_name!="":
131  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
132  elif depth==2:
133  mother_file_name="RelMonSummary.html"
134  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
135  if mother_name!="":
136  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
137  else:
138  if hashing_flag:
139  files = directory.mother_dir.split("/")
140  if len(files) != 1:
141  dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
142  else:
143  dir_name = files[-1]
144  dir_name = directory.mother_dir
145  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
146  else:
147  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
148  mother_file_name=mother_file_name.strip("_")
149 
150  link_to_mother='<a href="%s">..</a>' %mother_file_name
151  html= '<div class="span-20">'+\
152  '<h1>%s</h1>'%directory.name+\
153  '</div>'+\
154  '<div class="span-1">'+\
155  '<h1>%s</h1>'%link_to_mother+\
156  '</div>'+\
157  '<div class="span-3 last">'+\
158  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
159  '</div>'+\
160  '<hr>'
161  if len(mother_name)>0:
162  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
163  '<hr>'
164  return html
165 
166 #-------------------------------------------------------------------------------
167 def get_dir_stats(directory):
168  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
169  html+='<ul>'
170  if directory.n_successes>0:
171  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
172  if directory.n_nulls>0:
173  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
174  if directory.n_fails>0:
175  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
176  if directory.n_skiped>0:
177  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
178  if directory.n_missing_objs>0:
179  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
180  html+='</ul>'
181  return html
182 
183 #-------------------------------------------------------------------------------
184 
185 def get_subdirs_section(directory, hashing_flag):
186  if len(directory.subdirs)==0:
187  return ""
188  html= '<div class="span-20 colborder">'
189  html+='<h2 class="alt">Sub-Directories</h2>'
190  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
191  # so reverse :)
192  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
193  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
194  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
195  for subdir in sorted_subdirs:
196  name=subdir.name
197  if hashing_flag:
198  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
199  else:
200  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
201  link=link.strip("_")
202  html+='<div class="span-4 prepend-2 colborder">'
203  html+='<h3>%s</h3>'%name
204  html+='</div>'
205 
206  html+='<div class="span-7">'
207  html+=get_dir_stats(subdir)
208  html+='</div>'
209 
210  html+='<div class="span-6 last">'
211  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
212  html+='</div>'
213 
214  html+='<hr>'
215  return html+'</div>'
216 
217 
218 #-------------------------------------------------------------------------------
219 
220 def get_summary_section(directory,matrix_page=True):
221 
222  # Hack find the first comparison and fill test and threshold
223  # shall this be put in meta?
224  test_name=""
225  test_threshold=0
226  for comparison in directory.comparisons:
227  test_name=comparison.test_name
228  test_threshold=comparison.test_thr
229  break
230  if len(test_name)==0:
231  for subdir in directory.subdirs:
232  for comparison in subdir.comparisons:
233  test_name=comparison.test_name
234  test_threshold=comparison.test_thr
235  break
236  if len(test_name)!=0:break
237  if len(test_name)==0:
238  for subsubdir in subdir.subdirs:
239  for comparison in subsubdir.comparisons:
240  test_name=comparison.test_name
241  test_threshold=comparison.test_thr
242  break
243  if len(test_name)!=0:break
244  if len(test_name)==0:
245  for subsubsubdir in subsubdir.subdirs:
246  for comparison in subsubsubdir.comparisons:
247  test_name=comparison.test_name
248  test_threshold=comparison.test_thr
249  break
250  if len(test_name)!=0:break
251 
252 
253  meta=directory.meta
254 
255  html= '<div class="span-6">'+\
256  '<h3>Summary</h3>'
257  html+=get_dir_stats(directory)
258  html+='<a href="%s/%s">To the DQM GUI...</a>' %(server,base_url)
259  html+='</div>'
260 
261  html+='<div class="span-7 colborder">'+\
262  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
263  '</div>'+\
264  '<div class="span-9 last">'
265  if matrix_page:
266  html+='<h3>Sample:</h3>'+\
267  '<p class="caps">%s</p>'%meta.sample1+\
268  '<h3>Run1 and Run2:</h3>'+\
269  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
270  html+='<h3>Releases:</h3>'+\
271  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
272  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
273  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
274  '</div>'+\
275  '<hr>'
276  return html
277 
278 #-------------------------------------------------------------------------------
279 
280 def get_comparisons(category,directory):
281  """Prepare the comparisons between histograms and organise them in the page.
282  Moreover create separate pages with the overlay and the single plots.
283  """
284  counter=1
285  tot_counter=1
286 
287  # get the right ones
288  comparisons= filter (lambda comp: comp.status == cat_states[category] , directory.comparisons)
289  n_comparisons=len(comparisons)
290 
291  is_reverse=True
292  if category == FAIL:
293  is_reverse=False
294  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
295 
296 
297  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
298  html_comparisons=""
299  for comparison in comparisons:
300  plot_name=comparison.img_name
301  if "http://" not in plot_name:
302  plot_name= basename(comparison.img_name)
303  class_type="colborder"
304  if counter==3 or tot_counter==n_comparisons:
305  class_type=" colborder last"
306  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
307 
308 
309  if directory.do_pngs:
310  png_link=comparison.img_name
311  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
312  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
313  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
314  else:
315  big_fairy=fairy_url(directory.meta.run1,
316  directory.meta.run2,
317  directory.meta.sample1,
318  directory.meta.sample2,
319  directory.meta.release1,
320  directory.meta.release2,
321  comp_abs_path,
322  directory.meta.tier1,
323  directory.meta.tier2,
324  "",600,600)
325  small_fairy=fairy_url(directory.meta.run1,
326  directory.meta.run2,
327  directory.meta.sample1,
328  directory.meta.sample2,
329  directory.meta.release1,
330  directory.meta.release2,
331  comp_abs_path,
332  directory.meta.tier1,
333  directory.meta.tier2)
334 
335  single_fairy1=fairy_url_single(directory.meta.run1,
336  directory.meta.sample1,
337  directory.meta.release1,
338  comp_abs_path,
339  directory.meta.tier1,
340  "",500,500)
341  single_fairy2=fairy_url_single(directory.meta.run2,
342  directory.meta.sample2,
343  directory.meta.release2,
344  comp_abs_path,
345  directory.meta.tier2,
346  "",500,500)
347 
348  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
349  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
350  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
351  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
352  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
353 
354  if counter==3:
355  html_comparisons+="<hr>"
356  counter=0
357  counter+=1
358  tot_counter+=1
359 
360  if len(html_comparisons)!=0:
361  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
362  html+=html_comparisons
363  html+='<hr>'
364  return html
365  return ""
366 
367 #-------------------------------------------------------------------------------
368 
369 def get_rank_section(directory):
370 # do Rank histo png
371  imgname="RankSummary.png"
372  gStyle.SetPadTickY(0)
373  c=TCanvas("ranks","ranks",500,400)
374  #gStyle.SetOptStat(0)
375  c.cd()
376 
377  h=directory.rank_histo
378  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
379  rank_histof.SetLineWidth(2)
380  for i in xrange(0,h.GetNbinsX()+1):
381  rank_histof.SetBinContent(i,h.GetBinContent(i))
382  h.SetTitle("Ranks Summary;Rank;Frequency")
383  h.Draw("Hist")
384  c.Update()
385  rank_histof.ComputeIntegral()
386  integral = rank_histof.GetIntegral()
387  rank_histof.SetContent(integral)
388 
389  rightmax = 1.1*rank_histof.GetMaximum()
390  scale = gPad.GetUymax()/rightmax
391  rank_histof.SetLineColor(kRed)
392  rank_histof.Scale(scale)
393  rank_histof.Draw("same")
394 
395  #draw an axis on the right side
396  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
397  axis.SetTitle("Cumulative")
398  axis.SetTitleColor(kRed)
399  axis.SetLineColor(kRed)
400  axis.SetLabelColor(kRed)
401  axis.Draw()
402 
403  rank_histof.Draw("Same");
404 
405 
406  c.Print(imgname)
407 
408  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
409  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
410  page_html+='</div> <a href="#top">Top...</a><hr>'
411 
412  return page_html
413 
414 #-------------------------------------------------------------------------------
415 
416 #-------------------------------------------------------------------------------
417 def get_missing_objs_section(directory):
418  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
419  """
420  page_html = "Unpaired in %s</br>"%(directory.filename1)
421  for elem in directory.different_histograms['file1']:
422  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
423  page_html +="</br>"
424  page_html += "Unpaired in %s</br>"%(directory.filename2)
425  for elem in directory.different_histograms['file2']:
426  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
427  return page_html
428 #-------------------------------------------------------------------------------
429 
430 def directory2html(directory, hashing, standalone, depth=0):
431  """Converts a directory tree into html pages, very nice ones.
432  """
433  #print "d2html: depth", str(depth)," dir ",directory.name
434  depth+=1
435  #old_cwd=getcwd()
436  #if not exists(directory.name) and len(directory.name)>0:
437  #makedirs(directory.name)
438 
439  #if len(directory.name)>0:
440  #chdir(directory.name)
441 
442  for subdir in directory.subdirs:
443  directory2html(subdir,hashing,standalone, depth)
444 
445  page_html=get_page_header(directory, standalone)+\
446  get_title_section(directory,hashing, standalone, depth)+\
447  get_summary_section(directory)+\
448  get_subdirs_section(directory, hashing)
449 
450  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
451  (directory.n_comp_nulls >0,NULL ),
452 
453  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
454  (directory.n_comp_skiped >0,SKIPED)):
455  if do_cat:
456  page_html+=get_comparisons(cat,directory)
457 
458  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
459  page_html += get_missing_objs_section(directory)
460 
461  # Distribution of ranks
462 
463  if depth==1:
464  page_html+=get_rank_section(directory)
465 
466 
467  page_html+=get_page_footer()
468 
469  page_name=directory.name
470 
471  if len(page_name)==0:
472  page_name="RelMonSummary"
473  if hashing:
474  if page_name != "RelMonSummary":
475  #print " ## oFile path: %s"%(join(directory.full_path))
476  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
477  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
478  else:
479  ofilename = "RelMonSummary.html"
480  else:
481  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
482  ofilename=ofilename.strip("_")
483  #print "Writing on %s" %ofilename
484  ofile=open(ofilename,"w")
485  ofile.write(page_html)
486  ofile.close()
487 
488  #chdir(old_cwd)
489 
490 #-------------------------------------------------------------------------------
491 
492 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
493  total_success_rate_scaled=(total_success_rate-minrate)
494  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
495  if total_success_rate_scaled_repr<0:
496  total_success_rate_scaled_repr=0
497  size_s="200x100"
498  if small:
499  size_s="40x30"
500  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
501  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
502  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
503  if not small:
504  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
505  gauge_link+="&chma=10,10,10,0"
506  img_tag= '<img src="%s">'%gauge_link
507  if escaped:
508  img_tag=cgi.escape(img_tag)
509  return img_tag
510 
511 #-------------------------------------------------------------------------------
512 
513 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
514  # Let's make a summary for an overview in categories act on the global dir
515  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
516 
517  list_of_names=[]
518  if the_aggr_pairs==[]:
519  for samplename,sampledir in dir_dict.items():
520  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
521  if not subsysdirname in list_of_names:
522  list_of_names.append(subsysdirname)
523  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
524 
525  #print the_aggr_pairs
526  for cat_name, subdir_list in the_aggr_pairs:
527  total_successes=0.
528  total_directory_successes=0
529  total_weight=0.
530  present_subdirs={}
531  total_ndirs=0
532  # Loop on samples
533  for dirname, sampledir in dir_dict.items():
534  # Loop on directories
535  for subdirname,subdir in sampledir.get_subdirs_dict().items():
536  if subdirname in subdir_list:
537  nsucc=subdir.n_successes
538  total_successes+=nsucc
539  weight=subdir.weight
540  total_weight+=weight
541  total_ndirs+=1
542 
543  total_directory_successes+= float(nsucc)/weight
544  if subdirname in present_subdirs:
545  this_dir_dict=present_subdirs[subdirname]
546  this_dir_dict["nsucc"]+=nsucc
547  this_dir_dict["weight"]+=weight
548  else:
549  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
550  # Make it usable also for subdirectories
551  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
552  for pathname in filter(lambda name:"/" in name,subdir_list):
553  selected_subdirname,selected_subsubdirname = pathname.split("/")
554  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
555  #print "Studying directory ",subsubdirname," in directory ",subdirname
556  nsucc=subsubdir.n_successes
557  total_successes+=nsucc
558  weight=subsubdir.weight
559  total_weight+=weight
560  total_ndirs+=1
561  total_directory_successes+= float(nsucc)/weight
562 
563  if subsubdirname in present_subdirs:
564  this_dir_dict=present_subdirs[subsubdirname]
565  this_dir_dict["nsucc"]+=nsucc
566  this_dir_dict["weight"]+=weight
567  else:
568  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
569 
570  if total_ndirs == 0:
571  print "No directory of the category %s is present in the samples: skipping." %cat_name
572  continue
573 
574  average_success_rate=total_directory_successes/(total_ndirs)
575  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
576 
577  return aggr_pairs_info
578 
579 #-------------------------------------------------------------------------------
580 
581 def make_categories_summary(dir_dict,aggregation_rules):
582 
583  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
584 
585  #print aggr_pairs_info
586 
587  # Now Let's build the HTML
588 
589  html= '<div class="span-20 colborder">'
590  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
591 
592  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
593  #print cat_name,present_subdirs,total_weight,average_success_rate
594  html+='<div class="span-3 prepend-0 colborder">'
595  html+='<h3>%s</h3>'%cat_name
596  html+='<div><span class="alt">Avg. Success rate:</span></div>'
597  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
598  html+='</div>'
599  html+='<div class="span-9">'
600 
601  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
602  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
603  html+='<ul>'
604  for subdirname in sorted(present_subdirs.keys()):
605  this_dir_dict=present_subdirs[subdirname]
606  nsucc=this_dir_dict["nsucc"]
607  weight=this_dir_dict["weight"]
608  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
609  html+='</ul>'
610  html+='</div>'
611 
612  html+='<div class="span-6 last">'
613  html+=build_gauge(average_success_rate)
614  html+='</div>'
615 
616  html+='<hr>'
617  return html+'<br><a href="#top">Top...</a> </div><hr>'
618 
619  #-------------------------------------------------------------------------------
620 
621 def make_twiki_table(dir_dict,aggregation_rules):
622 
623  # decide the release
624  meta= dir_dict.items()[0][1].meta
625  releases=sorted([meta.release1,meta.release2])
626  latest_release=releases[1].split("-")[0]
627 
628 
629  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
630 
631  # Now Let's build the HTML
632 
633  html= '<div class="span-20 colborder">'
634  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
635  html+='<div>| Release | Comparison |'
636  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
637  html+=cat_name
638  html+=" | "
639  html+='</div>'
640 
641  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
642 
643  # Now add all the line with small gauges
644 
645  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
646  #print cat_name,present_subdirs,total_weight,average_success_rate
647  html+=build_gauge(average_success_rate,small=True,escaped=True)
648  html+=" | "
649 
650  html+='</div> <a href="#top">Top...</a>'
651  html+='<hr>'
652  return html+'</div>'
653 
654 #-------------------------------------------------------------------------------
655 
656 def get_pie_tooltip(directory):
657  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
658  return tooltip
659 
660 #-------------------------------------------------------------------------------
661 
662 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
663 
664  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
665 
666  script="""
667  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
668  <script type="text/javascript">
669  google.load("visualization", "1", {packages:["corechart"]});
670  google.setOnLoadCallback(drawChart);
671  function drawChart() {
672  var data = new google.visualization.DataTable();
673  data.addColumn('string', 'DQM Directory');
674  data.addColumn('number', 'Success Rate');
675  """
676  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
677  counter=0
678  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
679  #print subsystname,present_directories
680  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
681  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
682  counter+=1
683  script+="""
684  var chart = new google.visualization.BarChart(document.getElementById('%s'));
685  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
686  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
687  });
688  }
689  </script>
690  """%(name,40*counter,title)
691  return script
692 
693 
694 #-------------------------------------------------------------------------------
695 
696 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
697  """Create a table, with as rows the directories and as columns the samples.
698  Each box in the table will contain a pie chart linking to the directory.
699  """
700  #aggregation_rules={}
701  #aggregation_rules_twiki={}
702 
703  chdir(indir)
704  if os.path.isabs(indir):
705  title = basename(indir)
706  else:
707  title=indir
708  title=title.strip(".")
709  title=title.strip("/")
710 
711 
712  # Get the list of pickles
713  sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
714 
715  # Load directories, build a list of all first level subdirs
716  dir_unpicklers=[]
717  n_unpicklers=0
718  for sample_pkl in sample_pkls:
719  dir_unpickler=unpickler(sample_pkl)
720  dir_unpickler.start()
721  n_unpicklers+=1
722  dir_unpicklers.append(dir_unpickler)
723  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
724  n_unpicklers=0
725  for dir_unpickler in dir_unpicklers:
726  dir_unpickler.join()
727 
728  dir_dict={}
729 
730  # create a fake global directory
731  global_dir=Directory("global","")
732  for dir_unpickler in dir_unpicklers:
733  dir_unpickler.join()
734  directory=dir_unpickler.directory
735  #directory.prune("Run summary")
736  #directory.calcStats()
737  global_dir.meta=directory.meta
738  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
739  global_dir.subdirs.append(directory)
740 
741  global_dir.calcStats()
742 
743  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
744  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
745 
746  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
747  rel1=""
748  rel2=""
749  try:
750  rel1,rel2=title.split("VS")
751  except:
752  rel1=global_dir.meta.release1.split("-")[0]
753  rel2=global_dir.meta.release2.split("-")[0]
754  global_dir.meta.release1=rel1
755  global_dir.meta.release2=rel2
756 
757  # union of all subdirs names
758  all_subdirs=[]
759  for directory in dir_dict.values():
760  for subdir_name in directory.get_subdirs_names():
761  all_subdirs.append(subdir_name)
762  all_subdirs=sorted(list(set(all_subdirs)))
763 
764  #print " $$ all_subdirs: %s" %(all_subdirs)
765 
766  # Get The title
767  page_html+= '<div class="span-20">'+\
768  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
769  '</div>'+\
770  '<div class="span-1">'+\
771  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
772  '</div>'+\
773  '<hr>'
774  page_html+='<div class="span-24"><p></p></div>\n'*3
775 
776  # Get The summary section
777  page_html+= get_summary_section(global_dir,False)
778 
779  # Make the anchor sections
780  page_html+= '<div class="span-24">'
781  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
782  '<ul>'+\
783  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
784  '<li><a href="#categories">Categories</a></li>'+\
785  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
786  '<li><a href="#summary_table">Summary Table</a></li>'+\
787  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
788  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
789  '</ul>'+\
790  '</div><hr>'
791 
792 
793  # Make the CategoriesBar chart
794  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
795  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
796 
797  # Make the gauges per categories
798  page_html+=make_categories_summary(dir_dict,aggregation_rules)
799 
800  # Make the Directories chart
801  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
802  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
803 
804  # Barbarian vertical space. Suggestions are welcome
805  for i in xrange(2):
806  page_html+='<div class="span-24"><p></p></div>\n'
807 
808 
809  # Prepare the table
810  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
811 
812  for i in xrange(5):
813  page_html+='<div class="span-24"><p></p></div>\n'
814 
815  page_html+="""
816  <table border="1" >
817  <tr>
818  <td> </td>
819  """
820 
821  # First row with samples
822  page_html+="""
823  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
824 
825  sorted_samples=sorted(dir_dict.keys())
826  for sample in sorted_samples:
827  sample_nick=sample
828  ## For runs: put only the number after the _
829  #if "_" in sample:
830  #run_number=sample.split("_")[-1]
831  #if (not run_number.isalpha()) and len(run_number)>=6:
832  #sample_nick=run_number
833 
834 
835  page_html+="""
836  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
837  page_html+=" </tr>\n"
838 
839 
840  # FIRST ROW
841  # Now the summaries at the beginning of the table
842  page_html+="<tr>"
843  page_html+='<td style="background-color:white;"><div class="span-1">'
844 
845  page_html+='<b>Summary</b></div></td>'
846  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
847  for sample in sorted_samples:
848  col=dir_dict[sample]
849  # check if the directory was a top one or not
850  summary_page_name="RelMonSummary.html"
851  if col.name!="":
852  summary_page_name=hash_name(col.name, hashing_flag)+".html"
853  img_link=col.get_summary_chart_ajax(55,55)
854  page_html+='<td style="background-color:white;"><div class="span-1">'
855  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
856  page_html+="</tr>"
857 
858  # Now the content
859  for subdir_name in all_subdirs:
860 
861  page_html+=' <tr>\n'
862  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
863 
864  row_summary=Directory("row_summary","")
865  sample_counter=0
866  n_samples=len(sorted_samples)
867 
868  for sample in sorted_samples:
869  subdirs_dict=directory.get_subdirs_dict()
870  directory=dir_dict[sample]
871  dir_is_there=subdir_name in subdirs_dict
872  if dir_is_there:
873  row_summary.subdirs.append(subdirs_dict[subdir_name])
874 
875  # one first row for the summary!
876  row_summary.calcStats()
877  img_link=row_summary.get_summary_chart_ajax(55,55)
878  page_html+='<td style="background-color:white;"><div class="span-1">'
879  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
880 
881  for sample in sorted_samples:
882  sample_counter+=1
883 
884  directory=dir_dict[sample]
885  subdirs_dict=directory.get_subdirs_dict()
886 
887  # Check if the directory is the top one
888  if directory.name!="":
889  # We did not run on the topdir
890  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
891  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
892  if hashing_flag:
893  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
894  else:
895  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
896  else:
897  #print " ## summary_page: %s"%(directory.name+subdir_name)
898  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
899  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
900  dir_is_there=subdir_name in subdirs_dict
901 
902  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
903  img_tooltip="N/A"
904  if dir_is_there:
905  #row_summary.subdirs.append(subdirs_dict[subdir_name])
906  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
907  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
908 
909  page_html+='<td style="background-color:white;"><div class="span-1">'
910  if dir_is_there:
911  page_html+='<a href="%s">'%(summary_page)
912  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
913  if dir_is_there:
914  page_html+='</a>'
915  page_html+='</div></td>'
916 
917  page_html+=" </tr>\n"
918 
919 
920 
921  page_html+='</table> <a href="#top">Top...</a><hr>'
922 
923  page_html+=get_rank_section(global_dir)
924 
925  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
926 
927  page_html+=get_page_footer()
928  return page_html
929 
930 
931 #-----------UPDATES------
932 def hash_name(file_name, flag):
933  #print " HashFILE name: "+file_name
934  if flag: #if hashing flag is ON then return
935  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
936  else:
937  return file_name #return standart name
directories2html.build_gauge
def build_gauge(total_success_rate, minrate=.80, small=False, escaped=False)
Definition: directories2html.py:492
directories2html.hash_name
def hash_name(file_name, flag)
Definition: directories2html.py:932
directories2html.get_comparisons
def get_comparisons(category, directory)
Definition: directories2html.py:280
directories2html.get_subdirs_section
def get_subdirs_section(directory, hashing_flag)
Definition: directories2html.py:185
directories2html.get_pie_tooltip
def get_pie_tooltip(directory)
Definition: directories2html.py:656
directories2html.build_obj
def build_obj(run, sample, version, plot_path, tier)
Definition: directories2html.py:53
directories2html.get_aggr_pairs_info
def get_aggr_pairs_info(dir_dict, the_aggr_pairs=[])
Definition: directories2html.py:513
directories2html.make_summary_table
def make_summary_table(indir, aggregation_rules, aggregation_rules_twiki, hashing_flag, standalone_flag)
Definition: directories2html.py:696
directories2html.plot_size
def plot_size(h=250, w=200)
Definition: directories2html.py:40
directories2html.get_page_header
def get_page_header(directory=None, standalone=False, additional_header="")
Definition: directories2html.py:77
directories2html.get_title_section
def get_title_section(directory, hashing_flag, standalone, depth=2)
Definition: directories2html.py:121
directories2html.get_dir_stats
def get_dir_stats(directory)
Definition: directories2html.py:167
directories2html.make_barchart_summary
def make_barchart_summary(dir_dict, name="the_chart", title="DQM directory", the_aggr_pairs=[])
Definition: directories2html.py:662
directories2html.get_summary_section
def get_summary_section(directory, matrix_page=True)
Definition: directories2html.py:220
directories2html.directory2html
def directory2html(directory, hashing, standalone, depth=0)
Definition: directories2html.py:430
directories2html.make_twiki_table
def make_twiki_table(dir_dict, aggregation_rules)
Definition: directories2html.py:621
mps_monitormerge.items
list items
Definition: mps_monitormerge.py:27
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
ALCARECOTkAlBeamHalo_cff.filter
filter
Definition: ALCARECOTkAlBeamHalo_cff.py:27
directories2html.make_categories_summary
def make_categories_summary(dir_dict, aggregation_rules)
Definition: directories2html.py:581
directories2html.fairy_url_single
def fairy_url_single(run, sample, version, plot_path, tier, draw_opts="", h=250, w=200)
Definition: directories2html.py:67
directories2html.fairy_url
def fairy_url(run1, run2, sample1, sample2, version1, version2, plot_path, tier1, tier2, draw_opts="", h=250, w=200)
Definition: directories2html.py:57
directories2html.build_obj_addr
def build_obj_addr(run, sample, version, plot_path, tier)
Definition: directories2html.py:43
split
double split
Definition: MVATrainer.cc:139
directories2html.get_missing_objs_section
def get_missing_objs_section(directory)
Definition: directories2html.py:417
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
directories2html.get_rank_section
def get_rank_section(directory)
Definition: directories2html.py:369
Generated for CMSSW Reference Manual by doxygen 1.8.11