CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
directories2html.py
Go to the documentation of this file.
1 ################################################################################
2 # RelMon: a tool for automatic Release Comparison
3 # https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon
4 #
5 
6 #
7 #
8 # Danilo Piparo CERN - danilo.piparo@cern.ch
9 #
10 ################################################################################
11 
12 from os import chdir,getcwd,listdir,makedirs
13 from os.path import basename,join,exists
14 import cgi
15 
16 import sys
17 theargv=sys.argv
18 sys.argv=[]
19 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
20 sys.argv=theargv
21 
22 import os
23 if os.environ.has_key("RELMON_SA"):
24  from dirstructure import Comparison,Directory
25  from definitions import *
26  from utils import unpickler
27 else:
28  from Utilities.RelMon.dirstructure import Comparison,Directory
29  from Utilities.RelMon.definitions import *
30  from Utilities.RelMon.utils import unpickler
31 
32  import hashlib
33 #-------------------------------------------------------------------------------
34 
35 def encode_obj_url(url):
36  for old,new in url_encode_dict.items():
37  url=url.replace(old,new)
38  return url
39 
40 def plot_size(h=250,w=200):
41  return "w=%s;h=%s" %(h,w)
42 
43 def build_obj_addr(run,sample,version,plot_path,tier):
44  slash="/"
45  obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
46  obj_url+=plot_path
47  while obj_url.endswith(slash):
48  obj_url=obj_url[:-1]
49  while slash*2 in obj_url:
50  obj_url=obj_url.replace(slash*2,slash)
51  return obj_url
52 
53 def build_obj(run,sample,version,plot_path,tier):
54  obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
55  return encode_obj_url(obj_url)
56 
57 def fairy_url(run1,run2,sample1,sample2,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
58  fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
59  fairy_url+= build_obj(run1,sample1,version1,plot_path,tier1)
60  fairy_url+= build_obj(run2,sample2,version2,plot_path,tier2)
61  if len(draw_opts)>0:
62  fairy_url+="drawopts=%s;" %draw_opts
63  fairy_url+= plot_size(h,w)
64  return fairy_url
65 
66 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
67  fairy_url = "%s/%s/plotfairy/" %(server,base_url)
68  fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
69  fairy_url+= "?%s"%plot_size(h,w)
70  if len(draw_opts)>0:
71  fairy_url+="drawopts=%s;" %draw_opts
72  return fairy_url
73 
74 #-------------------------------------------------------------------------------
75 style_location="/cms-service-reldqm"
76 def get_page_header(directory=None, standalone=False, additional_header=""):
77  style_location="/cms-service-reldqm"
78  if standalone:
79  style_location = "http://cms-service-reldqm.web.cern.ch/" + style_location +"/"
80  javascripts=''
81  style=''
82  if directory!=None and len(directory.comparisons)>0:
83  meta=directory.meta
84  style='img.fail {border:1px solid #ff0000;}\n'+\
85  'img.succes {border:1px solid #00ff00;}\n'+\
86  'img.null {border:1px solid #ffff00;}\n'+\
87  'img.skiped {border:1px solid #7a7a7a;}\n'+\
88  'a.black_link:link {color: #333333}\n'+\
89  'a.black_link:hover {color: #737373}\n'+\
90  'a.black_link:visited {color: #333333}\n'+\
91  'a.black_link:active {color: #333333}\n'
92  javascripts=""
93 
94 
95  html='<html>'+\
96  '<head>'+\
97  '<title>RelMon Summary</title>'+\
98  '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
99  '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
100  '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
101  '<style type="text/css">'+\
102  '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
103  '%s'%style+\
104  '</style>'+\
105  '%s'%javascripts+\
106  '%s'%additional_header+\
107  '</head>'+\
108  '<body>'+\
109  '<div class="container">'
110 
111  return html
112 
113 #-------------------------------------------------------------------------------
114 
115 def get_page_footer():
116  return '</div></body></html>'
117 
118 #-------------------------------------------------------------------------------
119 
120 def get_title_section(directory, hashing_flag, standalone, depth=2):
121  if standalone:
122  cms_logo_url = "http://cms-service-reldqm.web.cern.ch/cms-service-reldqm/style/CMS.gif"
123  else:
124  cms_logo_url = "cms-service-reldqm/style/CMS.gif"
125  mother_name=basename(directory.mother_dir)
126  mother_file_name=""
127  if depth==1:
128  mother_file_name="../RelMonSummary.html"
129  if mother_name!="":
130  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
131  elif depth==2:
132  mother_file_name="RelMonSummary.html"
133  #mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
134  if mother_name!="":
135  mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
136  else:
137  if hashing_flag:
138  files = directory.mother_dir.split("/")
139  if len(files) != 1:
140  dir_name = files[-2]+files[-1] ##return the mother directory name only as the html file name by it
141  else:
142  dir_name = files[-1]
143  dir_name = directory.mother_dir
144  mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
145  else:
146  mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
147  mother_file_name=mother_file_name.strip("_")
148 
149  link_to_mother='<a href="%s">..</a>' %mother_file_name
150  html= '<div class="span-20">'+\
151  '<h1>%s</h1>'%directory.name+\
152  '</div>'+\
153  '<div class="span-1">'+\
154  '<h1>%s</h1>'%link_to_mother+\
155  '</div>'+\
156  '<div class="span-3 last">'+\
157  '<img src="%s" class="top right" width="54" hight="54">'%cms_logo_url+\
158  '</div>'+\
159  '<hr>'
160  if len(mother_name)>0:
161  html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
162  '<hr>'
163  return html
164 
165 #-------------------------------------------------------------------------------
166 def get_dir_stats(directory):
167  html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
168  html+='<ul>'
169  if directory.n_successes>0:
170  html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
171  if directory.n_nulls>0:
172  html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
173  if directory.n_fails>0:
174  html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
175  if directory.n_skiped>0:
176  html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
177  if directory.n_missing_objs>0:
178  html+='<li><span class="caps">Unpaired: %s</span></li>'%(directory.n_missing_objs)
179  html+='</ul>'
180  return html
181 
182 #-------------------------------------------------------------------------------
183 
184 def get_subdirs_section(directory, hashing_flag):
185  if len(directory.subdirs)==0:
186  return ""
187  html= '<div class="span-20 colborder">'
188  html+='<h2 class="alt">Sub-Directories</h2>'
189  # sort subdirs according to the number of fails and number of nulls and then alphabveticaly
190  # so reverse :)
191  sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
192  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
193  sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
194  for subdir in sorted_subdirs:
195  name=subdir.name
196  if hashing_flag:
197  link = "%s.html" %(hash_name(join(directory.full_path,name), hashing_flag)) #do hash with directory name + subdirname as single name hashing might get problems with same subdirs name in different parent dirs.
198  else:
199  link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
200  link=link.strip("_")
201  html+='<div class="span-4 prepend-2 colborder">'
202  html+='<h3>%s</h3>'%name
203  html+='</div>'
204 
205  html+='<div class="span-7">'
206  html+=get_dir_stats(subdir)
207  html+='</div>'
208 
209  html+='<div class="span-6 last">'
210  html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
211  html+='</div>'
212 
213  html+='<hr>'
214  return html+'</div>'
215 
216 
217 #-------------------------------------------------------------------------------
218 
219 def get_summary_section(directory,matrix_page=True):
220 
221  # Hack find the first comparison and fill test and threshold
222  # shall this be put in meta?
223  test_name=""
224  test_threshold=0
225  for comparison in directory.comparisons:
226  test_name=comparison.test_name
227  test_threshold=comparison.test_thr
228  break
229  if len(test_name)==0:
230  for subdir in directory.subdirs:
231  for comparison in subdir.comparisons:
232  test_name=comparison.test_name
233  test_threshold=comparison.test_thr
234  break
235  if len(test_name)!=0:break
236  if len(test_name)==0:
237  for subsubdir in subdir.subdirs:
238  for comparison in subsubdir.comparisons:
239  test_name=comparison.test_name
240  test_threshold=comparison.test_thr
241  break
242  if len(test_name)!=0:break
243  if len(test_name)==0:
244  for subsubsubdir in subsubdir.subdirs:
245  for comparison in subsubsubdir.comparisons:
246  test_name=comparison.test_name
247  test_threshold=comparison.test_thr
248  break
249  if len(test_name)!=0:break
250 
251 
252  meta=directory.meta
253 
254  html= '<div class="span-6">'+\
255  '<h3>Summary</h3>'
256  html+=get_dir_stats(directory)
257  html+='<a href="%s/%s">To the DQM GUI...</a>' %(server,base_url)
258  html+='</div>'
259 
260  html+='<div class="span-7 colborder">'+\
261  '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
262  '</div>'+\
263  '<div class="span-9 last">'
264  if matrix_page:
265  html+='<h3>Sample:</h3>'+\
266  '<p class="caps">%s</p>'%meta.sample1+\
267  '<h3>Run1 and Run2:</h3>'+\
268  '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
269  html+='<h3>Releases:</h3>'+\
270  '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
271  '<h3>Statistical Test (Pvalue threshold):</h3>'+\
272  '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
273  '</div>'+\
274  '<hr>'
275  return html
276 
277 #-------------------------------------------------------------------------------
278 
279 def get_comparisons(category,directory):
280  """Prepare the comparisons between histograms and organise them in the page.
281  Moreover create separate pages with the overlay and the single plots.
282  """
283  counter=1
284  tot_counter=1
285 
286  # get the right ones
287  comparisons= filter (lambda comp: comp.status == cat_states[category] , directory.comparisons)
288  n_comparisons=len(comparisons)
289 
290  is_reverse=True
291  if category == FAIL:
292  is_reverse=False
293  comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
294 
295 
296  dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
297  html_comparisons=""
298  for comparison in comparisons:
299  plot_name=comparison.img_name
300  if "http://" not in plot_name:
301  plot_name= basename(comparison.img_name)
302  class_type="colborder"
303  if counter==3 or tot_counter==n_comparisons:
304  class_type=" colborder last"
305  comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
306 
307 
308  if directory.do_pngs:
309  png_link=comparison.img_name
310  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
311  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
312  '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
313  else:
314  big_fairy=fairy_url(directory.meta.run1,
315  directory.meta.run2,
316  directory.meta.sample1,
317  directory.meta.sample2,
318  directory.meta.release1,
319  directory.meta.release2,
320  comp_abs_path,
321  directory.meta.tier1,
322  directory.meta.tier2,
323  "",600,600)
324  small_fairy=fairy_url(directory.meta.run1,
325  directory.meta.run2,
326  directory.meta.sample1,
327  directory.meta.sample2,
328  directory.meta.release1,
329  directory.meta.release2,
330  comp_abs_path,
331  directory.meta.tier1,
332  directory.meta.tier2)
333 
334  single_fairy1=fairy_url_single(directory.meta.run1,
335  directory.meta.sample1,
336  directory.meta.release1,
337  comp_abs_path,
338  directory.meta.tier1,
339  "",500,500)
340  single_fairy2=fairy_url_single(directory.meta.run2,
341  directory.meta.sample2,
342  directory.meta.release2,
343  comp_abs_path,
344  directory.meta.tier2,
345  "",500,500)
346 
347  html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
348  '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
349  '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
350  '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
351  '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
352 
353  if counter==3:
354  html_comparisons+="<hr>"
355  counter=0
356  counter+=1
357  tot_counter+=1
358 
359  if len(html_comparisons)!=0:
360  html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
361  html+=html_comparisons
362  html+='<hr>'
363  return html
364  return ""
365 
366 #-------------------------------------------------------------------------------
367 
368 def get_rank_section(directory):
369 # do Rank histo png
370  imgname="RankSummary.png"
371  gStyle.SetPadTickY(0)
372  c=TCanvas("ranks","ranks",500,400)
373  #gStyle.SetOptStat(0)
374  c.cd()
375 
376  h=directory.rank_histo
377  rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
378  rank_histof.SetLineWidth(2)
379  for i in xrange(0,h.GetNbinsX()+1):
380  rank_histof.SetBinContent(i,h.GetBinContent(i))
381  h.SetTitle("Ranks Summary;Rank;Frequency")
382  h.Draw("Hist")
383  c.Update()
384  rank_histof.ComputeIntegral()
385  integral = rank_histof.GetIntegral()
386  rank_histof.SetContent(integral)
387 
388  rightmax = 1.1*rank_histof.GetMaximum()
389  scale = gPad.GetUymax()/rightmax
390  rank_histof.SetLineColor(kRed)
391  rank_histof.Scale(scale)
392  rank_histof.Draw("same")
393 
394  #draw an axis on the right side
395  axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
396  axis.SetTitle("Cumulative")
397  axis.SetTitleColor(kRed)
398  axis.SetLineColor(kRed)
399  axis.SetLabelColor(kRed)
400  axis.Draw()
401 
402  rank_histof.Draw("Same");
403 
404 
405  c.Print(imgname)
406 
407  page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
408  page_html+='<div class="span-19"><img src="%s"></div>' %imgname
409  page_html+='</div> <a href="#top">Top...</a><hr>'
410 
411  return page_html
412 
413 #-------------------------------------------------------------------------------
414 
415 #-------------------------------------------------------------------------------
416 def get_missing_objs_section(directory):
417  """Method to get missing objects from directory: in case histogram/directory was in one ROOT file but not in other
418  """
419  page_html = "Unpaired in %s</br>"%(directory.filename1)
420  for elem in directory.different_histograms['file1']:
421  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file1'][elem])
422  page_html +="</br>"
423  page_html += "Unpaired in %s</br>"%(directory.filename2)
424  for elem in directory.different_histograms['file2']:
425  page_html += "name: %s type:%s </br>"%(elem,directory.different_histograms['file2'][elem])
426  return page_html
427 #-------------------------------------------------------------------------------
428 
429 def directory2html(directory, hashing, standalone, depth=0):
430  """Converts a directory tree into html pages, very nice ones.
431  """
432  #print "d2html: depth", str(depth)," dir ",directory.name
433  depth+=1
434  #old_cwd=getcwd()
435  #if not exists(directory.name) and len(directory.name)>0:
436  #makedirs(directory.name)
437 
438  #if len(directory.name)>0:
439  #chdir(directory.name)
440 
441  for subdir in directory.subdirs:
442  directory2html(subdir,hashing,standalone, depth)
443 
444  page_html=get_page_header(directory, standalone)+\
445  get_title_section(directory,hashing, standalone, depth)+\
446  get_summary_section(directory)+\
447  get_subdirs_section(directory, hashing)
448 
449  for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
450  (directory.n_comp_nulls >0,NULL ),
451 
452  (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
453  (directory.n_comp_skiped >0,SKIPED)):
454  if do_cat:
455  page_html+=get_comparisons(cat,directory)
456 
457  if (len(directory.different_histograms['file1']) >0) or (len(directory.different_histograms['file2']) >0):
458  page_html += get_missing_objs_section(directory)
459 
460  # Distribution of ranks
461 
462  if depth==1:
463  page_html+=get_rank_section(directory)
464 
465 
466  page_html+=get_page_footer()
467 
468  page_name=directory.name
469 
470  if len(page_name)==0:
471  page_name="RelMonSummary"
472  if hashing:
473  if page_name != "RelMonSummary":
474  #print " ## oFile path: %s"%(join(directory.full_path))
475  #print " ## oFile hash: %s"%(hash_name(join(directory.full_path), hashing))
476  ofilename = "%s.html" %(hash_name(join(directory.full_path), hashing)) #as links is generated: parentdi+subdir; we split and get the last parent dir
477  else:
478  ofilename = "RelMonSummary.html"
479  else:
480  ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
481  ofilename=ofilename.strip("_")
482  #print "Writing on %s" %ofilename
483  ofile=open(ofilename,"w")
484  ofile.write(page_html)
485  ofile.close()
486 
487  #chdir(old_cwd)
488 
489 #-------------------------------------------------------------------------------
490 
491 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
492  total_success_rate_scaled=(total_success_rate-minrate)
493  total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
494  if total_success_rate_scaled_repr<0:
495  total_success_rate_scaled_repr=0
496  size_s="200x100"
497  if small:
498  size_s="40x30"
499  #print "Total success rate %2.2f and scaled %2.2f "%(total_success_rate,total_success_rate_scaled)
500  gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
501  gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
502  if not small:
503  gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
504  gauge_link+="&chma=10,10,10,0"
505  img_tag= '<img src="%s">'%gauge_link
506  if escaped:
507  img_tag=cgi.escape(img_tag)
508  return img_tag
509 
510 #-------------------------------------------------------------------------------
511 
512 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
513  # Let's make a summary for an overview in categories act on the global dir
514  aggr_pairs_info=[]#(name,{directories names:{nsucc: nsucc,weight:weight}},weight,success_rate)
515 
516  list_of_names=[]
517  if the_aggr_pairs==[]:
518  for samplename,sampledir in dir_dict.items():
519  for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
520  if not subsysdirname in list_of_names:
521  list_of_names.append(subsysdirname)
522  the_aggr_pairs.append((subsysdirname,[subsysdirname]))
523 
524  #print the_aggr_pairs
525  for cat_name, subdir_list in the_aggr_pairs:
526  total_successes=0.
527  total_directory_successes=0
528  total_weight=0.
529  present_subdirs={}
530  total_ndirs=0
531  # Loop on samples
532  for dirname, sampledir in dir_dict.items():
533  # Loop on directories
534  for subdirname,subdir in sampledir.get_subdirs_dict().items():
535  if subdirname in subdir_list:
536  nsucc=subdir.n_successes
537  total_successes+=nsucc
538  weight=subdir.weight
539  total_weight+=weight
540  total_ndirs+=1
541 
542  total_directory_successes+= float(nsucc)/weight
543  if present_subdirs.has_key(subdirname):
544  this_dir_dict=present_subdirs[subdirname]
545  this_dir_dict["nsucc"]+=nsucc
546  this_dir_dict["weight"]+=weight
547  else:
548  present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
549  # Make it usable also for subdirectories
550  for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
551  for pathname in filter(lambda name:"/" in name,subdir_list):
552  selected_subdirname,selected_subsubdirname = pathname.split("/")
553  if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
554  #print "Studying directory ",subsubdirname," in directory ",subdirname
555  nsucc=subsubdir.n_successes
556  total_successes+=nsucc
557  weight=subsubdir.weight
558  total_weight+=weight
559  total_ndirs+=1
560  total_directory_successes+= float(nsucc)/weight
561 
562  if present_subdirs.has_key(subsubdirname):
563  this_dir_dict=present_subdirs[subsubdirname]
564  this_dir_dict["nsucc"]+=nsucc
565  this_dir_dict["weight"]+=weight
566  else:
567  present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
568 
569  if total_ndirs == 0:
570  print "No directory of the category %s is present in the samples: skipping." %cat_name
571  continue
572 
573  average_success_rate=total_directory_successes/(total_ndirs)
574  aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
575 
576  return aggr_pairs_info
577 
578 #-------------------------------------------------------------------------------
579 
580 def make_categories_summary(dir_dict,aggregation_rules):
581 
582  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
583 
584  #print aggr_pairs_info
585 
586  # Now Let's build the HTML
587 
588  html= '<div class="span-20 colborder">'
589  html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
590 
591  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
592  #print cat_name,present_subdirs,total_weight,average_success_rate
593  html+='<div class="span-3 prepend-0 colborder">'
594  html+='<h3>%s</h3>'%cat_name
595  html+='<div><span class="alt">Avg. Success rate:</span></div>'
596  html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
597  html+='</div>'
598  html+='<div class="span-9">'
599 
600  html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
601  html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
602  html+='<ul>'
603  for subdirname in sorted(present_subdirs.keys()):
604  this_dir_dict=present_subdirs[subdirname]
605  nsucc=this_dir_dict["nsucc"]
606  weight=this_dir_dict["weight"]
607  html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
608  html+='</ul>'
609  html+='</div>'
610 
611  html+='<div class="span-6 last">'
612  html+=build_gauge(average_success_rate)
613  html+='</div>'
614 
615  html+='<hr>'
616  return html+'<br><a href="#top">Top...</a> </div><hr>'
617 
618  #-------------------------------------------------------------------------------
619 
620 def make_twiki_table(dir_dict,aggregation_rules):
621 
622  # decide the release
623  meta= dir_dict.items()[0][1].meta
624  releases=sorted([meta.release1,meta.release2])
625  latest_release=releases[1].split("-")[0]
626 
627 
628  aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
629 
630  # Now Let's build the HTML
631 
632  html= '<div class="span-20 colborder">'
633  html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
634  html+='<div>| Release | Comparison |'
635  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
636  html+=cat_name
637  html+=" | "
638  html+='</div>'
639 
640  html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
641 
642  # Now add all the line with small gauges
643 
644  for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
645  #print cat_name,present_subdirs,total_weight,average_success_rate
646  html+=build_gauge(average_success_rate,small=True,escaped=True)
647  html+=" | "
648 
649  html+='</div> <a href="#top">Top...</a>'
650  html+='<hr>'
651  return html+'</div>'
652 
653 #-------------------------------------------------------------------------------
654 
655 def get_pie_tooltip(directory):
656  tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
657  return tooltip
658 
659 #-------------------------------------------------------------------------------
660 
661 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
662 
663  aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
664 
665  script="""
666  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
667  <script type="text/javascript">
668  google.load("visualization", "1", {packages:["corechart"]});
669  google.setOnLoadCallback(drawChart);
670  function drawChart() {
671  var data = new google.visualization.DataTable();
672  data.addColumn('string', 'DQM Directory');
673  data.addColumn('number', 'Success Rate');
674  """
675  script+="data.addRows(%i);\n"%len(aggr_pairs_info)
676  counter=0
677  for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
678  #print subsystname,present_directories
679  script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
680  script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
681  counter+=1
682  script+="""
683  var chart = new google.visualization.BarChart(document.getElementById('%s'));
684  chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
685  vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
686  });
687  }
688  </script>
689  """%(name,40*counter,title)
690  return script
691 
692 
693 #-------------------------------------------------------------------------------
694 
695 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag, standalone_flag):
696  """Create a table, with as rows the directories and as columns the samples.
697  Each box in the table will contain a pie chart linking to the directory.
698  """
699  #aggregation_rules={}
700  #aggregation_rules_twiki={}
701 
702  chdir(indir)
703  if os.path.isabs(indir):
704  title = basename(indir)
705  else:
706  title=indir
707  title=title.strip(".")
708  title=title.strip("/")
709 
710 
711  # Get the list of pickles
712  sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
713 
714  # Load directories, build a list of all first level subdirs
715  dir_unpicklers=[]
716  n_unpicklers=0
717  for sample_pkl in sample_pkls:
718  dir_unpickler=unpickler(sample_pkl)
719  dir_unpickler.start()
720  n_unpicklers+=1
721  dir_unpicklers.append(dir_unpickler)
722  if n_unpicklers>=1: #pickleing is very expensive. Do not overload cpu
723  n_unpicklers=0
724  for dir_unpickler in dir_unpicklers:
725  dir_unpickler.join()
726 
727  dir_dict={}
728 
729  # create a fake global directory
730  global_dir=Directory("global","")
731  for dir_unpickler in dir_unpicklers:
732  dir_unpickler.join()
733  directory=dir_unpickler.directory
734  #directory.prune("Run summary")
735  #directory.calcStats()
736  global_dir.meta=directory.meta
737  dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
738  global_dir.subdirs.append(directory)
739 
740  global_dir.calcStats()
741 
742  directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
743  categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
744 
745  page_html = get_page_header(standalone=standalone_flag, additional_header=directories_barchart+categories_barchart)
746  rel1=""
747  rel2=""
748  try:
749  rel1,rel2=title.split("VS")
750  except:
751  rel1=global_dir.meta.release1.split("-")[0]
752  rel2=global_dir.meta.release2.split("-")[0]
753  global_dir.meta.release1=rel1
754  global_dir.meta.release2=rel2
755 
756  # union of all subdirs names
757  all_subdirs=[]
758  for directory in dir_dict.values():
759  for subdir_name in directory.get_subdirs_names():
760  all_subdirs.append(subdir_name)
761  all_subdirs=sorted(list(set(all_subdirs)))
762 
763  #print " $$ all_subdirs: %s" %(all_subdirs)
764 
765  # Get The title
766  page_html+= '<div class="span-20">'+\
767  '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
768  '</div>'+\
769  '<div class="span-1">'+\
770  '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
771  '</div>'+\
772  '<hr>'
773  page_html+='<div class="span-24"><p></p></div>\n'*3
774 
775  # Get The summary section
776  page_html+= get_summary_section(global_dir,False)
777 
778  # Make the anchor sections
779  page_html+= '<div class="span-24">'
780  page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
781  '<ul>'+\
782  '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
783  '<li><a href="#categories">Categories</a></li>'+\
784  '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
785  '<li><a href="#summary_table">Summary Table</a></li>'+\
786  '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
787  '<li><a href="#twiki_table">Twiki Table</a></li>'+\
788  '</ul>'+\
789  '</div><hr>'
790 
791 
792  # Make the CategoriesBar chart
793  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
794  page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
795 
796  # Make the gauges per categories
797  page_html+=make_categories_summary(dir_dict,aggregation_rules)
798 
799  # Make the Directories chart
800  page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
801  page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
802 
803  # Barbarian vertical space. Suggestions are welcome
804  for i in xrange(2):
805  page_html+='<div class="span-24"><p></p></div>\n'
806 
807 
808  # Prepare the table
809  page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
810 
811  for i in xrange(5):
812  page_html+='<div class="span-24"><p></p></div>\n'
813 
814  page_html+="""
815  <table border="1" >
816  <tr>
817  <td> </td>
818  """
819 
820  # First row with samples
821  page_html+="""
822  <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
823 
824  sorted_samples=sorted(dir_dict.keys())
825  for sample in sorted_samples:
826  sample_nick=sample
827  ## For runs: put only the number after the _
828  #if "_" in sample:
829  #run_number=sample.split("_")[-1]
830  #if (not run_number.isalpha()) and len(run_number)>=6:
831  #sample_nick=run_number
832 
833 
834  page_html+="""
835  <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
836  page_html+=" </tr>\n"
837 
838 
839  # FIRST ROW
840  # Now the summaries at the beginning of the table
841  page_html+="<tr>"
842  page_html+='<td style="background-color:white;"><div class="span-1">'
843 
844  page_html+='<b>Summary</b></div></td>'
845  page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
846  for sample in sorted_samples:
847  col=dir_dict[sample]
848  # check if the directory was a top one or not
849  summary_page_name="RelMonSummary.html"
850  if col.name!="":
851  summary_page_name=hash_name(col.name, hashing_flag)+".html"
852  img_link=col.get_summary_chart_ajax(55,55)
853  page_html+='<td style="background-color:white;"><div class="span-1">'
854  page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
855  page_html+="</tr>"
856 
857  # Now the content
858  for subdir_name in all_subdirs:
859 
860  page_html+=' <tr>\n'
861  page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
862 
863  row_summary=Directory("row_summary","")
864  sample_counter=0
865  n_samples=len(sorted_samples)
866 
867  for sample in sorted_samples:
868  subdirs_dict=directory.get_subdirs_dict()
869  directory=dir_dict[sample]
870  dir_is_there=subdirs_dict.has_key(subdir_name)
871  if dir_is_there:
872  row_summary.subdirs.append(subdirs_dict[subdir_name])
873 
874  # one first row for the summary!
875  row_summary.calcStats()
876  img_link=row_summary.get_summary_chart_ajax(55,55)
877  page_html+='<td style="background-color:white;"><div class="span-1">'
878  page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
879 
880  for sample in sorted_samples:
881  sample_counter+=1
882 
883  directory=dir_dict[sample]
884  subdirs_dict=directory.get_subdirs_dict()
885 
886  # Check if the directory is the top one
887  if directory.name!="":
888  # We did not run on the topdir
889  #print " ## summary_page: %s"%(directory.name+"_"+subdir_name)
890  #print " ## summary_page hash: %s" %(hash_name(directory.name+"/"+subdir_name,hashing_flag))
891  if hashing_flag:
892  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"/"+subdir_name,hashing_flag)))
893  else:
894  summary_page=join(sample,"%s.html"%(hash_name(directory.name+"_"+subdir_name,hashing_flag)))
895  else:
896  #print " ## summary_page: %s"%(directory.name+subdir_name)
897  #print " ## summary_page hash: %s" %(hash_name(directory.name+subdir_name,hashing_flag))
898  summary_page=join(sample,"%s.html"%(hash_name(directory.name+subdir_name,hashing_flag)))
899  dir_is_there=subdirs_dict.has_key(subdir_name)
900 
901  img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
902  img_tooltip="N/A"
903  if dir_is_there:
904  #row_summary.subdirs.append(subdirs_dict[subdir_name])
905  img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
906  img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
907 
908  page_html+='<td style="background-color:white;"><div class="span-1">'
909  if dir_is_there:
910  page_html+='<a href="%s">'%(summary_page)
911  page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
912  if dir_is_there:
913  page_html+='</a>'
914  page_html+='</div></td>'
915 
916  page_html+=" </tr>\n"
917 
918 
919 
920  page_html+='</table> <a href="#top">Top...</a><hr>'
921 
922  page_html+=get_rank_section(global_dir)
923 
924  page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
925 
926  page_html+=get_page_footer()
927  return page_html
928 
929 
930 #-----------UPDATES------
931 def hash_name(file_name, flag):
932  #print " HashFILE name: "+file_name
933  if flag: #if hashing flag is ON then return
934  return hashlib.md5(file_name).hexdigest()[:10] #md5 hashed file name with length 10
935  else:
936  return file_name #return standart name
alcazmumu_cfi.filter
tuple filter
Definition: alcazmumu_cfi.py:34
join
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
split
double split
Definition: MVATrainer.cc:139
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
Generated for CMSSW Reference Manual by doxygen 1.8.5