00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015 from os import chdir,getcwd,listdir,makedirs
00016 from os.path import basename,join,exists
00017 import cgi
00018
00019 import sys
00020 theargv=sys.argv
00021 sys.argv=[]
00022 from ROOT import TCanvas,gStyle,TH1F,TGaxis,gPad,kRed
00023 sys.argv=theargv
00024
00025 import os
00026 if os.environ.has_key("RELMON_SA"):
00027 from dirstructure import Comparison,Directory
00028 from definitions import *
00029 from utils import unpickler
00030 else:
00031 from Utilities.RelMon.dirstructure import Comparison,Directory
00032 from Utilities.RelMon.definitions import *
00033 from Utilities.RelMon.utils import unpickler
00034
00035 import hashlib
00036
00037
00038 def encode_obj_url(url):
00039 for old,new in url_encode_dict.items():
00040 url=url.replace(old,new)
00041 return url
00042
00043 def plot_size(h=250,w=200):
00044 return "w=%s;h=%s" %(h,w)
00045
00046 def build_obj_addr(run,sample,version,plot_path,tier):
00047 slash="/"
00048 obj_url="archive/%s/%s/%s/%s/" %(run,sample,version,tier)
00049 obj_url+=plot_path
00050 while obj_url.endswith(slash):
00051 obj_url=obj_url[:-1]
00052 while slash*2 in obj_url:
00053 obj_url=obj_url.replace(slash*2,slash)
00054 return obj_url
00055
00056 def build_obj(run,sample,version,plot_path,tier):
00057 obj_url="obj=%s;" %build_obj_addr(run,sample,version,plot_path,tier)
00058 return encode_obj_url(obj_url)
00059
00060 def fairy_url(run,sample,version1,version2,plot_path,tier1,tier2,draw_opts="",h=250,w=200):
00061 fairy_url = "%s/%s/plotfairy/overlay?" %(server,base_url)
00062 fairy_url+= build_obj(run,sample,version1,plot_path,tier1)
00063 fairy_url+= build_obj(run,sample,version2,plot_path,tier2)
00064 if len(draw_opts)>0:
00065 fairy_url+="drawopts=%s;" %draw_opts
00066 fairy_url+= plot_size(h,w)
00067 return fairy_url
00068
00069 def fairy_url_single(run,sample,version,plot_path,tier,draw_opts="",h=250,w=200):
00070 fairy_url = "%s/%s/plotfairy/" %(server,base_url)
00071 fairy_url+=build_obj_addr(run,sample,version,plot_path,tier)
00072 fairy_url+= "?%s"%plot_size(h,w)
00073 if len(draw_opts)>0:
00074 fairy_url+="drawopts=%s;" %draw_opts
00075 return fairy_url
00076
00077
00078 style_location="/cms-service-reldqm"
00079 def get_page_header(directory=None,additional_header=""):
00080 javascripts=''
00081 style=''
00082 if directory!=None and len(directory.comparisons)>0:
00083 meta=directory.meta
00084 style='img.fail {border:1px solid #ff0000;}\n'+\
00085 'img.succes {border:1px solid #00ff00;}\n'+\
00086 'img.null {border:1px solid #ffff00;}\n'+\
00087 'img.skiped {border:1px solid #7a7a7a;}\n'+\
00088 'a.black_link:link {color: #333333}\n'+\
00089 'a.black_link:hover {color: #737373}\n'+\
00090 'a.black_link:visited {color: #333333}\n'+\
00091 'a.black_link:active {color: #333333}\n'
00092 javascripts=""
00093
00094
00095 html='<html>'+\
00096 '<head>'+\
00097 '<title>RelMon Summary</title>'+\
00098 '<link rel="stylesheet" href="%s/style/blueprint/screen.css" type="text/css" media="screen, projection">'%style_location+\
00099 '<link rel="stylesheet" href="%s/style/blueprint/print.css" type="text/css" media="print">'%style_location+\
00100 '<link rel="stylesheet" href="%s/style/blueprint/plugins/fancy-type/screen.css" type="text/css" media="screen, projection">'%style_location+\
00101 '<style type="text/css">'+\
00102 '.rotation {display: block;-webkit-transform: rotate(-90deg);-moz-transform: rotate(-90deg); }'+\
00103 '%s'%style+\
00104 '</style>'+\
00105 '%s'%javascripts+\
00106 '%s'%additional_header+\
00107 '</head>'+\
00108 '<body>'+\
00109 '<div class="container">'
00110
00111 return html
00112
00113
00114
00115 def get_page_footer():
00116 return '</div></body></html>'
00117
00118
00119
00120 def get_title_section(directory, hashing_flag, depth=2):
00121 mother_name=basename(directory.mother_dir)
00122 mother_file_name=""
00123 if depth==1:
00124 mother_file_name="../RelMonSummary.html"
00125 if mother_name!="":
00126 mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
00127 elif depth==2:
00128
00129 mother_file_name="%s.html" %(hash_name("RelMonSummary", hashing_flag))
00130 if mother_name!="":
00131 mother_file_name="%s.html" %(hash_name(mother_name, hashing_flag))
00132 else:
00133 if hashing_flag:
00134 files = directory.mother_dir.split("/")
00135 if len(files) != 1:
00136 dir_name = files[-2]
00137 else:
00138 dir_name = files[-1]
00139 mother_file_name="%s.html" %(hash_name(dir_name, hashing_flag))
00140 else:
00141 mother_file_name="%s.html" %directory.mother_dir.replace("/","_")
00142 mother_file_name=mother_file_name.strip("_")
00143
00144 link_to_mother='<a href="%s">..</a>' %mother_file_name
00145 html= '<div class="span-20">'+\
00146 '<h1>%s</h1>'%directory.name+\
00147 '</div>'+\
00148 '<div class="span-1">'+\
00149 '<h1>%s</h1>'%link_to_mother+\
00150 '</div>'+\
00151 '<div class="span-3 last">'+\
00152 '<img src="cms-service-reldqm/style/CMS.gif" class="top right" width="54" hight="54">'+\
00153 '</div>'+\
00154 '<hr>'
00155 if len(mother_name)>0:
00156 html+='<h2 class="alt">%s</h2>'% directory.mother_dir+\
00157 '<hr>'
00158 return html
00159
00160
00161 def get_dir_stats(directory):
00162 html='<p><span class="caps alt">%s comparisons:</span></p>'%directory.weight
00163 html+='<ul>'
00164 if directory.n_successes>0:
00165 html+='<li><span class="caps">Success: %.1f%% (%s)</span></li>'%(directory.get_success_rate(),directory.n_successes)
00166 if directory.n_nulls>0:
00167 html+='<li><span class="caps">Null: %.1f%% (%s)</span></li>'%(directory.get_null_rate(),directory.n_nulls)
00168 if directory.n_fails>0:
00169 html+='<li><span class="caps">Fail: %.1f%% (%s)</span></li>'%(directory.get_fail_rate(),directory.n_fails)
00170 if directory.n_skiped>0:
00171 html+='<li><span class="caps">Skipped: %.1f%% (%s)</span></li>'%(directory.get_skiped_rate(),directory.n_skiped)
00172 html+='</ul>'
00173 return html
00174
00175
00176
00177 def get_subdirs_section(directory, hashing_flag):
00178 if len(directory.subdirs)==0:
00179 return ""
00180 html= '<div class="span-20 colborder">'
00181 html+='<h2 class="alt">Sub-Directories</h2>'
00182
00183
00184 sorted_subdirs= sorted(directory.subdirs, key= lambda subdir: subdir.name)
00185 sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_nulls, reverse=True)
00186 sorted_subdirs= sorted(sorted_subdirs, key= lambda subdir: subdir.n_fails, reverse=True)
00187 for subdir in sorted_subdirs:
00188 name=subdir.name
00189 if hashing_flag:
00190 link = "%s.html" %(hash_name(name, hashing_flag))
00191 else:
00192 link="%s_%s_%s.html" %(directory.mother_dir.replace("/","_"),directory.name.replace("/","_"),name)
00193 link=link.strip("_")
00194 html+='<div class="span-4 prepend-2 colborder">'
00195 html+='<h3>%s</h3>'%name
00196 html+='</div>'
00197
00198 html+='<div class="span-7">'
00199 html+=get_dir_stats(subdir)
00200 html+='</div>'
00201
00202 html+='<div class="span-6 last">'
00203 html+='<a href="%s"><img src="%s" class="top right"></a>'%(link,subdir.get_summary_chart_ajax(150,100))
00204 html+='</div>'
00205
00206 html+='<hr>'
00207 return html+'</div>'
00208
00209
00210
00211
00212 def get_summary_section(directory,matrix_page=True):
00213
00214
00215
00216 test_name=""
00217 test_threshold=0
00218 for comparison in directory.comparisons:
00219 test_name=comparison.test_name
00220 test_threshold=comparison.test_thr
00221 break
00222 if len(test_name)==0:
00223 for subdir in directory.subdirs:
00224 for comparison in subdir.comparisons:
00225 test_name=comparison.test_name
00226 test_threshold=comparison.test_thr
00227 break
00228 if len(test_name)!=0:break
00229 if len(test_name)==0:
00230 for subsubdir in subdir.subdirs:
00231 for comparison in subsubdir.comparisons:
00232 test_name=comparison.test_name
00233 test_threshold=comparison.test_thr
00234 break
00235 if len(test_name)!=0:break
00236 if len(test_name)==0:
00237 for subsubsubdir in subsubdir.subdirs:
00238 for comparison in subsubsubdir.comparisons:
00239 test_name=comparison.test_name
00240 test_threshold=comparison.test_thr
00241 break
00242 if len(test_name)!=0:break
00243
00244
00245 meta=directory.meta
00246
00247 html= '<div class="span-6">'+\
00248 '<h3>Summary</h3>'
00249 html+=get_dir_stats(directory)
00250 html+='<a href="%s/%s">To the DQM GUI...</a>' %(server,base_url)
00251 html+='</div>'
00252
00253 html+='<div class="span-7 colborder">'+\
00254 '<img src="%s" class="top right">'%directory.get_summary_chart_ajax(200,200)+\
00255 '</div>'+\
00256 '<div class="span-9 last">'
00257 if matrix_page:
00258 html+='<h3>Sample:</h3>'+\
00259 '<p class="caps">%s</p>'%meta.sample+\
00260 '<h3>Run1 and Run2:</h3>'+\
00261 '<p class="caps">%s - %s</p>'%(meta.run1,meta.run2)
00262 html+='<h3>Releases:</h3>'+\
00263 '<ul><li><p>%s</p></li><li><p>%s</p></li></ul>'%(meta.release1,meta.release2)+\
00264 '<h3>Statistical Test (Pvalue threshold):</h3>'+\
00265 '<ul><li><p class="caps">%s (%s)</p></li></ul>'%(test_name,test_threshold)+\
00266 '</div>'+\
00267 '<hr>'
00268 return html
00269
00270
00271
00272 def get_comparisons(category,directory):
00273 """Prepare the comparisons between histograms and organise them in the page.
00274 Moreover create separate pages with the overlay and the single plots.
00275 """
00276 counter=1
00277 tot_counter=1
00278
00279
00280 comparisons= filter (lambda comp: comp.status == cat_states[category] , directory.comparisons)
00281 n_comparisons=len(comparisons)
00282
00283 is_reverse=True
00284 if category == FAIL:
00285 is_reverse=False
00286 comparisons=sorted(comparisons, key=lambda comp:comp.rank, reverse=is_reverse)
00287
00288
00289 dir_abs_path="%s/%s/" %(directory.mother_dir,directory.name)
00290 html_comparisons=""
00291 for comparison in comparisons:
00292 plot_name=comparison.img_name
00293 if "http://" not in plot_name:
00294 plot_name= basename(comparison.img_name)
00295 class_type="colborder"
00296 if counter==3 or tot_counter==n_comparisons:
00297 class_type=" colborder last"
00298 comp_abs_path="%s/%s" %(dir_abs_path,comparison.name)
00299
00300
00301 if directory.do_pngs:
00302 png_link=comparison.img_name
00303 html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
00304 '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
00305 '<a href="%s"><img src="%s" width="250" height="200" class="top center %s"></a></div>'%(png_link,png_link,cat_classes[category])
00306 else:
00307 big_fairy=fairy_url(directory.meta.run1,
00308 directory.meta.sample,
00309 directory.meta.release1,
00310 directory.meta.release2,
00311 comp_abs_path,
00312 directory.meta.tier1,
00313 directory.meta.tier2,
00314 "",600,600)
00315 small_fairy=fairy_url(directory.meta.run1,
00316 directory.meta.sample,
00317 directory.meta.release1,
00318 directory.meta.release2,
00319 comp_abs_path,
00320 directory.meta.tier1,
00321 directory.meta.tier2)
00322
00323 single_fairy1=fairy_url_single(directory.meta.run1,
00324 directory.meta.sample,
00325 directory.meta.release1,
00326 comp_abs_path,
00327 directory.meta.tier1,
00328 "",500,500)
00329 single_fairy2=fairy_url_single(directory.meta.run2,
00330 directory.meta.sample,
00331 directory.meta.release2,
00332 comp_abs_path,
00333 directory.meta.tier2,
00334 "",500,500)
00335
00336 html_comparisons+='<div class="span-6 %s"><p>%s</p>' %(class_type,comparison.name)+\
00337 '<p class="alt">%s: %.2E</p>' %(comparison.test_name,comparison.rank)+\
00338 '<div><a class="black_link" href="%s">%s</a></div>'%(single_fairy1,directory.meta.release1)+\
00339 '<div><a href="%s">%s</a></div>'%(single_fairy2,directory.meta.release2)+\
00340 '<a href="%s"><img src="%s" class="top center %s"></a></div>'%(big_fairy,small_fairy,cat_classes[category])
00341
00342 if counter==3:
00343 html_comparisons+="<hr>"
00344 counter=0
00345 counter+=1
00346 tot_counter+=1
00347
00348 if len(html_comparisons)!=0:
00349 html='<div class="span-20"><h2 class="alt">%s Comparisons</h2></div>' %cat_names[category]
00350 html+=html_comparisons
00351 html+='<hr>'
00352 return html
00353 return ""
00354
00355
00356
00357 def get_rank_section(directory):
00358
00359 imgname="RankSummary.png"
00360 gStyle.SetPadTickY(0)
00361 c=TCanvas("ranks","ranks",500,400)
00362
00363 c.cd()
00364
00365 h=directory.rank_histo
00366 rank_histof=TH1F(h.GetName(),"",h.GetNbinsX(),h.GetXaxis().GetXmin(),h.GetXaxis().GetXmax())
00367 rank_histof.SetLineWidth(2)
00368 for i in xrange(0,h.GetNbinsX()+1):
00369 rank_histof.SetBinContent(i,h.GetBinContent(i))
00370 h.SetTitle("Ranks Summary;Rank;Frequency")
00371 h.Draw("Hist")
00372 c.Update()
00373 rank_histof.ComputeIntegral()
00374 integral = rank_histof.GetIntegral()
00375 rank_histof.SetContent(integral)
00376
00377 rightmax = 1.1*rank_histof.GetMaximum()
00378 scale = gPad.GetUymax()/rightmax
00379 rank_histof.SetLineColor(kRed)
00380 rank_histof.Scale(scale)
00381 rank_histof.Draw("same")
00382
00383
00384 axis = TGaxis(gPad.GetUxmax(),gPad.GetUymin(),gPad.GetUxmax(), gPad.GetUymax(),0,rightmax,510,"+L")
00385 axis.SetTitle("Cumulative")
00386 axis.SetTitleColor(kRed)
00387 axis.SetLineColor(kRed)
00388 axis.SetLabelColor(kRed)
00389 axis.Draw()
00390
00391 rank_histof.Draw("Same");
00392
00393
00394 c.Print(imgname)
00395
00396 page_html='<div class="span-20"><h2 class="alt"><a name="rank_summary">Ranks Summary</a></h2>'
00397 page_html+='<div class="span-19"><img src="%s"></div>' %imgname
00398 page_html+='</div> <a href="#top">Top...</a><hr>'
00399
00400 return page_html
00401
00402
00403
00404 def directory2html(directory, hashing, depth=0):
00405 """Converts a directory tree into html pages, very nice ones.
00406 """
00407
00408 depth+=1
00409
00410
00411
00412
00413
00414
00415
00416 for subdir in directory.subdirs:
00417 directory2html(subdir,hashing, depth)
00418
00419 page_html=get_page_header(directory)+\
00420 get_title_section(directory,hashing, depth)+\
00421 get_summary_section(directory)+\
00422 get_subdirs_section(directory, hashing)
00423
00424 for do_cat,cat in ((directory.n_comp_fails >0,FAIL ),
00425 (directory.n_comp_nulls >0,NULL ),
00426
00427 (directory.n_comp_successes >0 and directory.draw_success,SUCCESS ),
00428 (directory.n_comp_skiped >0,SKIPED)):
00429 if do_cat:
00430 page_html+=get_comparisons(cat,directory)
00431
00432
00433
00434 if depth==1:
00435 page_html+=get_rank_section(directory)
00436
00437
00438 page_html+=get_page_footer()
00439
00440 page_name=directory.name
00441
00442 if len(page_name)==0:
00443 page_name="RelMonSummary"
00444 if hashing:
00445 ofilename = "%s.html" %(hash_name(page_name, hashing))
00446 else:
00447 ofilename="%s_%s.html" %(directory.mother_dir.replace("/","_"),page_name)
00448 ofilename=ofilename.strip("_")
00449
00450 ofile=open(ofilename,"w")
00451 ofile.write(page_html)
00452 ofile.close()
00453
00454
00455
00456
00457
00458 def build_gauge(total_success_rate,minrate=.80,small=False,escaped=False):
00459 total_success_rate_scaled=(total_success_rate-minrate)
00460 total_success_rate_scaled_repr=total_success_rate_scaled/(1-minrate)
00461 if total_success_rate_scaled_repr<0:
00462 total_success_rate_scaled_repr=0
00463 size_s="200x100"
00464 if small:
00465 size_s="40x30"
00466
00467 gauge_link ="https://chart.googleapis.com/chart?chs=%s&cht=gom"%size_s
00468 gauge_link+="&chd=t:%2.1f"%(total_success_rate_scaled_repr*100.)
00469 if not small:
00470 gauge_link+="&chxt=x,y&chxl=0:|%2.1f%%|1:|%i%%|%i%%|100%%"%(total_success_rate*100,minrate*100.,(1+minrate)*50)
00471 gauge_link+="&chma=10,10,10,0"
00472 img_tag= '<img src="%s">'%gauge_link
00473 if escaped:
00474 img_tag=cgi.escape(img_tag)
00475 return img_tag
00476
00477
00478
00479 def get_aggr_pairs_info(dir_dict,the_aggr_pairs=[]):
00480
00481 aggr_pairs_info=[]
00482
00483 list_of_names=[]
00484 if the_aggr_pairs==[]:
00485 for samplename,sampledir in dir_dict.items():
00486 for subsysdirname in sorted(sampledir.get_subdirs_dict().keys()):
00487 if not subsysdirname in list_of_names:
00488 list_of_names.append(subsysdirname)
00489 the_aggr_pairs.append((subsysdirname,[subsysdirname]))
00490
00491
00492 for cat_name, subdir_list in the_aggr_pairs:
00493 total_successes=0.
00494 total_directory_successes=0
00495 total_weight=0.
00496 present_subdirs={}
00497 total_ndirs=0
00498
00499 for dirname, sampledir in dir_dict.items():
00500
00501 for subdirname,subdir in sampledir.get_subdirs_dict().items():
00502 if subdirname in subdir_list:
00503 nsucc=subdir.n_successes
00504 total_successes+=nsucc
00505 weight=subdir.weight
00506 total_weight+=weight
00507 total_ndirs+=1
00508
00509 total_directory_successes+= float(nsucc)/weight
00510 if present_subdirs.has_key(subdirname):
00511 this_dir_dict=present_subdirs[subdirname]
00512 this_dir_dict["nsucc"]+=nsucc
00513 this_dir_dict["weight"]+=weight
00514 else:
00515 present_subdirs[subdirname]={"nsucc":nsucc,"weight":weight}
00516
00517 for subsubdirname,subsubdir in subdir.get_subdirs_dict().items():
00518 for pathname in filter(lambda name:"/" in name,subdir_list):
00519 selected_subdirname,selected_subsubdirname = pathname.split("/")
00520 if selected_subdirname == subdirname and selected_subsubdirname==subsubdirname:
00521
00522 nsucc=subsubdir.n_successes
00523 total_successes+=nsucc
00524 weight=subsubdir.weight
00525 total_weight+=weight
00526 total_ndirs+=1
00527 total_directory_successes+= float(nsucc)/weight
00528
00529 if present_subdirs.has_key(subsubdirname):
00530 this_dir_dict=present_subdirs[subsubdirname]
00531 this_dir_dict["nsucc"]+=nsucc
00532 this_dir_dict["weight"]+=weight
00533 else:
00534 present_subdirs[subsubdirname]={"nsucc":nsucc,"weight":weight}
00535
00536 if total_ndirs == 0:
00537 print "No directory of the category %s is present in the samples: skipping." %cat_name
00538 continue
00539
00540 average_success_rate=total_directory_successes/(total_ndirs)
00541 aggr_pairs_info.append((cat_name,present_subdirs,total_weight,average_success_rate))
00542
00543 return aggr_pairs_info
00544
00545
00546
00547 def make_categories_summary(dir_dict,aggregation_rules):
00548
00549 aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
00550
00551
00552
00553
00554
00555 html= '<div class="span-20 colborder">'
00556 html+='<h2 class="alt"><a name="categories">Categories:</a></h2>'
00557
00558 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
00559
00560 html+='<div class="span-3 prepend-0 colborder">'
00561 html+='<h3>%s</h3>'%cat_name
00562 html+='<div><span class="alt">Avg. Success rate:</span></div>'
00563 html+='<div><span class="alt">%2.1f%%</span></div>'%(average_success_rate*100)
00564 html+='</div>'
00565 html+='<div class="span-9">'
00566
00567 html+='<div><p><span class="caps alt">DQM Directories (%i comparisons):</span></p></div>' %total_weight
00568 html+='<div><p><span class="alt">name: succ. rate - rel. weight</span></p></div>'
00569 html+='<ul>'
00570 for subdirname in sorted(present_subdirs.keys()):
00571 this_dir_dict=present_subdirs[subdirname]
00572 nsucc=this_dir_dict["nsucc"]
00573 weight=this_dir_dict["weight"]
00574 html+='<li><span class="caps">%s: %2.1f%% - %2.1f%%</span></li>'%(subdirname,100*float(nsucc)/weight,100*float(weight)/total_weight)
00575 html+='</ul>'
00576 html+='</div>'
00577
00578 html+='<div class="span-6 last">'
00579 html+=build_gauge(average_success_rate)
00580 html+='</div>'
00581
00582 html+='<hr>'
00583 return html+'<br><a href="#top">Top...</a> </div><hr>'
00584
00585
00586
00587 def make_twiki_table(dir_dict,aggregation_rules):
00588
00589
00590 meta= dir_dict.items()[0][1].meta
00591 releases=sorted([meta.release1,meta.release2])
00592 latest_release=releases[1].split("-")[0]
00593
00594
00595 aggr_pairs_info= get_aggr_pairs_info(dir_dict,aggregation_rules)
00596
00597
00598
00599 html= '<div class="span-20 colborder">'
00600 html+='<h2 class="alt"><a name="twiki_table">Twiki snipppet for release managers</a></h2>'
00601 html+='<div>| Release | Comparison |'
00602 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
00603 html+=cat_name
00604 html+=" | "
00605 html+='</div>'
00606
00607 html+='<div>| %s | %%ICON{arrowdot}%% | '%latest_release
00608
00609
00610
00611 for cat_name,present_subdirs,total_weight,average_success_rate in aggr_pairs_info:
00612
00613 html+=build_gauge(average_success_rate,small=True,escaped=True)
00614 html+=" | "
00615
00616 html+='</div> <a href="#top">Top...</a>'
00617 html+='<hr>'
00618 return html+'</div>'
00619
00620
00621
00622 def get_pie_tooltip(directory):
00623 tooltip="%s\nS:%2.1f%% N:%2.1f%% F:%2.1f%% Sk:%2.1f%%" %(directory.name,directory.get_success_rate(),directory.get_null_rate(),directory.get_fail_rate(),directory.get_skiped_rate())
00624 return tooltip
00625
00626
00627
00628 def make_barchart_summary(dir_dict,name="the_chart",title="DQM directory",the_aggr_pairs=[]):
00629
00630 aggr_pairs_info= get_aggr_pairs_info(dir_dict,the_aggr_pairs)
00631
00632 script="""
00633 <script type="text/javascript" src="https://www.google.com/jsapi"></script>
00634 <script type="text/javascript">
00635 google.load("visualization", "1", {packages:["corechart"]});
00636 google.setOnLoadCallback(drawChart);
00637 function drawChart() {
00638 var data = new google.visualization.DataTable();
00639 data.addColumn('string', 'DQM Directory');
00640 data.addColumn('number', 'Success Rate');
00641 """
00642 script+="data.addRows(%i);\n"%len(aggr_pairs_info)
00643 counter=0
00644 for subsystname,present_directories,weight,success_rate in aggr_pairs_info:
00645
00646 script+="data.setValue(%i, 0, '%s');\n"%(counter,subsystname)
00647 script+="data.setValue(%i, 1, %2.2f);\n"%(counter,success_rate)
00648 counter+=1
00649 script+="""
00650 var chart = new google.visualization.BarChart(document.getElementById('%s'));
00651 chart.draw(data, {width: 1024, height: %i, title: 'Success Rate',
00652 vAxis: {title: '%s', titleTextStyle: {color: 'red'},textStyle: {fontSize: 14}}
00653 });
00654 }
00655 </script>
00656 """%(name,40*counter,title)
00657 return script
00658
00659
00660
00661
00662 def make_summary_table(indir,aggregation_rules,aggregation_rules_twiki, hashing_flag):
00663 """Create a table, with as rows the directories and as columns the samples.
00664 Each box in the table will contain a pie chart linking to the directory.
00665 """
00666
00667
00668
00669 chdir(indir)
00670 if os.path.isabs(indir):
00671 title = basename(indir)
00672 else:
00673 title=indir
00674 title=title.strip(".")
00675 title=title.strip("/")
00676
00677
00678
00679 sample_pkls=filter(lambda name: name.endswith(".pkl"),listdir("./"))
00680
00681
00682 dir_unpicklers=[]
00683 n_unpicklers=0
00684 for sample_pkl in sample_pkls:
00685 dir_unpickler=unpickler(sample_pkl)
00686 dir_unpickler.start()
00687 n_unpicklers+=1
00688 dir_unpicklers.append(dir_unpickler)
00689 if n_unpicklers>=1:
00690 n_unpicklers=0
00691 for dir_unpickler in dir_unpicklers:
00692 dir_unpickler.join()
00693
00694 dir_dict={}
00695
00696
00697 global_dir=Directory("global","")
00698 for dir_unpickler in dir_unpicklers:
00699 dir_unpickler.join()
00700 directory=dir_unpickler.directory
00701
00702
00703 global_dir.meta=directory.meta
00704 dir_dict[dir_unpickler.filename.replace(".pkl","")]=directory
00705 global_dir.subdirs.append(directory)
00706
00707 global_dir.calcStats()
00708
00709 directories_barchart=make_barchart_summary(dir_dict,'dir_chart',"DQM Directory")
00710 categories_barchart=make_barchart_summary(dir_dict,'cat_chart','Category',aggregation_rules)
00711
00712 page_html = get_page_header(additional_header=directories_barchart+categories_barchart)
00713 rel1=""
00714 rel2=""
00715 try:
00716 rel1,rel2=title.split("VS")
00717 except:
00718 rel1=global_dir.meta.release1.split("-")[0]
00719 rel2=global_dir.meta.release2.split("-")[0]
00720 global_dir.meta.release1=rel1
00721 global_dir.meta.release2=rel2
00722
00723
00724 all_subdirs=[]
00725 for directory in dir_dict.values():
00726 for subdir_name in directory.get_subdirs_names():
00727 all_subdirs.append(subdir_name)
00728 all_subdirs=sorted(list(set(all_subdirs)))
00729
00730
00731 page_html+= '<div class="span-20">'+\
00732 '<h2><a name="top" href="https://twiki.cern.ch/twiki/bin/view/CMSPublic/RelMon">RelMon</a> Global Report: %s</h2>'%title+\
00733 '</div>'+\
00734 '<div class="span-1">'+\
00735 '<h2><a href="%s">main...</a></h2>' %relmon_mainpage+\
00736 '</div>'+\
00737 '<hr>'
00738 page_html+='<div class="span-24"><p></p></div>\n'*3
00739
00740
00741 page_html+= get_summary_section(global_dir,False)
00742
00743
00744 page_html+= '<div class="span-24">'
00745 page_html+= '<div class="span-20 colborder"><h2 class="alt">Sections:</h2>'+\
00746 '<ul>'+\
00747 '<li><a href="#summary_barchart">Summary Barchart</a></li>'+\
00748 '<li><a href="#categories">Categories</a></li>'+\
00749 '<li><a href="#detailed_barchart">Detailed Barchart</a></li>'+\
00750 '<li><a href="#summary_table">Summary Table</a></li>'+\
00751 '<li><a href="#rank_summary">Ranks Summary</a></li>'+\
00752 '<li><a href="#twiki_table">Twiki Table</a></li>'+\
00753 '</ul>'+\
00754 '</div><hr>'
00755
00756
00757
00758 page_html+='<div class="span-24"><h2 class="alt"><a name="summary_barchart">Summary Barchart</a></h2></div>'
00759 page_html+='<div id="cat_chart"></div> <a href="#top">Top...</a><hr>'
00760
00761
00762 page_html+=make_categories_summary(dir_dict,aggregation_rules)
00763
00764
00765 page_html+='<div class="span-24"><h2 class="alt"><a name="detailed_barchart">Detailed Barchart</a></h2></div>'
00766 page_html+='<div id="dir_chart"></div> <a href="#top">Top...</a><hr>'
00767
00768
00769 for i in xrange(2):
00770 page_html+='<div class="span-24"><p></p></div>\n'
00771
00772
00773
00774 page_html+='<div class="span-24"><h2 class="alt"><a name="summary_table">Summary Table</a></h2></div>'
00775
00776 for i in xrange(5):
00777 page_html+='<div class="span-24"><p></p></div>\n'
00778
00779 page_html+="""
00780 <table border="1" >
00781 <tr>
00782 <td> </td>
00783 """
00784
00785
00786 page_html+="""
00787 <td><div class="span-1"><p class="rotation" style="alt"><b>Summary</b></p></div></td>"""
00788
00789 sorted_samples=sorted(dir_dict.keys())
00790 for sample in sorted_samples:
00791 sample_nick=sample
00792
00793
00794
00795
00796
00797
00798
00799 page_html+="""
00800 <td><div class="span-1"><p class="rotation" style="">%s</p></div></td>"""%sample_nick
00801 page_html+=" </tr>\n"
00802
00803
00804
00805
00806 page_html+="<tr>"
00807 page_html+='<td style="background-color:white;"><div class="span-1">'
00808
00809 page_html+='<b>Summary</b></div></td>'
00810 page_html+='<td style="background-color:white;" class = "colborder" ><div class="span-1"><img src="%s" alt="%s"></div></td>'%(global_dir.get_summary_chart_ajax(55,55),get_pie_tooltip(global_dir))
00811 for sample in sorted_samples:
00812 col=dir_dict[sample]
00813
00814 summary_page_name=hash_name("RelMonSummary", hashing_flag)+".html"
00815 if col.name!="":
00816 summary_page_name=hash_name(col.name, hashing_flag)+".html"
00817 img_link=col.get_summary_chart_ajax(55,55)
00818 page_html+='<td style="background-color:white;"><div class="span-1">'
00819 page_html+='<a href="%s/%s"><img src="%s" title="%s"></a></div></td>' %(sample,summary_page_name,img_link,get_pie_tooltip(col))
00820 page_html+="</tr>"
00821
00822
00823 for subdir_name in all_subdirs:
00824
00825 page_html+=' <tr>\n'
00826 page_html+=' <td style="background-color:white;">%s</td>\n' %subdir_name
00827
00828 row_summary=Directory("row_summary","")
00829 sample_counter=0
00830 n_samples=len(sorted_samples)
00831
00832 for sample in sorted_samples:
00833 subdirs_dict=directory.get_subdirs_dict()
00834 directory=dir_dict[sample]
00835 dir_is_there=subdirs_dict.has_key(subdir_name)
00836 if dir_is_there:
00837 row_summary.subdirs.append(subdirs_dict[subdir_name])
00838
00839
00840 row_summary.calcStats()
00841 img_link=row_summary.get_summary_chart_ajax(55,55)
00842 page_html+='<td style="background-color:white;"><div class="span-1">'
00843 page_html+='<img src="%s" title="%s"></div></td>' %(img_link,get_pie_tooltip(row_summary))
00844
00845 for sample in sorted_samples:
00846 sample_counter+=1
00847
00848 directory=dir_dict[sample]
00849 subdirs_dict=directory.get_subdirs_dict()
00850
00851
00852 summary_page=join(sample,"%s.html"%(hash_name(subdir_name, hashing_flag)))
00853 if directory.name!="":
00854
00855 summary_page=join(sample,"%s_%s.html"%(directory.name,hash_name(subdir_name,hashing_flag)))
00856 dir_is_there=subdirs_dict.has_key(subdir_name)
00857
00858 img_link="https://chart.googleapis.com/chart?cht=p3&chco=C0C0C0&chs=50x50&chd=t:1"
00859 img_tooltip="N/A"
00860 if dir_is_there:
00861
00862 img_link=subdirs_dict[subdir_name].get_summary_chart_ajax(50,50)
00863 img_tooltip=get_pie_tooltip(subdirs_dict[subdir_name])
00864
00865 page_html+='<td style="background-color:white;"><div class="span-1">'
00866 if dir_is_there:
00867 page_html+='<a href="%s">'%(summary_page)
00868 page_html+='<img src="%s" title="%s" height=50 width=50>' %(img_link,img_tooltip)
00869 if dir_is_there:
00870 page_html+='</a>'
00871 page_html+='</div></td>'
00872
00873 page_html+=" </tr>\n"
00874
00875
00876
00877 page_html+='</table> <a href="#top">Top...</a><hr>'
00878
00879 page_html+=get_rank_section(global_dir)
00880
00881 page_html+=make_twiki_table(dir_dict,aggregation_rules_twiki)
00882
00883 page_html+=get_page_footer()
00884 return page_html
00885
00886
00887
00888 def hash_name(file_name, flag):
00889 if flag:
00890 return hashlib.md5(file_name).hexdigest()[:10]
00891 else:
00892 return file_name
1.7.3