|
|
|
Classes | |
| class | ReldirExcept |
| Small functions. More... | |
| class | Row |
| class | Table |
Functions | |
| def | addtrailingslash |
| def | copytree4 |
| def | createCandlHTML |
| Create HTML pages for candles. | |
| def | createHTMLtab |
| def | createRegressHTML |
| def | createWebReports |
| Create web report index and create HTML file for each candle. | |
| def | delTmpDir |
| Delete tmp dir if we used it. | |
| def | dirname_cmp |
| def | docopy |
| def | fail |
| def | get_environ |
| Get require environment variables. | |
| def | getArchVersionFromLog |
| def | getcmd |
| def | getcmdBasic |
| def | getDate |
| def | getDirnameDirs |
| Grab dirs that end in strings defined in DirName. | |
| def | getNumOfEventsFromLog |
| def | getOutputNames |
| def | getRelativeDir |
| Some functions used for copying. | |
| def | getStageRepDirs |
| Determine locations of staging and report dirs. | |
| def | logrep_cmp |
| def | main |
| def | optionparse |
| Option parser. | |
| def | populateFromTupleRoot |
| def | prettySize |
| def | print_header |
| def | reg_dirname_cmp |
| def | rootfile_cmp |
| def | scanReportArea |
| Scan report area for required things. | |
| def | stageIgProfReports |
| def | step_cmp |
| def | syncToRemoteLoc |
| Upload stage to remote location. | |
| def | syscp |
| def | timerep_cmp |
Variables | |
| int | CallgrindNumOfEvents = 9999 |
| tuple | cpDirFilter = ( ) |
| tuple | cpFileFilter = ( "*.root", ) |
| string | DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal" |
| string | DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation" |
| tuple | DirName |
| int | IgProfNumOfEvents = 9999 |
| int | MemcheckNumOfEvents = 9999 |
| tuple | PROG_NAME = os.path.basename(sys.argv[0]) |
| tuple | Steps = set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"]) |
| int | TimeSizeNumOfEvents = 9999 |
| string | TMP_DIR = "" |
| def cmsPerfPublish::addtrailingslash | ( | adir | ) |
Definition at line 69 of file cmsPerfPublish.py.
| def cmsPerfPublish::copytree4 | ( | src, | |
| dest, | |||
keepTop = True |
|||
| ) |
Definition at line 1908 of file cmsPerfPublish.py.
01909 : 01910 def _getNewLocation(source,child,dst,keepTop=keepTop): 01911 place = getRelativeDir(source,child,keepTop=keepTop) 01912 return os.path.join(dst,place) 01913 def _copyFilter(source,dst,curdir,fsnodes,filter,dirs=False): 01914 for node in fsnodes: 01915 dontFilter = True 01916 filterExist = not len(filter) == 0 01917 if filterExist: 01918 dontFilter = not reduce(lambda x,y: x or y,map(lambda x: fnmatch.fnmatch(node,x),filter)) 01919 if dontFilter: 01920 node = os.path.join(curdir,node) # convert to absolute path 01921 try: 01922 newnode = _getNewLocation(source,node,dst) 01923 if dirs: 01924 os.mkdir(newnode) 01925 else: 01926 copy2(node,newnode) 01927 except IOError, detail: 01928 print "WARNING: Could not copy %s to %s because %s" % (node,newnode,detail) 01929 except OSError, detail: 01930 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail) 01931 except ReldirExcept: 01932 print "WARNING: Could not determine new location for source %s into destination %s" % (source,dst) 01933 else: 01934 if len(filter) > 0: 01935 try: 01936 match = fnmatch.fnmatch(node,filter[0]) 01937 assert not match 01938 except AssertionError, detail: 01939 print node, filter[0], match 01940 raise RuntimeError 01941 if _verbose: 01942 if "root" in node: 01943 print "Filter %s Copied %s to %s" % (dontFilter,node,newnode) 01944 print "fnmatch %s" % fnmatch.fnmatch(node,cpFileFilter[0]) 01945 01946 gen = os.walk(src) 01947 try: 01948 newloc = _getNewLocation(src,src,dest) 01949 01950 os.mkdir(newloc) 01951 try: 01952 while True: 01953 step = gen.next() 01954 curdir = step[0] 01955 dirs = step[1] 01956 files = step[2] 01957 01958 _copyFilter(src,dest,curdir,dirs,cpDirFilter,dirs=True) 01959 _copyFilter(src,dest,curdir,files,cpFileFilter) 01960 01961 except StopIteration: 01962 pass 01963 except IOError, detail: 01964 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail) 01965 except OSError, detail: 01966 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail) 01967 except ReldirExcept: 01968 print "WARNING: Could not determine the new location for source %s into destination %s" % (src,dest)
| def cmsPerfPublish::createCandlHTML | ( | tmplfile, | |
| candlHTML, | |||
| CurrentCandle, | |||
| WebArea, | |||
| repdir, | |||
| ExecutionDate, | |||
| LogFiles, | |||
| cmsScimarkResults, | |||
| date, | |||
| prevrev | |||
| ) |
Create HTML pages for candles.
Definition at line 845 of file cmsPerfPublish.py.
00846 : 00847 global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents 00848 def _stripbase(base, astr): 00849 basereg = re.compile("^%s/?(.*)" % base) 00850 out = astr 00851 found = basereg.search(astr) 00852 if found: 00853 out = found.groups()[0] 00854 return out 00855 00856 def _getProfileReportLink(repdir,CurrentCandle,CurDir,step,CurrentProfile,Profiler): 00857 #FIXME: 00858 #Pileup now has it own directory... should add it in the DirName dictionary at the beginning? 00859 ProfileTemplate=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step,CurrentProfile),Profiler) 00860 #print ProfileTemplate 00861 #There was the issue of SIM vs sim (same for DIGI) between the previous RelVal based performance suite and the current. 00862 ProfileTemplateLowCaps=os.path.join(repdir, "%s_%s" % (CurrentCandle,CurDir), "*_%s_%s*" % (step.lower(),CurrentProfile),Profiler) 00863 ProfileReportLink = glob.glob(ProfileTemplate) 00864 #Filter out the Pile up directories when step does not contain Pile up 00865 #if not ('PILEUP' in step): 00866 # print "NPNPNPNP BEFORE %s"%ProfileReportLink 00867 # ProfileReportLink=filter(lambda x: "PU" not in x,ProfileReportLink) 00868 # print "NPNPNPNP %s"%ProfileReportLink 00869 #print ProfileReportLink 00870 if len(ProfileReportLink) > 0: 00871 #print ProfileReportLink 00872 if not reduce(lambda x,y: x or y,map(lambda x: CurrentCandle in x,ProfileReportLink)):# match with caps try low caps 00873 ProfileReportLink = glob.glob(ProfileTemplateLowCaps) 00874 else: 00875 ProfileReportLink = glob.glob(ProfileTemplateLowCaps) 00876 ProfileReportLink = map(lambda x: _stripbase(repdir,x),ProfileReportLink) 00877 #print ProfileReportLink 00878 return ProfileReportLink 00879 00880 def _writeReportLink(INDEX,ProfileReportLink,CurrentProfile,step,NumOfEvents,Profiler=""): 00881 if Profiler == "": 00882 INDEX.write("<li><a href=\"%s\">%s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,step,NumOfEvents)) 00883 else: 00884 #FIXME: need to fix this: why do we have the number of Memcheck events hardcoded? 00885 if CurrentProfile == "memcheck_valgrind":#FIXME:quick and dirty hack to report we have changed memcheck to 5 events now... 00886 INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,"5")) 00887 else: 00888 INDEX.write("<li><a href=\"%s\">%s %s %s (%s events)</a></li>\n" % (ProfileReportLink,CurrentProfile,Profiler,step,NumOfEvents)) 00889 def IgProfDumpsTable(INDEX,ProfileLinks,step): 00890 #See if the end of job profiles IgProfMemTotal.res or IgProfMemLive.res are in the list as they should: 00891 EndOfJobProfileLink=filter(lambda x: "IgProfMemTotal.res" in x or "IgProfMemLive.res" in x, ProfileLinks)[0] #Assume only one of the two is there, as it should. 00892 #Remove it from the list so we can order it: 00893 ProfileLinks.remove(EndOfJobProfileLink) 00894 #Sort the list in numerical order: 00895 ProfileLinks.sort(key=lambda x: int(x.split(".")[-2])) 00896 #Prepare regexp to find and replace MemTotal with MemLive for .gz links 00897 IgProfMemLive_regexp=re.compile("IgProfMemLive") 00898 if IgProfMemLive_regexp.search(EndOfJobProfileLink): 00899 MemProfile="IgProf MEM LIVE" 00900 else: 00901 MemProfile="IgProf MEM TOTAL" 00902 #Prepare the table header: 00903 INDEX.write("<li>%s"%MemProfile) 00904 INDEX.write("<table><tr><td>Profile after event</td><td>Total Memory Size (bytes)</td><td>Total Calls (number)</td><td>Link to gzipped IgProf profile</td></tr>") 00905 for link in ProfileLinks: 00906 #Build and check the link to the .gz profile that is always called IgProfMemTotal also for MemLive: 00907 gzProfile=os.path.join(link.split("/")[-3],link.split("/")[-1])[:-3]+"gz" 00908 if IgProfMemLive_regexp.search(gzProfile): 00909 gzProfile=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzProfile) 00910 INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%(link.split(".")[-2],open(link,"r").readlines()[6].split()[1],open(link,"r").readlines()[6].split()[2],gzProfile,os.path.basename(gzProfile))) 00911 #Put in the end of job one by hand: 00912 gzEndOfJobProfileLink=os.path.join(EndOfJobProfileLink.split("/")[-3],EndOfJobProfileLink.split("/")[-1])[:-3]+"gz" 00913 if IgProfMemLive_regexp.search(gzEndOfJobProfileLink): 00914 gzEndOfJobProfileLink=IgProfMemLive_regexp.sub(r"IgProfMemTotal",gzEndOfJobProfileLink) 00915 INDEX.write("<tr><td>%s</td><td>%s</td><td>%s</td><td><a href=%s>%s</a></td></tr>"%("End of job",open(EndOfJobProfileLink,"r").readlines()[6].split()[1],open(EndOfJobProfileLink,"r").readlines()[6].split()[2],gzEndOfJobProfileLink,os.path.basename(gzEndOfJobProfileLink))) 00916 #Closing the table and the list item tags 00917 INDEX.write("</table>") 00918 INDEX.write("</li>") 00919 #FIXME: 00920 #These numbers are used in the index.html they are not automatically matched to the actual 00921 #ones (one should automate this, by looking into the cmsCreateSimPerfTestPyRelVal.log logfile) 00922 00923 NumOfEvents={ 00924 DirName[0] : TimeSizeNumOfEvents, 00925 DirName[1] : IgProfNumOfEvents, 00926 DirName[2] : IgProfNumOfEvents, 00927 DirName[3] : IgProfNumOfEvents, 00928 DirName[4] : CallgrindNumOfEvents, 00929 DirName[5] : MemcheckNumOfEvents, 00930 DirName[6] : TimeSizeNumOfEvents, 00931 DirName[7] : IgProfNumOfEvents, 00932 DirName[8] : IgProfNumOfEvents, 00933 DirName[9] : IgProfNumOfEvents, 00934 DirName[10] : CallgrindNumOfEvents, 00935 DirName[11] : MemcheckNumOfEvents 00936 } 00937 00938 Profile=( 00939 #These need to match the profile directory names ending within the candle directories 00940 "TimingReport", 00941 "TimeReport", 00942 "SimpleMemReport", 00943 "EdmSize", 00944 "IgProfperf", 00945 "IgProfMemTotal", 00946 "IgProfMemLive", 00947 "IgProfMemAnalyse", 00948 "valgrind", 00949 "memcheck_valgrind" 00950 ) 00951 IgProfMemAnalyseOut=( #This is the special case of IgProfMemAnalyse 00952 "doBeginJob_output.html", 00953 "doEvent_output.html", 00954 "mem_live.html", 00955 "mem_total.html" 00956 ) 00957 memcheck_valgrindOut=( #This is the special case of Valgrind MemCheck (published via Giovanni's script) 00958 "beginjob.html", 00959 "edproduce.html", 00960 "esproduce.html" 00961 ) 00962 OutputHtml={ #These are the filenames to be linked in the candle html page for each profile 00963 Profile[0] : "*TimingReport.html", #The wildcard spares the need to know the candle name 00964 Profile[1] : "TimeReport.html", #This is always the same (for all candles) 00965 Profile[2] : "*.html", #This is supposed to be *SimpleMemoryCheck.html, but there is a bug in cmsRelvalreport.py and it is called TimingReport.html! 00966 Profile[3] : "objects_pp.html", #This is one of 4 objects_*.html files, it's indifferent which one to pick, just need consistency 00967 Profile[4] : "overall.html", #This is always the same (for all candles) 00968 Profile[5] : "overall.html", #This is always the same (for all candles) 00969 Profile[6] : "overall.html", #This is always the same (for all candles) 00970 Profile[7] : "doBeginJob_output.html", #This is complicated... there are 3 html to link... (see IgProf MemAnalyse below) 00971 Profile[8] : "overall.html", #This is always the same (for all candles) 00972 Profile[9] : "beginjob.html" #This is complicated there are 3 html to link here too... (see Valgrind MemCheck below) 00973 } 00974 00975 00976 candnreg = re.compile("CandleName") 00977 candhreg = re.compile("CandlesHere") 00978 try: 00979 CAND = open(candlHTML,"w") 00980 for line in open(tmplfile): 00981 if candhreg.search(line): 00982 CAND.write('<div id="content">') 00983 CAND.write("<h2>") 00984 CAND.write(CurrentCandle) 00985 CAND.write("</h2>\n") 00986 00987 if _verbose: 00988 print "Producing candles html: ", CurrentCandle 00989 00990 for CurDir in DirName: 00991 00992 LocalPath = os.path.join(repdir,"%s_%s" % (CurrentCandle,CurDir)) 00993 LocalDirname = os.path.basename(LocalPath) 00994 00995 if not prevrev == "": 00996 00997 profs = [] 00998 #FIXME! 00999 #Check what this was used for.. it now has different DirName list since we added IgProf_Perf and IgProf_Mem on top of IgProf alone (still there) 01000 if CurDir == DirName[0] or CurDir == DirName[6]: #TimeSize tests (with and without PU) 01001 profs = Profile[0:4] 01002 elif CurDir == DirName[1] or CurDir == DirName[7]: #IgProf tests (with and without PU) 01003 profs = Profile[4:8] 01004 elif CurDir == DirName[2] or CurDir == DirName[8]: #IgProf Perf tests (with and without PU) 01005 profs =[Profile[4]] #Keeping format to a list... 01006 elif CurDir == DirName[3] or CurDir == DirName[9]: #IgProf Mem tests (with and without PU) 01007 profs =Profile[5:8] #Keeping format to a list... 01008 elif CurDir == DirName[4] or CurDir == DirName[10]: #Callgrind tests (with and without PU) 01009 profs = [Profile[8]] #Keeping format to a list... 01010 elif CurDir == DirName[5] or CurDir == DirName[11]: #Memcheck tests (with and without PU) 01011 profs = [Profile[9]] #Keeping format to a list... 01012 #This could be optimized, but for now just comment the code: 01013 #This for cycle takes care of the case in which there are regression reports to link to the html: 01014 for prof in profs: 01015 if _verbose: 01016 print "Scanning for profile information for: ", prof 01017 01018 printed = False 01019 fullprof = (CurrentCandle,prof) 01020 outd = "" 01021 nologext = "" 01022 if prof == "TimingReport": 01023 timeReports = glob.glob(os.path.join(LocalPath,"%s_*_%s.log" % (CandFname[CurrentCandle],prof))) 01024 if len(timeReports) > 0: 01025 if not printed: 01026 CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis")) 01027 printed = True 01028 for log in timeReports: 01029 reportName = os.path.basename(log) 01030 (nologext, outd) = getOutputNames(LocalDirname,reportName) 01031 CAND.write("<h4>%s</h4>\n" % reportName) 01032 htmNames = ["changes.gif"] 01033 otherNames = ["graphs.gif" , "histos.gif"] 01034 regressHTML= "%s-regress.html" % nologext 01035 pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,outd,x),otherNames))) 01036 html = "" 01037 if pathsExist: 01038 html = "<p>Performance graphs and histograms superimposed for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML) 01039 else: 01040 html = "<p>No change in performance graph available</p>\n" 01041 regressHTML="%s/%s" % (WebArea,regressHTML) 01042 01043 for x in htmNames: 01044 abspath = os.path.join(repdir,outd,x) 01045 if os.path.exists(abspath): 01046 html += "<p><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></p>\n" % (outd,x,outd,x) 01047 else: 01048 html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath) 01049 01050 createRegressHTML(regressHTML,repdir,outd,CurrentCandle,otherNames) 01051 CAND.write(html) 01052 CAND.write("\n</tr></table>") 01053 01054 01055 elif prof == "SimpleMemReport": 01056 simMemReports = glob.glob(os.path.join(LocalPath,"%s_*_%s" % (CandFname[CurrentCandle],prof))) 01057 simMemReports = map(lambda x: (CandFname[CurrentCandle],prof,x), simMemReports ) 01058 simMemReports.sort(cmp=dirname_cmp) 01059 simMemReports = map(lambda x: x[2], simMemReports ) 01060 if len(simMemReports) > 0: 01061 if not printed: 01062 CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis")) 01063 printed = True 01064 for adir in simMemReports: 01065 reportName = os.path.basename(adir) 01066 CAND.write("<h4>%s</h4>\n" % reportName) 01067 htmNames = ["vsize_change.gif", "rss_change.gif"] 01068 otherNames = ["vsize_graphs.gif","rss_graphs.gif"] 01069 nologext = reportName 01070 outd = reportName 01071 regressHTML= "%s-regress.html" % nologext 01072 pathsExist = reduce (lambda x,y: x or y,map(os.path.exists,map(lambda x: os.path.join(repdir,LocalDirname,outd,x),otherNames))) 01073 html = "" 01074 if pathsExist: 01075 html = "<p>Superimposed memory performance graphs for %s are <a href=\"%s\">here</a></p>\n" % (reportName,regressHTML) 01076 else: 01077 html = "<p>No change in performance graph available</p>\n" 01078 regressHTML="%s/%s" % (WebArea,regressHTML) 01079 01080 for x in htmNames: 01081 abspath = os.path.join(repdir,LocalDirname,outd,x) 01082 if os.path.exists(abspath): 01083 html += "<p><a href=\"./%s/%s/%s\"><img src=\"./%s/%s/%s\" /></a></p>\n" % (LocalDirname,outd,x,LocalDirname,outd,x) 01084 else: 01085 html += "<p>%s does not exist probably because the log file for the previous release was missing</p>" % (abspath) 01086 01087 createRegressHTML(regressHTML,repdir,"%s/%s" % (LocalDirname,outd),CurrentCandle,otherNames) 01088 CAND.write(html) 01089 CAND.write("\n</tr></table>") 01090 01091 elif prof == "EdmSize" or prof == "IgProfMemTotal" or prof == "IgProfMemLive" or prof== "IgProfperf" or prof == "Callgrind": 01092 regresPath = os.path.join(LocalPath,"%s_*_%s_regression" % (CandFname[CurrentCandle],prof)) 01093 regresses = glob.glob(regresPath) 01094 stepreg = re.compile("%s_([^_]*(_PILEUP)?)_%s_regression" % (CandFname[CurrentCandle],prof)) 01095 if len(regresses) > 0: 01096 if not printed: 01097 CAND.write("<p><strong>%s %s</strong></p>\n" % (prof,"Regression Analysis")) 01098 printed = True 01099 regresses = map(lambda x: (CandFname[CurrentCandle],prof,x),regresses) 01100 regresses.sort(cmp=reg_dirname_cmp) 01101 regresses = map(lambda x: x[2],regresses) 01102 for rep in regresses: 01103 base = os.path.basename(rep) 01104 found = stepreg.search(base) 01105 step = "Unknown-step" 01106 if found: 01107 step = found.groups()[0] 01108 htmlpage = "" 01109 if prof == "IgProfMemLive" or prof == "IgProfMemTotal" or prof== "IgProfperf" or prof == "Callgrind": 01110 htmlpage = "overall.html" 01111 else: 01112 htmlpage = "objects_pp.html" 01113 CAND.write("<a href=\"%s/%s/%s\">%s %s regression report</a><br/>\n" % (LocalDirname,base,htmlpage,prof,step)) 01114 #Here we go back to the "non-regression" reports listing 01115 CandleLogFiles = [] 01116 if os.path.exists(LocalPath): 01117 thedir = os.listdir(LocalPath) 01118 CandleLogFiles = filter(lambda x: (x.endswith(".log") or x.endswith("EdmSize")) and not os.path.isdir(x) and os.path.exists(x), map(lambda x: os.path.abspath(os.path.join(LocalPath,x)),thedir)) 01119 01120 if (len(CandleLogFiles)>0): 01121 01122 syscp(CandleLogFiles,WebArea + "/") 01123 base = os.path.basename(LocalPath) 01124 lfileshtml = "" 01125 01126 for cand in CandleLogFiles: 01127 cand = os.path.basename(cand) 01128 if _verbose: 01129 print "Found %s in %s\n" % (cand,LocalPath) 01130 01131 if not "EdmSize" in cand: 01132 lfileshtml += "<a href=\"./%s/%s\">%s </a><br/>" % (base,cand,cand) 01133 01134 CAND.write("<p><strong>Logfiles for %s</strong></p>\n" % CurDir) 01135 CAND.write(lfileshtml) 01136 01137 PrintedOnce = False 01138 for CurrentProfile in Profile: 01139 #print Steps 01140 for step in Steps: #Using Steps here that is Step+ProductionSteps! 01141 #print step 01142 #print "DEBUG:%s,%s,%s,%s,%s,%s"%(repdir,CurrentCandle,CurDir,step,CurrentProfile,OutputHtml[CurrentProfile]) 01143 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle, 01144 CurDir, 01145 step, 01146 CurrentProfile, 01147 OutputHtml[CurrentProfile]) 01148 #if ProfileReportLink: 01149 # print ProfileReportLink 01150 isProfinLink = False 01151 if len (ProfileReportLink) > 0: 01152 isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink)) 01153 01154 if isProfinLink: 01155 #It could also not be there 01156 01157 if (PrintedOnce==False): 01158 #Making sure it's printed only once per directory (TimeSize, IgProf, Valgrind) each can have multiple profiles 01159 01160 #This is the "title" of a series of profiles, (TimeSize, IgProf, Valgrind) 01161 CAND.write("<p><strong>%s</strong></p>\n" % CurDir) 01162 CAND.write("<ul>\n") 01163 PrintedOnce=True 01164 01165 #Special cases first (IgProf MemAnalyse and Valgrind MemCheck) 01166 #Add among the special cases any IgProfMem (5,6,7) since now we added the dumping: 01167 if (CurrentProfile == Profile[5] or CurrentProfile == Profile[6]): 01168 for prolink in ProfileReportLink: 01169 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir]) 01170 for igprofmem in ["*IgProfMemTotal*.res","*IgProfMemLive*.res"]: 01171 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle, 01172 CurDir, 01173 step, 01174 CurrentProfile, 01175 igprofmem) 01176 isProfinLink = False 01177 if len (ProfileReportLink) > 0: 01178 isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink)) 01179 if isProfinLink :#It could also not be there 01180 #for prolink in ProfileReportLink: 01181 IgProfDumpsTable(CAND,ProfileReportLink,step) 01182 # _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=os.path.basename(prolink)) 01183 01184 elif (CurrentProfile == Profile[7]): 01185 for igprof in IgProfMemAnalyseOut: 01186 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle, 01187 CurDir, 01188 step, 01189 CurrentProfile, 01190 igprof) 01191 isProfinLink = False 01192 if len (ProfileReportLink) > 0: 01193 isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink)) 01194 if isProfinLink :#It could also not be there 01195 for prolink in ProfileReportLink: 01196 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=igprof) 01197 01198 01199 elif (CurrentProfile == Profile[9]): 01200 01201 for memprof in memcheck_valgrindOut: 01202 #print memprof 01203 ProfileReportLink = _getProfileReportLink(repdir,CurrentCandle, 01204 CurDir, 01205 step, 01206 CurrentProfile, 01207 memprof 01208 ) 01209 isProfinLink = False 01210 if len (ProfileReportLink) > 0: 01211 isProfinLink = reduce(lambda x,y: x or y,map(lambda x: CurrentProfile in x,ProfileReportLink)) 01212 if isProfinLink :#It could also not be there 01213 for prolink in ProfileReportLink: 01214 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir],Profiler=memprof) 01215 01216 else: 01217 for prolink in ProfileReportLink: 01218 if "regression" not in prolink: #To avoid duplication of links to regression reports! 01219 _writeReportLink(CAND,prolink,CurrentProfile,step,NumOfEvents[CurDir]) 01220 #print "Step is %s, CurrentProfile is %s and ProfileReportLink is %s and prolink is %s"%(step,CurrentProfile,ProfileReportLink,prolink) 01221 01222 01223 if PrintedOnce: 01224 CAND.write("</ul>\n") 01225 PrintedOnce=False 01226 01227 CAND.write("<hr />") 01228 CAND.write("<br />\n") 01229 CAND.write("</div>\n") 01230 elif candnreg.search(line): 01231 CAND.write(CurrentCandle) 01232 else: 01233 CAND.write(line) 01234 01235 CAND.close() 01236 except IOError, detail: 01237 print "ERROR: Could not write candle html %s because %s" % (os.path.basename(candlHTML),detail) 01238
| def cmsPerfPublish::createHTMLtab | ( | INDEX, | |
| table_dict, | |||
| ordered_keys, | |||
| header, | |||
| caption, | |||
| name, | |||
mode = 0 |
|||
| ) |
Definition at line 1303 of file cmsPerfPublish.py.
01304 : 01305 cols = len(ordered_keys) 01306 totcols = (cols * 3) + 1 01307 innercol = 3 01308 colspan = totcols - 1 01309 labels = [] 01310 if mode == 1: 01311 labels = ["fs1","fs2","Δ"] 01312 elif mode == 2 or mode == 3: 01313 colspan = cols 01314 innercol = 1 01315 else: 01316 labels = ["t1" ,"t2" ,"Δ"] 01317 01318 01319 INDEX.write("<h3>%s</h3>\n" % header) 01320 INDEX.write("<table>\n") 01321 INDEX.write("<caption>%s</caption>\n" % caption) 01322 INDEX.write("<thead><tr><th></th><th colspan=\"%s\" scope=\"colgroup\">%s</th></tr></thead>" % (colspan,name)) 01323 INDEX.write("<tbody>\n") 01324 for key in ordered_keys: 01325 INDEX.write("<tr>") 01326 if key == None: 01327 INDEX.write("<th></th>") 01328 else: 01329 INDEX.write("<td scope=\"row\">") 01330 INDEX.write(key) 01331 INDEX.write("</td>") 01332 for col in table_dict[None]: 01333 if key == None: 01334 INDEX.write("<th colspan=\"%s\" scope=\"col\">" % innercol) 01335 INDEX.write(col) 01336 INDEX.write("</th>") 01337 else: 01338 rowdict = table_dict[key].getRowDict() 01339 if rowdict.has_key(col): 01340 if mode == 2: 01341 dat = prettySize(rowdict[col]) 01342 INDEX.write("<td>") 01343 INDEX.write("%s" % dat) 01344 INDEX.write("</td>") 01345 01346 elif mode == 3: 01347 dat = rowdict[col] 01348 INDEX.write("<td>") 01349 INDEX.write("%6.2f" % dat) 01350 INDEX.write("</td>") 01351 else: 01352 (data1, data2) = rowdict[col] 01353 diff = data2 - data1 01354 01355 if mode == 1: 01356 diff = prettySize(diff) 01357 data1 = prettySize(data1) 01358 data2 = prettySize(data2) 01359 01360 seq = [ data1, data2, diff ] 01361 for dat in seq: 01362 INDEX.write("<td id=\"data\">") 01363 if mode == 1: 01364 INDEX.write("%s" % dat) # %s if 01365 else: 01366 INDEX.write("%6.2f" % dat) # %s if 01367 01368 INDEX.write("</td>") 01369 else: 01370 if mode == 2 or mode == 3: 01371 INDEX.write("<td>") 01372 INDEX.write("N/A") 01373 INDEX.write("</td>") 01374 else: 01375 for i in range(3): 01376 INDEX.write("<td>") 01377 INDEX.write("N/A") 01378 INDEX.write("</td>") 01379 INDEX.write("</tr>\n") 01380 # write an additional row if this row is the header row 01381 # we need to describe the sub columns 01382 if not (mode == 2 or mode == 3 ): 01383 if key == None: 01384 INDEX.write("<tr>") 01385 INDEX.write("<th>Candles</th>") 01386 for col in table_dict[None]: 01387 INDEX.write("<th>%s</th>" % labels[0]) 01388 INDEX.write("<th>%s</th>" % labels[1]) 01389 INDEX.write("<th>%s</th>" % labels[2]) 01390 INDEX.write("</tr>\n") 01391 INDEX.write("</tbody></table>\n") 01392 01393 INDEX.write("<br />")
| def cmsPerfPublish::createRegressHTML | ( | reghtml, | |
| repdir, | |||
| outd, | |||
| CurrentCandle, | |||
| htmNames | |||
| ) |
Definition at line 682 of file cmsPerfPublish.py.
00683 : 00684 RegressTmplHTML="%s/doc/regress.html" % (BASE_PERFORMANCE) 00685 candnreg = re.compile("CandleName") 00686 candhreg = re.compile("CandlesHere") 00687 try: 00688 REGR = open(reghtml,"w") 00689 for line in open(RegressTmplHTML): 00690 if candhreg.search(line): 00691 html = "<table>" 00692 00693 for x in htmNames: 00694 abspath = os.path.join(repdir,outd) 00695 if os.path.exists(abspath): 00696 html += "<tr><td><a href=\"./%s/%s\"><img src=\"./%s/%s\" /></a></td></tr>\n" % (outd,x,outd,x) 00697 else: 00698 html += "<tr><td> %s does not exist probably because the log file for the previous release was missing</td></tr>" % (abspath) 00699 html += "</table>" 00700 REGR.write(html) 00701 elif candnreg.search(line): 00702 REGR.write(CurrentCandle) 00703 else: 00704 REGR.write(line) 00705 except IOError, detail: 00706 print "ERROR: Could not write regression html %s because %s" % (os.path.basename(reghtml),detail)
| def cmsPerfPublish::createWebReports | ( | WebArea, | |
| repdir, | |||
| ExecutionDate, | |||
| LogFiles, | |||
| cmsScimarkResults, | |||
| date, | |||
| prevrev | |||
| ) |
Create web report index and create HTML file for each candle.
Definition at line 1447 of file cmsPerfPublish.py.
01448 : 01449 01450 #Some nomenclature 01451 01452 Candle = Candles #These need to match the directory names in the work area 01453 01454 CmsDriverCandle = CandFname #{ #These need to match the cmsDriver.py output filenames 01455 01456 01457 #Produce a "small" index.html file to navigate the html reports/logs etc 01458 IndexFile="%s/index.html" % WebArea 01459 TemplateHtml="%s/doc/index.html" % BASE_PERFORMANCE 01460 01461 cmsverreg = re.compile("CMSSW_VERSION") 01462 hostreg = re.compile("HOST") 01463 lpathreg = re.compile("LocalPath") 01464 fsizereg = re.compile("FSizeTable") 01465 cpureg = re.compile("CPUTable") 01466 proddreg = re.compile("ProductionDate") 01467 logfreg = re.compile("LogfileLinks") 01468 dirbreg = re.compile("DirectoryBrowsing") 01469 pubdreg = re.compile("PublicationDate") 01470 candhreg = re.compile("CandlesHere") 01471 #Loop line by line to build our index.html based on the template one 01472 #Copy the perf_style.css file from Validation/Performance/doc 01473 01474 CandlTmpltHTML="%s/doc/candle.html" % BASE_PERFORMANCE 01475 if _verbose: 01476 print "Copying %s/doc/perf_style.css style file to %s/." % (BASE_PERFORMANCE,WebArea) 01477 print "Template used: %s" % TemplateHtml 01478 01479 syscp((BASE_PERFORMANCE + "/doc/perf_style.css"),WebArea + "/.") 01480 pureg = re.compile("(.*)_PILEUP") 01481 try: 01482 INDEX = open(IndexFile,"w") 01483 for NewFileLine in open(TemplateHtml) : 01484 if cmsverreg.search(NewFileLine): 01485 if prevrev == "": 01486 INDEX.write("Performance Reports for %s\n" % CMSSW_VERSION) 01487 else: 01488 globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev)) 01489 globs = glob.glob(globpath) 01490 if len(globs) < 1: 01491 pass 01492 else: 01493 latestreg = re.compile("REGRESSION.%s.vs.(.*)" % prevrev) 01494 found = latestreg.search(os.path.basename(globs[0])) 01495 if found: 01496 latestrel = found.groups()[0] 01497 INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,latestrel)) 01498 else: 01499 INDEX.write("Performance Reports with regression: %s VS %s\n" % (prevrev,CMSSW_VERSION)) 01500 elif hostreg.search(NewFileLine): 01501 INDEX.write(HOST + "\n") 01502 #File Size Summary Table 01503 elif fsizereg.search(NewFileLine): 01504 #Case of NO-REGRESSION: 01505 if prevrev == "": 01506 fsize_tab = Table() 01507 01508 for cand in Candles: 01509 fname = CandFname[cand] 01510 globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname) 01511 rootfiles = glob.glob(globpath) 01512 rootfiles = map(lambda x: (fname,x), rootfiles) 01513 rootfiles.sort(cmp=rootfile_cmp) 01514 rootfiles = map(lambda x: x[1], rootfiles) 01515 stepreg = re.compile("%s_(.*).root" % fname) 01516 createNewRow = True 01517 curRow = None 01518 createPURow = True 01519 puRow = None 01520 for rootf in rootfiles: 01521 base = os.path.basename(rootf) 01522 found = stepreg.search(base) 01523 step = "Unknown-step" 01524 if found: 01525 step = found.groups()[0] 01526 realstep = "Unknown-step" 01527 if "PILEUP" in step: 01528 found = pureg.search(step) 01529 if found: 01530 realstep = found.groups()[0] 01531 if createPURow: 01532 createPURow = False 01533 puRow = Row(fsize_tab) 01534 try: 01535 statinfo = os.stat(rootf) 01536 fsize = statinfo.st_size 01537 if createNewRow: 01538 createNewRow = False 01539 curRow = fsize_tab.newRow(cand) 01540 01541 if "PILEUP" in step: 01542 puRow.addEntry(realstep,fsize) 01543 else: 01544 if createNewRow: 01545 createNewRow = False 01546 curRow = fsize_tab.newRow(cand) 01547 01548 curRow.addEntry(step,fsize) 01549 except IOError, detail: 01550 print detail 01551 except OSError, detail: 01552 print detail 01553 if puRow == None: 01554 pass 01555 else: 01556 fsize_tab.addRow(puRow,"%s PILEUP" %cand) 01557 (ordered_keys,table_dict) = fsize_tab.getTable(1) 01558 cols = len(ordered_keys) 01559 01560 if len(table_dict) > 1 and cols > 0: 01561 createHTMLtab(INDEX,table_dict,ordered_keys, 01562 "Release ROOT file sizes", 01563 "Table showing current release ROOT filesizes in (k/M/G) bytes.", 01564 "Filesizes",2) 01565 #Case of REGRESSION vs previous release: 01566 else: 01567 try: 01568 globpath = os.path.join(repdir,"REGRESSION.%s.vs.*" % (prevrev)) 01569 globs = glob.glob(globpath) 01570 if len(globs) < 1: 01571 raise IOError(2,globpath,"File does not exist","File does not exist") 01572 idfile = open(globs[0]) 01573 oldpath = "" 01574 for line in idfile: 01575 oldpath = line 01576 oldpath = oldpath.strip() 01577 #print "##########TABLE DEBUG :oldpath is %s"%oldpath 01578 fsize_tab = Table() 01579 01580 for cand in Candles: 01581 fname = CandFname[cand] 01582 globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*.root" % fname) 01583 rootfiles = glob.glob(globpath) 01584 rootfiles = map(lambda x: (fname,x), rootfiles) 01585 rootfiles.sort(cmp=rootfile_cmp) 01586 rootfiles = map(lambda x: x[1], rootfiles) 01587 stepreg = re.compile("%s_(.*).root" % fname) 01588 createNewRow = True 01589 curRow = None 01590 createPURow = True 01591 puRow = None 01592 for rootf in rootfiles: 01593 base = os.path.basename(rootf) 01594 found = stepreg.search(base) 01595 step = "Unknown-step" 01596 if found: 01597 step = found.groups()[0] 01598 01599 realstep = "Unknown-step" 01600 if "PILEUP" in step: 01601 found = pureg.search(step) 01602 if found: 01603 realstep = found.groups()[0] 01604 if createPURow: 01605 createPURow = False 01606 puRow = Row(fsize_tab) 01607 try: 01608 statinfo = os.stat(rootf) 01609 fsize2 = statinfo.st_size 01610 oldfile = os.path.join(oldpath,"%s_TimeSize" % cand,base) 01611 if "PILEUP" in step: 01612 oldfile = os.path.join(oldpath,"%s_PU_TimeSize" % cand,base) 01613 fsize1 = 0 01614 01615 if os.path.exists(oldfile): 01616 statinfo = os.stat(oldfile) 01617 fsize1 = statinfo.st_size 01618 else: 01619 print "######DID NOT FIND Previous file (needed for the filesize table): %s"%oldfile 01620 01621 if createNewRow: 01622 createNewRow = False 01623 curRow = fsize_tab.newRow(cand) 01624 01625 data_tuple = (fsize1,fsize2) 01626 if "PILEUP" in step: 01627 puRow.addEntry(realstep,data_tuple) 01628 else: 01629 if createNewRow: 01630 createNewRow = False 01631 curRow = fsize_tab.newRow(cand) 01632 01633 curRow.addEntry(step,data_tuple) 01634 except IOError, detail: 01635 print detail 01636 except OSError, detail: 01637 print detail 01638 if puRow == None: 01639 pass 01640 else: 01641 fsize_tab.addRow(puRow,"%s PILEUP" %cand) 01642 01643 (ordered_keys,table_dict) = fsize_tab.getTable() 01644 cols = len(ordered_keys) 01645 01646 if len(table_dict) > 1 and cols > 0: 01647 createHTMLtab(INDEX,table_dict,ordered_keys, 01648 "Release ROOT file sizes", 01649 "Table showing previous release ROOT filesizes, fs1, latest sizes, fs2, and the difference between them Δ in (k/M/G) bytes.", 01650 "Filesizes",1) 01651 except IOError, detail: 01652 print detail 01653 except OSError, detail: 01654 print detail 01655 #CPU Time Summary Table 01656 elif cpureg.search(NewFileLine): 01657 #Case of NO REGRESSION 01658 if prevrev == "": 01659 time_tab = Table() 01660 01661 for cand in Candles: 01662 fname = CandFname[cand] 01663 globpath = os.path.join(repdir,"%s*_TimeSize" % cand,"%s_*_TimingReport.log" % fname) 01664 logfiles = glob.glob(globpath) 01665 logfiles = map(lambda x: (fname,x), logfiles) 01666 logfiles.sort(cmp=logrep_cmp) 01667 logfiles = map(lambda x: x[1], logfiles) 01668 01669 stepreg = re.compile("%s_(.*)_TimingReport.log" % fname) 01670 createNewRow = True 01671 curRow = None 01672 createPURow = True 01673 puRow = None 01674 for log in logfiles: 01675 base = os.path.basename(log) 01676 found = stepreg.search(base) 01677 step = "Unknown-step" 01678 if found: 01679 step = found.groups()[0] 01680 01681 realstep = "Unknown-step" 01682 if "PILEUP" in step: 01683 found = pureg.search(step) 01684 if found: 01685 realstep = found.groups()[0] 01686 if createPURow: 01687 createPURow = False 01688 puRow = Row(time_tab) 01689 01690 data = cpr.getTimingLogData(log) 01691 mean = 0 01692 i = 0 01693 for evtnum, time in data: 01694 mean += time 01695 i += 1 01696 try: 01697 mean = mean / float(i) 01698 except ZeroDivisionError, detail: 01699 print "WARNING: Could not calculate mean CPU time from log because no events could be parsed", log 01700 01701 if "PILEUP" in step: 01702 puRow.addEntry(realstep,mean) 01703 else: 01704 if createNewRow: 01705 createNewRow = False 01706 curRow = time_tab.newRow(cand) 01707 01708 curRow.addEntry(step,mean) 01709 if puRow == None: 01710 pass 01711 else: 01712 time_tab.addRow(puRow,"%s PILEUP" %cand) 01713 01714 (ordered_keys,table_dict) = time_tab.getTable(1) 01715 cols = len(ordered_keys) 01716 01717 if len(table_dict) > 1 and cols > 0: 01718 createHTMLtab(INDEX,table_dict,ordered_keys, 01719 "Release CPU times", 01720 "Table showing current release CPU times in secs.", 01721 "CPU Times (s)",3) 01722 #Case of REGRESSION (CPU Time Summary Table) 01723 else: 01724 01725 01726 #### 01727 # 01728 # Create the table data structure 01729 # 01730 cpu_time_tab = populateFromTupleRoot("cpu_time_tuple",repdir,"timing-regress.root",pureg) 01731 01732 01733 ########### 01734 # 01735 # Create HTML table from table data structure 01736 # 01737 01738 (ordered_keys,table_dict) = cpu_time_tab.getTable() 01739 01740 cols = len(ordered_keys) 01741 if len(table_dict) > 1 and cols > 0: 01742 createHTMLtab(INDEX,table_dict,ordered_keys, 01743 "Release CPU times", 01744 "Table showing previous release CPU times, t1, latest times, t2, and the difference between them Δ in secs.", 01745 "CPU Times (s)") 01746 01747 01748 01749 elif lpathreg.search(NewFileLine): 01750 INDEX.write(repdir + "\n") 01751 elif proddreg.search(NewFileLine): 01752 INDEX.write(ExecutionDate + "\n") 01753 elif logfreg.search(NewFileLine): 01754 INDEX.write("<br />\n") 01755 for log in LogFiles: 01756 log = os.path.basename(log) 01757 if _verbose: 01758 print "linking log file %s" % log 01759 INDEX.write("<a href=\"./%s\"> %s </a>" % (log,log)) 01760 INDEX.write("<br />\n") 01761 #Add the cmsScimark results here: 01762 INDEX.write("Results for cmsScimark2 benchmark (running on the other cores) available at:\n") 01763 for cmssci in cmsScimarkResults: 01764 scimarkdirs=cmssci.split("/") 01765 localdir=scimarkdirs[-2] 01766 #print localdir 01767 cmssci = os.path.basename(cmssci) 01768 relativelink=os.path.join(localdir,cmssci) 01769 #print relativelink 01770 INDEX.write("<a href=\"%s\"> %s </a>" % (relativelink,cmssci)) 01771 INDEX.write("<br />\n") 01772 01773 01774 elif dirbreg.search(NewFileLine): 01775 #Create a subdirectory DirectoryBrowsing to circumvent the fact the dir is not browsable if there is an index.html in it. 01776 #Bug! This does not work since it points to index.html automatically! 01777 #os.symlink("./","%s/DirectoryBrowsing" % WebArea) 01778 #Actually all the following is done later... 01779 #Create a physical directory first 01780 #os.mkdir("%s/DirectoryBrowsing" % WebArea) 01781 #Then will populate it with symbolic links(once they have been copied there!) from all WebArea files, except index.html, see down towards the end! 01782 INDEX.write("Click <a href=\"./DirectoryBrowsing/\">here</a> to browse the directory containing all results (except the root files)\n") 01783 01784 elif pubdreg.search(NewFileLine): 01785 INDEX.write(date + "\n") 01786 elif candhreg.search(NewFileLine): 01787 for acandle in Candle: 01788 globpath = os.path.join(repdir,"%s_*" % acandle) 01789 globs = glob.glob(globpath) 01790 if len(globs) > 0: 01791 candlHTML = "%s.html" % acandle 01792 INDEX.write("<a href=\"./%s\"> %s </a>" % (candlHTML,acandle)) 01793 INDEX.write("<br />\n") 01794 01795 candlHTML=os.path.join(WebArea,candlHTML) 01796 createCandlHTML(CandlTmpltHTML,candlHTML,acandle,WebArea,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev) 01797 else: 01798 INDEX.write(NewFileLine) 01799 01800 #End of while loop on template html file 01801 INDEX.close() 01802 except IOError, detail: 01803 print "Error: Could not create index Html file for some reason, check position. Details : %s" % detail
| def cmsPerfPublish::delTmpDir | ( | ) |
Delete tmp dir if we used it.
Definition at line 1847 of file cmsPerfPublish.py.
| def cmsPerfPublish::dirname_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 727 of file cmsPerfPublish.py.
00728 : 00729 (candle,prof,x) = x 00730 (candle,prof,y) = y 00731 x = os.path.basename(x) 00732 y = os.path.basename(y) 00733 stepreg = re.compile("%s_(..*)_%s" % (candle,prof)) 00734 if stepreg.search(x): 00735 x = stepreg.search(x).groups()[0] 00736 if stepreg.search(y): 00737 y = stepreg.search(y).groups()[0] 00738 return step_cmp(x,y)
| def cmsPerfPublish::docopy | ( | src, | |
| dest | |||
| ) |
Definition at line 1897 of file cmsPerfPublish.py.
01898 : 01899 try: 01900 copy2(src,dest) 01901 except OSError, detail: 01902 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail) 01903 except IOError, detail: 01904 print "WARNING: Could not copy %s to %s because %s" % (src,dest,detail) 01905 else: 01906 if _verbose: 01907 print "Copied %s to %s" % (src,dest)
| def cmsPerfPublish::fail | ( | errstr = "" | ) |
Definition at line 64 of file cmsPerfPublish.py.
Referenced by CalibrationTask::checkAndSave(), CalibrationScanTask::checkAndSave(), HLTLevel1GTSeed::convertStringToBitNumber(), EgHLTOfflineClient::createLooseTightTrigEff(), TagProbeFitter::doCntEfficiency(), editESInputTag(), editInputTag(), editNumericParameter(), editVInputTag(), SurveyPxbImage::fill(), ZeeCandidateFilter::filter(), dqmTnP::GaussianPlusLinearFitter::fit(), dqmTnP::VoigtianPlusExponentialFitter::fit(), lumi::from_string(), HcalDbOmds::from_string(), SurveyResidual::inverseCovariance(), EcalUncalibRecHitFixedAlphaBetaAlgo< C >::PerformAnalyticFit(), ContentsYRange::runTest(), ContentsXRange::runTest(), ContentsWithinExpected::runTest(), DeadChannel::runTest(), NoisyChannel::runTest(), SimpleCosmicBONSeeder::seeds(), cscdqm::stringToNumber(), TagProbeFitTreeAnalyzer::TagProbeFitTreeAnalyzer(), L1GtTriggerMenuXmlParser::workAlgorithm(), and L1GtTriggerMenuXmlParser::workTechTrigger().
| def cmsPerfPublish::get_environ | ( | ) |
Get require environment variables.
Definition at line 301 of file cmsPerfPublish.py.
00302 : 00303 global CMSSW_VERSION, CMSSW_RELEASE_BASE, CMSSW_BASE, HOST, USER, BASE_PERFORMANCE, CMSSW_WORK 00304 global DEF_RELVAL, DEF_SIMUL 00305 00306 try: 00307 CMSSW_VERSION=os.environ['CMSSW_VERSION'] 00308 CMSSW_RELEASE_BASE=os.environ['CMSSW_RELEASE_BASE'] 00309 CMSSW_BASE=os.environ['CMSSW_BASE'] 00310 HOST=os.environ['HOST'] 00311 USER=os.environ['USER'] 00312 CMSSW_WORK = os.path.join(CMSSW_BASE,"work/Results") 00313 except KeyError, detail: 00314 fail("ERROR: Could not retrieve some necessary environment variables. Have you ran scramv1 runtime -csh yet?. Details: %s" % detail) 00315 00316 LocalPath=getcmdBasic("pwd") 00317 ShowTagsResult=getcmdBasic("showtags -r") 00318 00319 #Adding a check for a local version of the packages 00320 PerformancePkg="%s/src/Validation/Performance" % CMSSW_BASE 00321 if (os.path.exists(PerformancePkg)): 00322 BASE_PERFORMANCE=PerformancePkg 00323 print "**Using LOCAL version of Validation/Performance instead of the RELEASE version**" 00324 else: 00325 BASE_PERFORMANCE="%s/src/Validation/Performance" % CMSSW_RELEASE_BASE 00326 00327 return (LocalPath,ShowTagsResult)
| def cmsPerfPublish::getArchVersionFromLog | ( | logfile | ) |
Another very fragile function to get the architecture and the CMSSW version parsing the logfile...
Definition at line 475 of file cmsPerfPublish.py.
00476 : 00477 '''Another very fragile function to get the architecture and the CMSSW version parsing the logfile...''' 00478 log=open(logfile,"r") 00479 arch=re.compile("^Current Architecture is") 00480 version=re.compile("^Current CMSSW version is") 00481 CMSSW_arch="UNKNOWN_ARCH" 00482 CMSSW_version="UNKNOWN_VERSION" 00483 for line in log: 00484 if arch.search(line): 00485 CMSSW_arch=line.split()[3] 00486 if version.search(line): 00487 CMSSW_version=line.split()[4] 00488 return(CMSSW_arch,CMSSW_version) 00489
| def cmsPerfPublish::getcmd | ( | command | ) |
Definition at line 83 of file cmsPerfPublish.py.
| def cmsPerfPublish::getcmdBasic | ( | cmd | ) |
Definition at line 78 of file cmsPerfPublish.py.
| def cmsPerfPublish::getDate | ( | ) |
Definition at line 75 of file cmsPerfPublish.py.
| def cmsPerfPublish::getDirnameDirs | ( | repdir, | |
| WebArea | |||
| ) |
Grab dirs that end in strings defined in DirName.
Definition at line 1808 of file cmsPerfPublish.py.
01809 : 01810 Dir = os.listdir(repdir) 01811 def _containsDirName(elem): 01812 return reduce(lambda x,y: x or y,map(lambda x: x in elem, DirName)) 01813 def _print4Lambda(elem,WebArea): 01814 if _verbose: 01815 print "Copying %s to %s\n" % (elem,WebArea) 01816 01817 dirstocp = filter(lambda x: _containsDirName(x),map(lambda x: repdir + x,Dir)) 01818 map(lambda x: _print4Lambda(x,WebArea),dirstocp) 01819 syscp(dirstocp,WebArea + "/") 01820 os.mkdir("%s/DirectoryBrowsing" % WebArea) 01821 for file in os.listdir(WebArea): 01822 if file != "index.html": #Potential maintenance issue if the index.html changes name to something the server automatically displays when pointing to the directory... 01823 #Use relative path ".." instead of WebArea to avoid problems when copying stuff to a remote server! 01824 os.symlink("%s/%s"%("..",file),"%s/DirectoryBrowsing/%s" % (WebArea,file))
| def cmsPerfPublish::getNumOfEventsFromLog | ( | logfile | ) |
A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!
Definition at line 450 of file cmsPerfPublish.py.
00451 : 00452 '''A very fragile function to get the Number of events for each test by parsing the logfile of the Suite. This relies on the fact that nobody will turn off the print out of the options in the cmsPerfSuite.py output... ARGH!''' 00453 log=open(logfile,"r") 00454 TimeSizeEvents=0 00455 IgProfEvents=0 00456 CallgrindEvents=0 00457 MemcheckEvents=0 00458 for line in log: 00459 #FIXME: 00460 #For robustness could read this from the Launching the X tests (.....) with N events each and keep that format decent for parsing. 00461 #One more place where XML would seem a better choice to extract information (Timing of the performance suite in general is also, but publishing and harvesting some other info as well). 00462 if 'TimeSizeEvents' in line and not TimeSizeEvents: 00463 lineitems=line.split() 00464 TimeSizeEvents=lineitems[lineitems.index('TimeSizeEvents')+1] 00465 if 'IgProfEvents' in line and not IgProfEvents: 00466 lineitems=line.split() 00467 IgProfEvents=lineitems[lineitems.index('IgProfEvents')+1] 00468 if 'CallgrindEvents' in line and not CallgrindEvents: 00469 lineitems=line.split() 00470 CallgrindEvents=lineitems[lineitems.index('CallgrindEvents')+1] 00471 if 'MemcheckEvents' in line and not MemcheckEvents: 00472 lineitems=line.split() 00473 MemcheckEvents=lineitems[lineitems.index('MemcheckEvents')+1] 00474 return (TimeSizeEvents,IgProfEvents,CallgrindEvents,MemcheckEvents)
| def cmsPerfPublish::getOutputNames | ( | base, | |
| reportName | |||
| ) |
Definition at line 707 of file cmsPerfPublish.py.
| def cmsPerfPublish::getRelativeDir | ( | parent, | |
| child, | |||
keepTop = True |
|||
| ) |
Some functions used for copying.
Definition at line 1855 of file cmsPerfPublish.py.
01856 : 01857 def _walkpath(path): 01858 dirs = [] 01859 while True: 01860 head , tail = os.path.split(path) 01861 if tail == "": 01862 break 01863 dirs.append(tail) 01864 path = head 01865 for i in range(len(dirs)-1,-1,-1): 01866 adir = dirs[i] 01867 yield adir 01868 return 01869 pwalk = _walkpath(parent) 01870 n = 0 01871 try: 01872 while True: 01873 pwalk.next() 01874 n += 1 01875 except StopIteration: 01876 pass 01877 01878 if keepTop: 01879 n = n - 1 01880 01881 cwalk = _walkpath(child) 01882 try: 01883 #prewalk 01884 for x in range(n): 01885 cwalk.next() 01886 except StopIteration: 01887 print "ERROR: Unable to determine relative dir" 01888 raise ReldirExcept 01889 01890 relpath = "" 01891 try: 01892 while True: 01893 relpath=os.path.join(relpath,cwalk.next()) 01894 except StopIteration: 01895 pass 01896 return relpath
| def cmsPerfPublish::getStageRepDirs | ( | options, | |
| args | |||
| ) |
Determine locations of staging and report dirs.
Definition at line 494 of file cmsPerfPublish.py.
00495 : 00496 global TMP_DIR, IS_TMP, DEF_LOCAL, CMSSW_VERSION 00497 DEF_LOCAL = CMSSW_WORK 00498 numofargs = len(args) 00499 00500 repdir = os.path.abspath(options.repdir) 00501 repdir = addtrailingslash(repdir) 00502 00503 if not os.path.exists(repdir): 00504 fail("ERROR: The specified report directory %s to retrieve report information from does not exist, exiting" % repdir) 00505 00506 previousrev = options.previousrev 00507 if previousrev == "": 00508 regressfiles = glob.glob("%s/REGRESSION.*.vs.*" % repdir) 00509 if not len(regressfiles) == 0: 00510 regressID = regressfiles[0] 00511 base = os.path.basename(regressID) 00512 split = base.split(".") 00513 previousrev = split[1] 00514 currentrel = split[3] 00515 print "Regression Identification file exists, renaming report title for regression report. Old ver: %s" % previousrev 00516 else: 00517 print "No regression ID file exists and previous release name was not specified. Producing normal report." 00518 else: 00519 print "Previous release name was specified, renaming report title for regression report. Old ver %s" % previousrev 00520 00521 uri = "" 00522 defaultlocal = False 00523 if options.simulation: 00524 uri = DEF_SIMUL 00525 elif options.relval: 00526 uri = DEF_RELVAL 00527 elif numofargs >= 1: 00528 uri = args[0] # Add directory CMSSW_VERSION later in temp! Not now, otherwise we get into a mess if this is a remote dir 00529 else: 00530 defaultlocal = True 00531 uri = DEF_LOCAL 00532 00533 #### 00534 # 00535 # Determine if location is remote 00536 # 00537 # Try not to re-arrange we don't want to assume that default locations are not remote 00538 # 00539 #### 00540 00541 drive = "" 00542 path = "" 00543 if ":" in uri: 00544 drive, path = uri.split(":",1) 00545 else: 00546 path = uri 00547 00548 if drive == "": 00549 path = os.path.abspath(path) 00550 remote = not drive == "" 00551 00552 if remote: 00553 unResolved = True 00554 try: 00555 socket.getaddrinfo(drive,53) 00556 unResolved = False 00557 except socket.gaierror: 00558 unResolved = True 00559 00560 # try see if it's an ipaddress 00561 if unResolved: 00562 try: 00563 socket.gethostbyaddr(drive) 00564 unResolved = False 00565 except socket.gaierror: 00566 unResolved = True 00567 if unResolved: 00568 print "ERROR: Can not determine your hostname or ipv{4,6} address %s" % drive 00569 if not (_dryrun or _test): 00570 fail("exiting...") 00571 00572 if (not remote) and (not options.port == 873) : 00573 print "WARNING: Can not use a port if not performing a remote copy, ignoring" 00574 port = options.port 00575 00576 ### 00577 # 00578 # Determine Staging Area 00579 # 00580 00581 StagingArea="" 00582 localExists = os.path.exists("%s/%s" % (CMSSW_WORK,CMSSW_VERSION)) 00583 00584 if remote: 00585 #Cannot use this since the /tmp is limited to 2GB on lxbuild machines! 00586 #TMP_DIR=tmp.mkdtemp(prefix="/tmp/%s" % PROG_NAME) 00587 TMP_DIR=tmp.mkdtemp(prefix="/build/%s" % PROG_NAME) 00588 StagingArea = TMP_DIR 00589 #Local but dir already exists 00590 elif defaultlocal and localExists: 00591 TMP_DIR=tmp.mkdtemp(prefix="%s/%s" % (CMSSW_WORK,CMSSW_VERSION)) 00592 StagingArea = TMP_DIR 00593 print "WARNING: %s already exists, creating a temporary staging area %s" % (CMSSW_WORK,TMP_DIR) 00594 #Local cases 00595 elif defaultlocal: 00596 StagingArea = CMSSW_WORK 00597 try: 00598 os.mkdir(os.path.join(CMSSW_BASE,"work")) 00599 os.mkdir(os.path.join(CMSSW_BASE,"work","Results")) 00600 except OSError: 00601 pass 00602 print "**User did not specify location of results, staging in default %s**" % StagingArea 00603 else: 00604 print "**User chose to publish results in a local directory**" 00605 StagingArea = path 00606 if not os.path.exists(path): 00607 try: 00608 os.mkdir("%s" % path) 00609 except OSError, detail: 00610 if detail.errno == 13: 00611 fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea) 00612 elif detail.errno == 17: 00613 #If the directory already exists just carry on 00614 pass 00615 else: 00616 fail("ERROR: There was some problem (%s) when creating the staging directory" % detail) 00617 00618 IS_TMP = not TMP_DIR == "" 00619 ###### 00620 # 00621 # create Version dir 00622 # This is why when we create a Tmp dir we get the version(tmpname)/version 00623 # structure. We should remove this if we don't want it but i dont think it matters 00624 StagingArea="%s/%s" % (StagingArea,CMSSW_VERSION) 00625 try: 00626 os.mkdir("%s" % StagingArea) 00627 except OSError, detail: 00628 if detail.errno == 13: 00629 fail("ERROR: Failed to create staging area %s because permission was denied " % StagingArea) 00630 elif detail.errno == 17: 00631 #If the directory already exists just carry on 00632 pass 00633 else: 00634 fail("ERROR: There was some problem (%s) when creating the staging directory" % detail) 00635 00636 return (drive,path,remote,StagingArea,port,repdir,previousrev,options.ig_remotedir)
| def cmsPerfPublish::logrep_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 751 of file cmsPerfPublish.py.
00752 : 00753 (fname,x) = x 00754 (fname,y) = y 00755 x = os.path.basename(x) 00756 y = os.path.basename(y) 00757 stepreg = re.compile("%s_(..*)_TimingReport.log" % fname) 00758 if stepreg.search(x): 00759 x = stepreg.search(x).groups()[0] 00760 if stepreg.search(y): 00761 y = stepreg.search(y).groups()[0] 00762 return step_cmp(x,y)
| def cmsPerfPublish::main | ( | ) |
Definition at line 202 of file cmsPerfPublish.py.
00203 : 00204 global TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents 00205 #Bad design... why is this function defined here? 00206 #Either no function, or define somewhere else. 00207 def _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage): 00208 """Use function syscp to copy LogFiles and cmsScimarkDir over to staging area""" 00209 if _verbose: 00210 print "Copying the logfiles to %s/." % stage 00211 print "Copying the cmsScimark2 results to the %s/." % stage 00212 00213 syscp(LogFiles , stage + "/") 00214 syscp(cmsScimarkDir, stage + "/") 00215 00216 #Same comment as above about the opportunity of this function definition here. 00217 def _createLogFile(LogFile,date,LocalPath,ShowTagsResult): 00218 """Creating a small logfile with basic publication script running information (never used really by other scripts in the suite).""" 00219 try: 00220 LOG = open(LogFile,"w") 00221 if _verbose: 00222 print "Writing Production Host, Location, Release and Tags information in %s" % LogFile 00223 LOG.write("These performance tests were executed on host %s and published on %s" % (HOST,date)) 00224 LOG.write("They were run in %s" % LocalPath) 00225 LOG.write("Results of showtags -r in the local release:\n%s" % ShowTagsResult) 00226 LOG.close() 00227 except IOError, detail: 00228 print "WARNING: Can't create log file" 00229 print detail 00230 00231 # Print Program header 00232 print_header() 00233 00234 # Get environment variables 00235 #FIXME: should check into this and make sure the proper variables for the tests being published are read from logfile (case of deprecated releases...) 00236 print "\n Getting Environment variables..." 00237 (LocalPath, ShowTagsResult) = get_environ() 00238 00239 # Parse options 00240 (options,args) = optionparse() 00241 00242 # Determine program parameters and input/staging locations 00243 print "\n Determining locations for input and staging..." 00244 (drive,path,remote,stage,port,repdir,prevrev,igprof_remotedir) = getStageRepDirs(options,args) 00245 00246 #Get the number of events for each test from logfile: 00247 print "\n Getting the number of events for each test..." 00248 #Let's do a quick implementation of something that looks at the logfile: 00249 cmsPerfSuiteLogfile="%s/cmsPerfSuite.log"%repdir 00250 00251 if os.path.exists(cmsPerfSuiteLogfile): 00252 try: 00253 (TimeSizeNumOfEvents,IgProfNumOfEvents,CallgrindNumOfEvents,MemcheckNumOfEvents)=getNumOfEventsFromLog(cmsPerfSuiteLogfile) 00254 #Get the CMSSW version and SCRAM architecture from log (need these for the IgProf new publishing with igprof-navigator) 00255 (CMSSW_arch,CMSSW_version)=getArchVersionFromLog(cmsPerfSuiteLogfile) 00256 #For now keep the dangerous default? Better set it to a negative number... 00257 except: 00258 print "There was an issue in reading out the number of events for the various tests or the architecture/CMSSW version using the standard logfile %s"%cmsPerfSuiteLogFile 00259 print "Check that the format was not changed: this scripts relies on the initial perfsuite arguments to be dumped in the logfile one per line!" 00260 print "For now taking the default values for all tests (0)!" 00261 00262 print "\n Scan report directory..." 00263 # Retrieve some directories and information about them 00264 (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir) = scanReportArea(repdir) 00265 print "cmsScimarkResults are %s"%cmsScimarkResults 00266 print "\n Copy report files to staging directory..." 00267 # Copy reports to staging area 00268 _copyReportsToStaging(repdir,LogFiles,cmsScimarkDir,stage) 00269 00270 print "\n Creating log file..." 00271 # Produce a small logfile with basic info on the Production area 00272 _createLogFile("%s/ProductionLog.txt" % stage,date,repdir,ShowTagsResult) 00273 00274 #Check if there are IgProf tests: 00275 for dirname in os.listdir(repdir): 00276 if "IgProf" in dirname: 00277 print "\n Handling IgProf reports..." 00278 # add a function to handle the IgProf reports 00279 stageIgProfReports(igprof_remotedir,CMSSW_arch,CMSSW_version) 00280 00281 print "\n Creating HTML files..." 00282 # create HTML files 00283 createWebReports(stage,repdir,ExecutionDate,LogFiles,cmsScimarkResults,date,prevrev) 00284 00285 print "\n Copy profiling logs to staging directory..." 00286 # Copy over profiling logs... 00287 getDirnameDirs(repdir,stage) 00288 00289 # Send files to remote location 00290 if remote: 00291 print "\n Uploading web report to remote location..." 00292 syncToRemoteLoc(stage,drive,path,port) 00293 print "\n Finished uploading! Now removing staging directory..." 00294 delTmpDir() 00295 00296 print "\n Finished!!!"
| def cmsPerfPublish::optionparse | ( | ) |
Option parser.
Definition at line 331 of file cmsPerfPublish.py.
00332 : 00333 global PROG_NAME, _debug, _dryrun, _verbose 00334 00335 parser = opt.OptionParser(usage=("""%s [HOST:]DEST_PATH [Options] 00336 00337 Arguments: 00338 [HOST:]DEST_PATH - This is where the report should be published, you can specify a local path or a directory on a remote machine (HOST is optional) 00339 00340 Examples: 00341 Publish report to default local directory 00342 ./%s 00343 Publish report to \"/some/local/dir\" 00344 ./%s /some/local/dir 00345 Publish report to \"/some/other/dir\" remote host \"hal.cern.ch\" 00346 ./%s hal.cern.ch:/some/other/dir 00347 Publish report to default relval location (this could be remote or local depending on the hardcoded default) 00348 ./%s --relval""" 00349 % ( PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME, PROG_NAME))) 00350 00351 devel = opt.OptionGroup(parser, "Developer Options", 00352 "Caution: use these options at your own risk." 00353 "It is believed that some of them bite.\n") 00354 parser.add_option( 00355 '--relval', 00356 action="store_true", 00357 dest='relval', 00358 help='Use the default RelVal location', 00359 #metavar='<STEPS>', 00360 ) 00361 00362 parser.add_option( 00363 '-v', 00364 '--verbose', 00365 action="store_true", 00366 dest='verbose', 00367 help='output more information', 00368 #metavar='<STEPS>', 00369 ) 00370 00371 parser.add_option( 00372 '--simul', 00373 action="store_true", 00374 dest='simulation', 00375 help='Use the default simulation location', 00376 #metavar='<STEPS>', 00377 ) 00378 00379 parser.add_option( 00380 '--prev', 00381 type="string", 00382 dest='previousrev', 00383 help='The override the name of the previous release. Default is the string obtain from the identification file REGRESSION.<prevrel>.vs.<newrel>', 00384 metavar='<NAME>', 00385 default="", 00386 ) 00387 00388 parser.add_option( 00389 '--input', 00390 type="string", 00391 dest='repdir', 00392 help='The location of the report files to be published', 00393 metavar='<DIR>' 00394 ) 00395 00396 parser.add_option( 00397 '-p', 00398 '--port', 00399 type='int', 00400 dest='port', 00401 help='Use a particular port number to rsync material to a remote server', 00402 metavar='<PORT>' 00403 ) 00404 parser.add_option( 00405 '--igprof', 00406 type='string', 00407 dest='ig_remotedir', 00408 default='IgProfData', #Reverting to local default publication for now#'/afs/cern.ch/cms/sdt/web/qa/igprof-testbed/data', #For now going straight into AFS... later implement security via local disk on cmsperfvm and cron job there: 00409 #default='cmsperfvm:/data/projects/conf/PerfSuiteDB/IgProfData', #This should not change often! In this virtual machine a cron job will run to move stuff to AFS. 00410 help='Specify an AFS or host:mydir remote directory instead of default one', 00411 metavar='<IGPROF REMOTE DIRECTORY>' 00412 ) 00413 00414 devel.add_option( 00415 '-d', 00416 '--debug', 00417 type='int', 00418 dest='debug', 00419 help='Show debug output', 00420 #metavar='DEBUG', 00421 ) 00422 00423 devel.add_option( 00424 '--dry-run', 00425 action="store_true", 00426 dest='dryrun', 00427 help='Do not send files to remote server, but run everything else', 00428 #metavar='DEBUG', 00429 ) 00430 00431 repdirdef = os.getcwd() 00432 parser.set_defaults(debug=0,simulation=False,relval=False,port=873,pretend=False,repdir=repdirdef,verbose=False) 00433 parser.add_option_group(devel) 00434 00435 (options, args) = parser.parse_args() 00436 00437 _debug = options.debug 00438 _dryrun = options.dryrun 00439 _verbose = options.verbose 00440 00441 numofargs = len(args) 00442 00443 if (options.simulation and options.relval) or ((options.simulation or options.relval) and numofargs >= 1): 00444 parser.error("You can not specify simulation and relval together. Neither can you specify simulation or relval AND a path") 00445 sys.exit() 00446 00447 return (options, args) 00448 00449 #A function to get the number of events for each test from the logfile:
| def cmsPerfPublish::populateFromTupleRoot | ( | tupname, | |
| repdir, | |||
| rootfile, | |||
| pureg | |||
| ) |
Definition at line 1239 of file cmsPerfPublish.py.
01240 : 01241 table = Table() 01242 for cand in Candles: 01243 fname = CandFname[cand] 01244 globpath = os.path.join(repdir,"%s_TimeSize" % cand,"%s_*_TimingReport" % fname) 01245 stepDirs = glob.glob(globpath) 01246 stepDirs = map(lambda x: (fname,x), stepDirs) 01247 stepDirs.sort(cmp=timerep_cmp) 01248 stepDirs = map(lambda x: x[1], stepDirs) 01249 stepreg = re.compile("%s_(.*)_TimingReport" % fname) 01250 createNewRow = True 01251 curRow = None 01252 createPURow = True 01253 puRow = None 01254 for stepdir in stepDirs: 01255 base = os.path.basename(stepdir) 01256 found = stepreg.search(base) 01257 step = "Unknown-step" 01258 if found: 01259 step = found.groups()[0] 01260 realstep = "Unknown-step" 01261 if "PILEUP" in step: 01262 found = pureg.search(step) 01263 if found: 01264 realstep = found.groups()[0] 01265 if createPURow: 01266 createPURow = False 01267 puRow = Row(table) 01268 rootf = os.path.join(stepdir,rootfile) 01269 01270 if os.path.exists(rootf): 01271 f = ROOT.TFile(rootf) 01272 01273 cpu_time_tree = ROOT.TTree() 01274 f.GetObject("cpu_time_tuple;1",cpu_time_tree) 01275 if cpu_time_tree: 01276 if cpu_time_tree.InheritsFrom("TTree"): 01277 data1 = None 01278 data2 = None 01279 for t in cpu_time_tree: 01280 data1 = t.total1 01281 data2 = t.total2 01282 if data1 and data2: 01283 if createNewRow: 01284 createNewRow = False 01285 curRow = table.newRow(cand) 01286 data_tuple = (data1,data2) 01287 01288 if "PILEUP" in step: 01289 puRow.addEntry(realstep,data_tuple) 01290 else: 01291 if createNewRow: 01292 createNewRow = False 01293 curRow = table.newRow(cand) 01294 01295 curRow.addEntry(step,data_tuple) 01296 f.Close() 01297 if puRow == None: 01298 pass 01299 else: 01300 table.addRow(puRow,"%s PILEUP" %cand) 01301 return table 01302
| def cmsPerfPublish::prettySize | ( | size | ) |
Definition at line 90 of file cmsPerfPublish.py.
00091 : 00092 nega = size < 0 00093 if nega: 00094 size = -size 00095 suffixes = [("B",2**10), ("k",2**20), ("M",2**30), ("G",2**40), ("T",2**50)] 00096 for suf, lim in suffixes: 00097 if size > lim: 00098 continue 00099 else: 00100 if nega: 00101 return "-" + round(size/float(lim/2**10),2).__str__() + suf 00102 else: 00103 return round(size/float(lim/2**10),2).__str__()+suf
| def cmsPerfPublish::print_header | ( | ) |
Definition at line 1989 of file cmsPerfPublish.py.
| def cmsPerfPublish::reg_dirname_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 739 of file cmsPerfPublish.py.
00740 : 00741 (candle,prof,x) = x 00742 (candle,prof,y) = y 00743 x = os.path.basename(x) 00744 y = os.path.basename(y) 00745 stepreg = re.compile("%s_(..*)_%s_regression" % (candle,prof)) 00746 if stepreg.search(x): 00747 x = stepreg.search(x).groups()[0] 00748 if stepreg.search(y): 00749 y = stepreg.search(y).groups()[0] 00750 return step_cmp(x,y)
| def cmsPerfPublish::rootfile_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 715 of file cmsPerfPublish.py.
00716 : 00717 (fname,x) = x 00718 (fname,y) = y 00719 x = os.path.basename(x) 00720 y = os.path.basename(y) 00721 stepreg = re.compile("%s_(..*)\.root" % fname) 00722 if stepreg.search(x): 00723 x = stepreg.search(x).groups()[0] 00724 if stepreg.search(y): 00725 y = stepreg.search(y).groups()[0] 00726 return step_cmp(x,y)
| def cmsPerfPublish::scanReportArea | ( | repdir | ) |
Scan report area for required things.
Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results. It returns Execution date (completion), current date, list of logfiles and cmsScimark results
Definition at line 641 of file cmsPerfPublish.py.
00642 : 00643 """Scans the working directory for cms*.logs (cmsPerfSuite.log and cmsScimark*.log, and cmsScimark results. 00644 It returns Execution date (completion), current date, list of logfiles and cmsScimark results""" 00645 date=getDate() 00646 LogFiles = glob.glob(repdir + "cms*.log") 00647 if _verbose: 00648 print "Found the following log files:" 00649 print LogFiles 00650 00651 cmsScimarkDir = glob.glob(repdir + "cmsScimarkResults_*") 00652 if _verbose: 00653 print "Found the following cmsScimark2 results directories:" 00654 print cmsScimarkDir 00655 00656 cmsScimarkResults = [] 00657 for adir in cmsScimarkDir: 00658 htmlfiles = glob.glob(adir + "/*.html") 00659 #FIXME: 00660 #Really unnecessary use of map in my opinion 00661 #Could do with cmsScimarkResults.extend(htmlfiles) 00662 map(cmsScimarkResults.append,htmlfiles) 00663 00664 ExecutionDateLast = "" 00665 ExecutionDate = "" 00666 ExecutionDateSec=0 00667 cmsreg = re.compile("^cmsPerfSuite") 00668 for logf in LogFiles: 00669 if cmsreg.search(logf): 00670 ExecutionDateLastSec = os.stat(logf)[ST_CTIME] 00671 ExecutionDateLast = os.stat(logf)[ST_MTIME] 00672 if _verbose: 00673 print "Execution (completion) date for %s was: %s" % (logf,ExecutionDateLast) 00674 if (ExecutionDateLastSec > ExecutionDateSec): 00675 ExecutionDateSec = ExecutionDateLastSec 00676 ExecutionDate = ExecutionDateLast 00677 00678 if ExecutionDate == "": 00679 ExecutionDate = ExecutionDateLast 00680 00681 return (ExecutionDate,LogFiles,date,cmsScimarkResults,cmsScimarkDir)
| def cmsPerfPublish::stageIgProfReports | ( | remotedir, | |
| arch, | |||
| version | |||
| ) |
Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.
Definition at line 1394 of file cmsPerfPublish.py.
01395 : 01396 '''Publish all IgProf files into one remote directory (new naming convention). Can publish to AFS location or to a local directory on a remote (virtual) machine.''' 01397 #FIXME: can eliminate this part if using tar pipes... was done with rsynch in mind 01398 #Compose command to create remote dir: 01399 if ":" in remotedir: #Remote host local directory case 01400 (host,dir)=remotedir.split(":") 01401 mkdir_cmd="ssh %s (mkdir %s;mkdir %s/%s)"%(host,dir,dir,arch) 01402 else: #AFS or local case 01403 mkdir_cmd="mkdir %s;mkdir %s/%s"%(remotedir,remotedir,arch) 01404 01405 #Create remote dir: 01406 try: 01407 print mkdir_cmd 01408 os.system(mkdir_cmd) 01409 print "Successfully created publication directory" 01410 except: 01411 print "Issues with publication directory existence/creation!" 01412 01413 #Copy files over to remote dir 01414 #replacing rsync with tar pipes since it can hang on AFS (Andreas' experience): 01415 #rsync_cmd="rsync -avz *_IgProf_*/*.sql3 %s/%s/%s"%(remotedir,arch,version) 01416 if ":" in remotedir: 01417 tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | ssh %s "cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*"'%(host,dir,arch,version,version) 01418 else: 01419 tarpipe_cmd='tar cf - *_IgProf_*/*.sql3 | (cd %s/%s; mkdir %s; cd %s; tar xf -; mv *_IgProf_*/*.sql3 .; rmdir *_IgProf_*)'%(remotedir,arch,version,version) 01420 try: 01421 # print rsync_cmd 01422 # os.system(rsync_cmd) 01423 print tarpipe_cmd 01424 os.system(tarpipe_cmd) 01425 print "Successfully copied IgProf reports to %s"%remotedir 01426 except: 01427 print "Issues with rsyncing to the remote directory %s!"%remotedir 01428 01429 #Make sure permissions are set for group to be able to write: 01430 if ":" in remotedir: #Remote host local directory case 01431 chmod_cmd="ssh %s chmod -R 775 %s/%s"%(host,dir,arch) 01432 else: 01433 chmod_cmd="chmod -R 775 %s/%s"%(remotedir,arch) 01434 try: 01435 print chmod_cmd 01436 os.system(chmod_cmd) 01437 print "Successfully set permissions for IgProf reports directory %s"%remotedir 01438 except: 01439 print "(Potential) issues with chmoding the remote directory %s!"%remotedir 01440 01441 return #Later, report here something like the web link to the reports in igprof-navigator... 01442
| def cmsPerfPublish::step_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 775 of file cmsPerfPublish.py.
00776 : 00777 xstr = x 00778 ystr = y 00779 x_idx = -1 00780 y_idx = -1 00781 bestx_idx = -1 00782 besty_idx = -1 00783 sndbst_x = -1 00784 sndbst_y = -1 00785 last_x = -1 00786 last_y = -1 00787 for i in range(len(Step)): 00788 stepreg = re.compile("^%s.*" % Step[i]) 00789 # fallback 00790 if Step[i] in xstr and sndbst_x == -1: 00791 last_x = i 00792 if Step[i] in ystr and sndbst_y == -1: 00793 last_y = i 00794 # second best 00795 if xstr in Step[i] and bestx_idx == -1: 00796 sndbst_x = i 00797 if ystr in Step[i] and besty_idx == -1: 00798 sndbst_y = i 00799 # next best 00800 if stepreg.search(xstr) and x_idx == -1: 00801 bestx_idx = i # If an exact match has not been found but something similar has, set x best index 00802 if stepreg.search(ystr) and y_idx == -1: 00803 besty_idx = i # If an exact match has not been found but something similar has, set y best index 00804 # actual 00805 if Step[i] == xstr and x_idx == -1: 00806 x_idx = i # if an exact match has been found then set x index 00807 if Step[i] == ystr and y_idx == -1: 00808 y_idx = i # if an exact match has been found then set y index 00809 if not ( x_idx == -1 or y_idx == -1): 00810 break # if an exact match has been found for both, stop 00811 00812 # use best match if we still can't find indices 00813 if x_idx == -1: 00814 x_idx = bestx_idx 00815 if y_idx == -1: 00816 y_idx = besty_idx 00817 00818 # use second best if we still can't find indices 00819 if x_idx == -1: 00820 x_idx = sndbst_x 00821 if y_idx == -1: 00822 y_idx = sndbst_y 00823 00824 # use fallback if we still can't find indices 00825 if x_idx == -1: 00826 x_idx = last_x 00827 if y_idx == -1: 00828 y_idx = last_y 00829 00830 if x_idx == -1 or y_idx == -1: 00831 print "WARNING: No valid step names could be found in the logfiles or root filenames being sorted: x: %s y: %s." % (xstr,ystr) 00832 print "x", x_idx, "y", y_idx 00833 00834 if x_idx < y_idx: 00835 return -1 00836 elif x_idx == y_idx: 00837 return 0 00838 elif y_idx < x_idx: 00839 return 1 00840
| def cmsPerfPublish::syncToRemoteLoc | ( | stage, | |
| drive, | |||
| path, | |||
| port | |||
| ) |
Upload stage to remote location.
Definition at line 1828 of file cmsPerfPublish.py.
01829 : 01830 stage = addtrailingslash(stage) 01831 cmd = "rsync -avz" 01832 # We must, MUST, do os.path.normpath otherwise rsync will dump the files in the directory 01833 # we specify on the remote server, rather than creating the CMS_VERSION directory 01834 #--rsh=\"ssh -l relval\" 01835 args = "--port=%s %s %s:%s" % (port,os.path.normpath(stage),drive,path) 01836 retval = -1 01837 if _dryrun: 01838 print cmd + " --dry-run " + args 01839 retval = os.system(cmd + " --dry-run " + args ) 01840 else: 01841 print cmd+" "+args 01842 retval = os.system(cmd + " " + args) 01843 return retval
| def cmsPerfPublish::syscp | ( | srcs, | |
| dest | |||
| ) |
Definition at line 1969 of file cmsPerfPublish.py.
01970 : 01971 if type(srcs) == type(""): 01972 if os.path.exists(srcs): 01973 if os.path.isdir(srcs): 01974 copytree4(srcs,dest) 01975 else: 01976 docopy(srcs,dest) 01977 else: 01978 print "ERROR: file to be copied %s does not exist" % foo 01979 else: 01980 for src in srcs: 01981 if os.path.exists(src): 01982 if os.path.isdir(src): 01983 #copy tree 01984 copytree4(src,dest) 01985 else: 01986 docopy(src,dest) 01987 else: 01988 print "ERROR: file to be copied %s does not exist" % foo
| def cmsPerfPublish::timerep_cmp | ( | x, | |
| y | |||
| ) |
Definition at line 763 of file cmsPerfPublish.py.
00764 : 00765 (fname,x) = x 00766 (fname,y) = y 00767 x = os.path.basename(x) 00768 y = os.path.basename(y) 00769 stepreg = re.compile("%s_(..*)_TimingReport" % fname) 00770 if stepreg.search(x): 00771 x = stepreg.search(x).groups()[0] 00772 if stepreg.search(y): 00773 y = stepreg.search(y).groups()[0] 00774 return step_cmp(x,y)
| int cmsPerfPublish::CallgrindNumOfEvents = 9999 |
Definition at line 34 of file cmsPerfPublish.py.
| tuple cmsPerfPublish::cpDirFilter = ( ) |
Definition at line 30 of file cmsPerfPublish.py.
| tuple cmsPerfPublish::cpFileFilter = ( "*.root", ) |
Definition at line 29 of file cmsPerfPublish.py.
| string cmsPerfPublish::DEF_RELVAL = "/afs/cern.ch/cms/sdt/web/performance/RelVal" |
Definition at line 26 of file cmsPerfPublish.py.
| string cmsPerfPublish::DEF_SIMUL = "/afs/cern.ch/cms/sdt/web/performance/simulation" |
Definition at line 27 of file cmsPerfPublish.py.
| tuple cmsPerfPublish::DirName |
00001 ( #These need to match the candle directory names ending (depending on the type of profiling) 00002 "TimeSize", 00003 "IgProf", 00004 "IgProf_Perf", 00005 "IgProf_Mem", 00006 "Callgrind", 00007 "Memcheck", 00008 #Adding the extra PU directories: 00009 "PU_TimeSize", 00010 "PU_IgProf", 00011 "PU_IgProf_Perf", 00012 "PU_IgProf_Mem", 00013 "PU_Callgrind", 00014 "PU_Memcheck" 00015 )
Definition at line 37 of file cmsPerfPublish.py.
Referenced by CaloMETAnalyzer::analyze(), PFMETAnalyzer::analyze(), MuCorrMETAnalyzer::analyze(), METAnalyzer::analyze(), TcMETAnalyzer::analyze(), METAnalyzer::beginJob(), TcMETAnalyzer::beginJob(), PFMETAnalyzer::beginJob(), MuCorrMETAnalyzer::beginJob(), CaloMETAnalyzer::beginJob(), PFMETAnalyzer::endRun(), CaloMETAnalyzer::endRun(), MuCorrMETAnalyzer::endRun(), TcMETAnalyzer::endRun(), and METAnalyzer::endRun().
| int cmsPerfPublish::IgProfNumOfEvents = 9999 |
Definition at line 33 of file cmsPerfPublish.py.
| int cmsPerfPublish::MemcheckNumOfEvents = 9999 |
Definition at line 35 of file cmsPerfPublish.py.
| tuple cmsPerfPublish::PROG_NAME = os.path.basename(sys.argv[0]) |
Definition at line 25 of file cmsPerfPublish.py.
| tuple cmsPerfPublish::Steps = set(Step+ProductionSteps+["GEN,FASTSIM","GEN,FASTSIM_PILEUP"]) |
Definition at line 53 of file cmsPerfPublish.py.
| int cmsPerfPublish::TimeSizeNumOfEvents = 9999 |
Definition at line 32 of file cmsPerfPublish.py.
| string cmsPerfPublish::TMP_DIR = "" |
Definition at line 28 of file cmsPerfPublish.py.
1.7.3