25 def operate(timelog, memlog, json_f, num):
30 from datetime
import datetime
32 script_name=os.path.basename(__file__)
35 timefile=open(timelog,
'r')
36 timelog_lines=timefile.readlines()
39 memfile=open(memlog, 'r')
40 memlog_lines=memfile.readlines()
44 max_rss=average=error=
' '
46 while i<len(timelog_lines):
48 if 'Uncertainty of Average Time' in line:
50 line_list=line.split(
' ')
55 while i<len(memlog_lines):
57 if 'Maximum rss' in line:
59 line_list=line.split(
' ')
65 IB=os.path.basename(commands.getoutput(
"echo $CMSSW_BASE"))
72 if average ==
' ' or re.match(regex, average)
is None:
73 raise RuntimeError(
'Could not parse \"' + timelog +
'\" properly. ' +\
74 'Check if Average Time is defined correctly.')
75 if error ==
' ' or re.match(regex, error)
is None:
76 raise RuntimeError(
'Could not parse \"' + timelog +
'\" properly. ' +\
77 'Check if Uncertainty of Average Time is defined correctly.')
78 if max_rss ==
' ' or re.match(regex, max_rss)
is None:
79 raise RuntimeError(
'Could not parse \"' + memlog +
'\" properly. ' +\
80 ' Check if Maximum rss is defined correct.')
83 regex =
'(19|20|21)\d\d-(0[1-9]|1[012])-(0[1-9]|[12]'+\
84 '[0-9]|3[01])-([01][0-9]|2[0-4])([0-5][0-9])$'
85 if re.search(regex, IB)
is None:
86 raise RuntimeError(
'Not a valid IB. Valid IB: ' +\
87 '[CMSSW_X_X_X_YYYY-MM-DD-HHMM]')
88 except Exception, err:
89 sys.stderr.write(script_name +
': Error: ' + str(err) +
'\n')
93 json_db=open(json_f,
"r")
94 dict=json.load(json_db)
99 cmsrelease=ib_list[0] +
'_' + ib_list[1] +\
100 '_' + ib_list[2] +
'_' + ib_list[3]
101 data={
"IB" : ib_list[4],
"average" : float(average),
"error" : float(error),
"max_rss" : float(max_rss)}
103 if data
in dict[
"strips"]:
104 sys.stderr.write(script_name +
": Warning: Entry already exists " +\
105 "in json file and will not be stored! " +\
106 "Only the strip charts will be created.\n")
108 dict[
"strips"].
append(data)
109 print 'Storing entry to \"' + json_f +\
110 '\" file with attribute values:\n' +\
111 'IB=' + IB +
'\naverage=' + average +\
112 '\nUncertainty of average=' + error +
'\nmax_rss=' + max_rss
114 json_db = open(json_f,
"w+")
115 json.dump(dict, json_db, indent=2)
117 print 'File \"%s\" was updated successfully!' % json_f
120 for record
in dict[
"strips"]:
121 time_list = record[
'IB'].
split(
'-')
122 d = datetime(int(time_list[0]), int(time_list[1]),
123 int(time_list[2]), int(time_list[3][0:2]),
124 int(time_list[3][2:]))
128 list = sorted(dict[
"strips"], key=
lambda k : k[
'IB'], reverse=
True)
133 sys.stderr.write(script_name +
': Warning: There are less than ' +\
134 str(num) +
' entries in json file. Changed number to ' +\
135 str(new_num) +
'.\n')
139 ROOT.gROOT.SetStyle(
"Plain")
143 rootfilename=outdir +
'/histograms.root'
144 myfile=ROOT.TFile(rootfilename,
'RECREATE')
147 histo1=ROOT.TH1F(
"AveCPU per IB",
"Ave CPU per IB", num, 0., num)
148 histo1.SetTitle(cmsrelease +
": Showing last " + str(num) +
" IB's")
149 histo1.SetName(
'avecpu_histo')
152 histo2=ROOT.TH1F(
"Max rrs per IB",
"Max rss per IB", num, 0., num)
153 histo2.SetTitle(cmsrelease +
": Showing last " + str(num) +
" IB's")
154 histo2.SetName(
'maxrss_histo')
159 datime = list[i][
'IB'].__format__(
'%Y-%b-%d %H:%M')
160 average = list[i][
'average']
161 max_rss = list[i][
'max_rss']
162 error = list[i][
'error']
164 histo1.GetXaxis().SetBinLabel(num-i, datime)
165 histo1.SetBinContent(num-i, average)
166 histo1.SetBinError(num-i, error)
167 histo2.GetXaxis().SetBinLabel(num-i, datime)
168 histo2.SetBinContent(num-i, max_rss)
171 histo1.GetYaxis().SetTitle(
"Average CPU time")
172 histo1.GetYaxis().SetTitleOffset(1.8)
173 histo1.GetXaxis().SetTitle(
"Integration Build")
174 histo1.GetXaxis().SetTitleOffset(4.)
175 histo1.GetXaxis().CenterTitle()
176 histo1.GetXaxis().LabelsOption(
'v')
181 min = min-interval*0.1
182 max = max+interval*0.1
183 histo1.GetYaxis().SetRangeUser(min, max)
186 histo2.GetYaxis().SetTitle(
"Maximum rss")
187 histo2.GetYaxis().SetTitleOffset(1.8)
188 histo2.GetXaxis().SetTitle(
"Integration Build")
189 histo2.GetXaxis().SetTitleOffset(4.)
190 histo2.GetXaxis().CenterTitle()
191 histo2.GetXaxis().LabelsOption(
'v')
195 ave_canvas = ROOT.TCanvas(cmsrelease +
'_average_canvas')
196 ave_canvas.SetGridy()
197 ave_canvas.SetBottomMargin(0.28)
198 ave_canvas.SetLeftMargin(0.18)
201 histo1.SetLineColor(2)
202 histo1.SetLineWidth(2)
203 histo1.DrawCopy(
"HISTO L")
205 histo1.SetLineColor(1)
206 histo1.SetLineStyle(2)
207 histo1.SetLineWidth(1)
208 histo1.SetMarkerStyle(8)
209 histo1.SetMarkerSize(.6)
210 histo1.SetMarkerColor(1)
211 histo1.Draw(
"E1P SAME")
212 ave_canvas.Print(outdir +
"/average_cpu_histo.png",
"png")
214 rss_canvas = ROOT.TCanvas(cmsrelease +
'_maxrss_canvas')
215 rss_canvas.SetGridy()
216 rss_canvas.SetBottomMargin(0.28)
217 rss_canvas.SetLeftMargin(0.18)
220 histo2.SetLineColor(2)
221 histo2.SetLineWidth(2)
224 histo2.SetMarkerStyle(8)
225 histo2.SetMarkerSize(.6)
226 histo2.SetMarkerColor(1)
227 histo2.Draw(
"P SAME")
228 rss_canvas.Print(outdir +
"/maximum_rss_histo.png",
"png")