CMS 3D CMS Logo
beamvalidation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #____________________________________________________________
3 #
4 #
5 # A very simple
6 #
7 # Francisco Yumiceva
8 # yumiceva@fnal.gov
9 #
10 # Fermilab, 2010
11 #
12 #____________________________________________________________
13 
14 """
15  beam spot validation
16 
17  A very simple script
18 
19  usage: %prog -t <tag name>
20  -o, --output = OUTPUT: filename of output html file.
21  -p, --path = PATH: path to beam spot scripts.
22  -w, --web = WEB: html path to website.
23 
24  Francisco Yumiceva (yumiceva@fnal.gov)
25  Fermilab 2010
26 
27 """
28 from __future__ import print_function
29 
30 
31 from builtins import range
32 import os, string, re, sys, math
33 import commands, time
34 import six
35 
36 #_______________OPTIONS________________
37 import optparse
38 
39 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
40 
41 def nonzero(self): # will become the nonzero method of optparse.Values
42  "True if options were given"
43  for v in six.itervalues(self.__dict__):
44  if v is not None: return True
45  return False
46 
47 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
48 
49 class ParsingError(Exception): pass
50 
51 optionstring=""
52 
53 def exit(msg=""):
54  raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
55 
56 def parse(docstring, arglist=None):
57  global optionstring
58  optionstring = docstring
59  match = USAGE.search(optionstring)
60  if not match: raise ParsingError("Cannot find the option string")
61  optlines = match.group(1).splitlines()
62  try:
63  p = optparse.OptionParser(optlines[0])
64  for line in optlines[1:]:
65  opt, help=line.split(':')[:2]
66  short,long=opt.split(',')[:2]
67  if '=' in opt:
68  action='store'
69  long=long.split('=')[0]
70  else:
71  action='store_true'
72  p.add_option(short.strip(),long.strip(),
73  action = action, help = help.strip())
74  except (IndexError,ValueError):
75  raise ParsingError("Cannot parse the option string correctly")
76  return p.parse_args(arglist)
77 
78 #_______________________________
79 
80 def cmp_tags(a,b):
81 
82  yeara = int(a.split('_')[1])
83  yearb = int(b.split('_')[1])
84 
85  if yeara < yearb: return -1
86  if yeara > yearb: return 1
87 
88  suffix = "_offline"
89  if a.find("_express") != -1:
90  suffix = "_express"
91  if a.find("_prompt") != -1:
92  suffix = "_prompt"
93 
94  tmpa = a.replace("BeamSpotObjects_2009_v","")
95  tmpa = tmpa.replace(suffix,"")
96 
97  tmpb = b.replace("BeamSpotObjects_2009_v","")
98  tmpb = tmpb.replace(suffix,"")
99 
100  na = int(tmpa)
101  nb = int(tmpb)
102  if na < nb: return -1
103  if na == nb: return 0
104  if na > nb: return 1
105 
106 #___
107 
108 def dump_header(lines):
109 
110  lines.append('''
111 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
112 <head><title>Beam Spot Calibration Status</title></head>
113 
114 <body>
115 
116 <H1>Status of the Beam Spot Calibration</H1>
117 <BR>
118 <BR>
119 This page is updated every 10 minutes:
120 <BR>
121 <strong><script src="datemod.js"
122 type="text/javascript"></script></strong>
123 
124 ''')
125 
126 #____
127 
128 def dump_footer(lines):
129 
130  lines.append('</body>\n</html>\n')
131 
132 #______________
133 def write_tags(tags, lines):
134 
135  end = '\n'
136  br = '<BR>'+end
137 
138  lines.append('<tr>'+end)
139  for i in tags.keys():
140 
141  lines.append('<th>'+i)
142  lines.append('</th>'+end)
143  lines.append('</tr>'+end)
144 
145  for ntags in range(0,len(tags['offline'])):
146  lines.append('<tr>'+end)
147  for i in tags.keys():
148  alist = tags[i]
149  if ntags < len(tags[i]):
150  lines.append('<td> '+alist[ntags]+' </td>'+end)
151  else:
152  lines.append('<td> </td>')
153  lines.append('</tr>'+end)
154 
155 #______________
156 def write_iovs(iovs, lines):
157 
158  end = '\n'
159  br = '<BR>'+end
160  # hearder
161  lines.append('<tr>'+end)
162  for i in iovs.keys():
163  lines.append('<th>'+i)
164  lines.append('</th>'+end)
165  lines.append('</tr>'+end)
166  # lumi type
167  lines.append('<tr>'+end)
168  for i in iovs.keys():
169  aIOVlist = iovs[i]
170  aIOV = IOV()
171  if len(aIOVlist) > 0:
172  aIOV = aIOVlist[0]
173  lines.append('<td> '+aIOV.type+' </td>'+end)
174  lines.append('</tr>'+end)
175  # print iovs
176  for niovs in range(0,len(iovs[iovs.keys()[0]])):
177  lines.append('<tr>'+end)
178  for i in iovs.keys():
179  aIOVlist = iovs[i]
180  aIOV = IOV()
181  if len(aIOVlist) > niovs:
182  aIOV = aIOVlist[niovs]
183  lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
184  lines.append('</tr>'+end)
185 
186 #______________
187 def get_listoftags(dest, auth,):
188 
189  queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
190  print(queryTags_cmd)
191  outcmd = commands.getstatusoutput( queryTags_cmd )
192  print(outcmd[1])
193 
194  listtags = outcmd[1].split()
195 
196  listtags_offline = []
197  for itag in listtags:
198  if itag[len(itag)-7:len(itag)] == "offline":
199  listtags_offline.append(itag)
200  listtags_express = []
201  for itag in listtags:
202  if itag[len(itag)-7:len(itag)] == "express":
203  listtags_express.append(itag)
204  listtags_prompt = []
205  for itag in listtags:
206  if itag[len(itag)-6:len(itag)] == "prompt":
207  listtags_prompt.append(itag)
208 
209  listtags_offline.sort( cmp = cmp_tags )
210  listtags_offline.reverse()
211  listtags_express.sort( cmp = cmp_tags )
212  listtags_express.reverse()
213  listtags_prompt.sort( cmp = cmp_tags )
214  listtags_prompt.reverse()
215 
216  result = {}
217  result['offline'] = listtags_offline
218  result['express'] = listtags_express
219  result['prompt'] = listtags_prompt
220 
221  return result
222 
223 #______________________
224 class IOV:
225  def __init__(self):
226  self.type = ''
227  self.IOVfirst = ''
228  self.IOVlast = ''
229 
230 #_________________
231 def get_lastIOVs( listoftags, dest, auth ):
232 
233  dbtags = ['offline','express','prompt']
234 
235  results = {}
236  for itag in dbtags:
237 
238  lasttag = listoftags[itag][0]
239  #fix for the moment to read old tags
240  if itag != "offline":
241  lasttag = listoftags[itag][1]
242 
243  queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
244  print(queryIOVs_cmd)
245 
246  outcmd = commands.getstatusoutput( queryIOVs_cmd )
247 
248  tmparr = outcmd[1].split('\n')
249 
250  TimeType = tmparr[1].split()[1]
251  listIOVs = []
252 
253  # look at number of payloads
254  lastline = tmparr[len(tmparr)-1].split()
255  npayloads = int( lastline[len(lastline)-1] )
256 
257  maxIOVs = 3
258  if npayloads < 3:
259  maxIOVs = npayloads
260  # pick the last three IOVs
261  for i in range(0,maxIOVs):
262  tmpline = tmparr[len(tmparr) -2 -i]
263  aIOV = IOV()
264  aIOV.IOVfirst = tmpline.split()[0]
265  aIOV.IOVlast = tmpline.split()[1]
266  aIOV.type = TimeType
267  listIOVs.append( aIOV )
268 
269  results[lasttag] = listIOVs
270 
271  return results
272 
273 #
274 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
275 def pack(high,low):
276  """pack high,low 32bit unsigned int to one unsigned 64bit long long
277  Note:the print value of result number may appear signed, if the sign bit is used.
278  """
279  h=high<<32
280  return (h|low)
281 
282 def unpack(i):
283  """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
284  """
285  high=i>>32
286  low=i&0xFFFFFFFF
287  return(high,low)
288 
289 def unpackLumiid(i):
290  """unpack 64bit lumiid to dictionary {'run','lumisection'}
291  """
292  j=unpack(i)
293  return {'run':j[0],'lumisection':j[1]}
294 
295 #____________________________
296 def get_plots(path,output, iovs, tag):
297 
298  initial=iovs[len(iovs)-1].IOVfirst
299  final =iovs[0].IOVfirst
300  if iovs[0].type == "lumiid":
301  initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
302  final = str(unpack(final)[0])+":"+str(unpack(final)[1])
303 
304  initial = str(int(initial) -100 )
305  cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
306  print(cmd)
307  outcmd = commands.getstatusoutput( cmd )
308  print(outcmd[1])
309 
310  cmd = "ls *.png"
311  outcmd = commands.getstatusoutput( cmd )
312 
313  pngfiles = outcmd[1].split('\n')
314  print(pngfiles)
315 
316  cmd = "cp *.png "+os.path.dirname(output)
317  outcmd = commands.getstatusoutput( cmd )
318  cmd = "rm *.png"
319  outcmd = commands.getstatusoutput( cmd )
320 
321  pngfiles.sort()
322  return pngfiles
323 
324 #_______________________________
325 def write_plots(lines, plots,web):
326 
327  end = '\n'
328  br = '<BR>'+end
329 
330  lines.append(br)
331  lines.append('''
332 <table border="1">
333 
334 ''')
335  for i in range(0,len(plots)):
336  plot = plots[i]
337  plot = os.path.basename(plot)
338  if i%2 == 0:
339  lines.append("<tr>"+end)
340  lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
341  if i%2 == 1:
342  lines.append("</tr>"+end)
343 
344  lines.append('</table>'+end)
345 
346 #________________________________
347 def get_productionFiles( directory ):
348 
349  list = commands.getstatusoutput('ls -t '+directory)
350  list = list[1].split()
351  newlist = []
352  for i in list:
353  if i.find('BeamFit_')!=-1:
354  newlist.append(i)
355 
356  return newlist
357 #_______________________________
358 def get_productionIOVs( directory ):
359 
360  files = get_productionFiles( directory )
361  listofruns = []
362  for f in files:
363  ii = f.find('Run')
364  arun = f[ii:len(f)-4]
365  listofruns.append(arun)
366  return listofruns
367 
368 #______________________________
369 if __name__ == '__main__':
370 
371 
372  # COMMAND LINE OPTIONS
373  #################################
374  option,args = parse(__doc__)
375  if not args and not option: exit()
376 
377  #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
378  htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
379  if option.web: htmlwebsite = option.web
380 
381  ## Get the latest tags
382  #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
383  dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
384  auth = "/afs/cern.ch/cms/DB/conddb"
385  #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb -a
386 
387  list_tags = get_listoftags( dest, auth)
388 
389  # Get the latest IOVs from last tag
390  list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
391 
392  # get latest processed runs
393  processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
394 
395  # create web page
396  lines = []
397  end = '\n'
398  br = '<BR>'+end
399 
400  dump_header(lines)
401 
402  lines.append('<h2>The three latest IOVs</h2>'+end)
403  lines.append(br)
404  lines.append('''
405 <table border="1">
406 ''')
407  write_iovs( list_lastIOVs, lines )
408  lines.append('</table>'+end)
409 
410  lines.append('<h2>Latest data processed</h2>'+end)
411  lines.append(br)
412  #lines.append('to be written'+end)
413  lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
414  print(processedruns)
415  lines.append(br)
416 
417  lines.append('<h2>The Latest Tags</h2>'+end)
418  lines.append(br)
419  lines.append('''
420 <table border="1">
421 ''')
422  write_tags( list_tags, lines)
423  lines.append('</table>'+end)
424 
425  lasttag = list_lastIOVs.keys()[0]
426  lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
427  lines.append(br)
428 
429  pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
430  write_plots( lines, pngfiles, htmlwebsite )
431 
432  dump_footer(lines)
433 
434  outfile = open(option.output,'w')
435  #print lines
436  outfile.writelines( lines )
437 
438 
439 
440 
441 
442 
443 
444 
beamvalidation.parse
def parse(docstring, arglist=None)
Definition: beamvalidation.py:56
beamvalidation.get_productionIOVs
def get_productionIOVs(directory)
Definition: beamvalidation.py:358
beamvalidation.write_plots
def write_plots(lines, plots, web)
Definition: beamvalidation.py:325
beamvalidation.get_productionFiles
def get_productionFiles(directory)
Definition: beamvalidation.py:347
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
beamvalidation.dump_header
def dump_header(lines)
Definition: beamvalidation.py:108
mathSSE::return
return((rh^lh)&mask)
beamvalidation.get_listoftags
def get_listoftags(dest, auth)
Definition: beamvalidation.py:187
beamvalidation.write_iovs
def write_iovs(iovs, lines)
Definition: beamvalidation.py:156
beamvalidation.get_plots
def get_plots(path, output, iovs, tag)
Definition: beamvalidation.py:296
beamvalidation.write_tags
def write_tags(tags, lines)
Definition: beamvalidation.py:133
beamvalidation.get_lastIOVs
def get_lastIOVs(listoftags, dest, auth)
Definition: beamvalidation.py:231
beamvalidation.dump_footer
def dump_footer(lines)
Definition: beamvalidation.py:128
beamvalidation.nonzero
def nonzero(self)
Definition: beamvalidation.py:41
beamvalidation.pack
def pack(high, low)
Definition: beamvalidation.py:275
str
#define str(s)
Definition: TestProcessor.cc:48
split
double split
Definition: MVATrainer.cc:139
beamvalidation.exit
def exit(msg="")
Definition: beamvalidation.py:53
beamvalidation.cmp_tags
def cmp_tags(a, b)
Definition: beamvalidation.py:80
Generated for CMSSW Reference Manual by doxygen 1.8.11