CMS 3D CMS Logo
beamvalidation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #____________________________________________________________
3 #
4 #
5 # A very simple
6 #
7 # Francisco Yumiceva
8 # yumiceva@fnal.gov
9 #
10 # Fermilab, 2010
11 #
12 #____________________________________________________________
13 
14 """
15  beam spot validation
16 
17  A very simple script
18 
19  usage: %prog -t <tag name>
20  -o, --output = OUTPUT: filename of output html file.
21  -p, --path = PATH: path to beam spot scripts.
22  -w, --web = WEB: html path to website.
23 
24  Francisco Yumiceva (yumiceva@fnal.gov)
25  Fermilab 2010
26 
27 """
28 
29 
30 import os, string, re, sys, math
31 import commands, time
32 import six
33 
34 #_______________OPTIONS________________
35 import optparse
36 
37 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
38 
39 def nonzero(self): # will become the nonzero method of optparse.Values
40  "True if options were given"
41  for v in six.itervalues(self.__dict__):
42  if v is not None: return True
43  return False
44 
45 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
46 
47 class ParsingError(Exception): pass
48 
49 optionstring=""
50 
51 def exit(msg=""):
52  raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
53 
54 def parse(docstring, arglist=None):
55  global optionstring
56  optionstring = docstring
57  match = USAGE.search(optionstring)
58  if not match: raise ParsingError("Cannot find the option string")
59  optlines = match.group(1).splitlines()
60  try:
61  p = optparse.OptionParser(optlines[0])
62  for line in optlines[1:]:
63  opt, help=line.split(':')[:2]
64  short,long=opt.split(',')[:2]
65  if '=' in opt:
66  action='store'
67  long=long.split('=')[0]
68  else:
69  action='store_true'
70  p.add_option(short.strip(),long.strip(),
71  action = action, help = help.strip())
72  except (IndexError,ValueError):
73  raise ParsingError("Cannot parse the option string correctly")
74  return p.parse_args(arglist)
75 
76 #_______________________________
77 
78 def cmp_tags(a,b):
79 
80  yeara = int(a.split('_')[1])
81  yearb = int(b.split('_')[1])
82 
83  if yeara < yearb: return -1
84  if yeara > yearb: return 1
85 
86  suffix = "_offline"
87  if a.find("_express") != -1:
88  suffix = "_express"
89  if a.find("_prompt") != -1:
90  suffix = "_prompt"
91 
92  tmpa = a.replace("BeamSpotObjects_2009_v","")
93  tmpa = tmpa.replace(suffix,"")
94 
95  tmpb = b.replace("BeamSpotObjects_2009_v","")
96  tmpb = tmpb.replace(suffix,"")
97 
98  na = int(tmpa)
99  nb = int(tmpb)
100  if na < nb: return -1
101  if na == nb: return 0
102  if na > nb: return 1
103 
104 #___
105 
106 def dump_header(lines):
107 
108  lines.append('''
109 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
110 <head><title>Beam Spot Calibration Status</title></head>
111 
112 <body>
113 
114 <H1>Status of the Beam Spot Calibration</H1>
115 <BR>
116 <BR>
117 This page is updated every 10 minutes:
118 <BR>
119 <strong><script src="datemod.js"
120 type="text/javascript"></script></strong>
121 
122 ''')
123 
124 #____
125 
126 def dump_footer(lines):
127 
128  lines.append('</body>\n</html>\n')
129 
130 #______________
131 def write_tags(tags, lines):
132 
133  end = '\n'
134  br = '<BR>'+end
135 
136  lines.append('<tr>'+end)
137  for i in tags.keys():
138 
139  lines.append('<th>'+i)
140  lines.append('</th>'+end)
141  lines.append('</tr>'+end)
142 
143  for ntags in range(0,len(tags['offline'])):
144  lines.append('<tr>'+end)
145  for i in tags.keys():
146  alist = tags[i]
147  if ntags < len(tags[i]):
148  lines.append('<td> '+alist[ntags]+' </td>'+end)
149  else:
150  lines.append('<td> </td>')
151  lines.append('</tr>'+end)
152 
153 #______________
154 def write_iovs(iovs, lines):
155 
156  end = '\n'
157  br = '<BR>'+end
158  # hearder
159  lines.append('<tr>'+end)
160  for i in iovs.keys():
161  lines.append('<th>'+i)
162  lines.append('</th>'+end)
163  lines.append('</tr>'+end)
164  # lumi type
165  lines.append('<tr>'+end)
166  for i in iovs.keys():
167  aIOVlist = iovs[i]
168  aIOV = IOV()
169  if len(aIOVlist) > 0:
170  aIOV = aIOVlist[0]
171  lines.append('<td> '+aIOV.type+' </td>'+end)
172  lines.append('</tr>'+end)
173  # print iovs
174  for niovs in range(0,len(iovs[iovs.keys()[0]])):
175  lines.append('<tr>'+end)
176  for i in iovs.keys():
177  aIOVlist = iovs[i]
178  aIOV = IOV()
179  if len(aIOVlist) > niovs:
180  aIOV = aIOVlist[niovs]
181  lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
182  lines.append('</tr>'+end)
183 
184 #______________
185 def get_listoftags(dest, auth,):
186 
187  queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
188  print queryTags_cmd
189  outcmd = commands.getstatusoutput( queryTags_cmd )
190  print outcmd[1]
191 
192  listtags = outcmd[1].split()
193 
194  listtags_offline = []
195  for itag in listtags:
196  if itag[len(itag)-7:len(itag)] == "offline":
197  listtags_offline.append(itag)
198  listtags_express = []
199  for itag in listtags:
200  if itag[len(itag)-7:len(itag)] == "express":
201  listtags_express.append(itag)
202  listtags_prompt = []
203  for itag in listtags:
204  if itag[len(itag)-6:len(itag)] == "prompt":
205  listtags_prompt.append(itag)
206 
207  listtags_offline.sort( cmp = cmp_tags )
208  listtags_offline.reverse()
209  listtags_express.sort( cmp = cmp_tags )
210  listtags_express.reverse()
211  listtags_prompt.sort( cmp = cmp_tags )
212  listtags_prompt.reverse()
213 
214  result = {}
215  result['offline'] = listtags_offline
216  result['express'] = listtags_express
217  result['prompt'] = listtags_prompt
218 
219  return result
220 
221 #______________________
222 class IOV:
223  def __init__(self):
224  self.type = ''
225  self.IOVfirst = ''
226  self.IOVlast = ''
227 
228 #_________________
229 def get_lastIOVs( listoftags, dest, auth ):
230 
231  dbtags = ['offline','express','prompt']
232 
233  results = {}
234  for itag in dbtags:
235 
236  lasttag = listoftags[itag][0]
237  #fix for the moment to read old tags
238  if itag != "offline":
239  lasttag = listoftags[itag][1]
240 
241  queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
242  print queryIOVs_cmd
243 
244  outcmd = commands.getstatusoutput( queryIOVs_cmd )
245 
246  tmparr = outcmd[1].split('\n')
247 
248  TimeType = tmparr[1].split()[1]
249  listIOVs = []
250 
251  # look at number of payloads
252  lastline = tmparr[len(tmparr)-1].split()
253  npayloads = int( lastline[len(lastline)-1] )
254 
255  maxIOVs = 3
256  if npayloads < 3:
257  maxIOVs = npayloads
258  # pick the last three IOVs
259  for i in range(0,maxIOVs):
260  tmpline = tmparr[len(tmparr) -2 -i]
261  aIOV = IOV()
262  aIOV.IOVfirst = tmpline.split()[0]
263  aIOV.IOVlast = tmpline.split()[1]
264  aIOV.type = TimeType
265  listIOVs.append( aIOV )
266 
267  results[lasttag] = listIOVs
268 
269  return results
270 
271 #
272 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
273 def pack(high,low):
274  """pack high,low 32bit unsigned int to one unsigned 64bit long long
275  Note:the print value of result number may appear signed, if the sign bit is used.
276  """
277  h=high<<32
278  return (h|low)
279 
280 def unpack(i):
281  """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
282  """
283  high=i>>32
284  low=i&0xFFFFFFFF
285  return(high,low)
286 
287 def unpackLumiid(i):
288  """unpack 64bit lumiid to dictionary {'run','lumisection'}
289  """
290  j=unpack(i)
291  return {'run':j[0],'lumisection':j[1]}
292 
293 #____________________________
294 def get_plots(path,output, iovs, tag):
295 
296  initial=iovs[len(iovs)-1].IOVfirst
297  final =iovs[0].IOVfirst
298  if iovs[0].type == "lumiid":
299  initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
300  final = str(unpack(final)[0])+":"+str(unpack(final)[1])
301 
302  initial = str(int(initial) -100 )
303  cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
304  print cmd
305  outcmd = commands.getstatusoutput( cmd )
306  print outcmd[1]
307 
308  cmd = "ls *.png"
309  outcmd = commands.getstatusoutput( cmd )
310 
311  pngfiles = outcmd[1].split('\n')
312  print pngfiles
313 
314  cmd = "cp *.png "+os.path.dirname(output)
315  outcmd = commands.getstatusoutput( cmd )
316  cmd = "rm *.png"
317  outcmd = commands.getstatusoutput( cmd )
318 
319  pngfiles.sort()
320  return pngfiles
321 
322 #_______________________________
323 def write_plots(lines, plots,web):
324 
325  end = '\n'
326  br = '<BR>'+end
327 
328  lines.append(br)
329  lines.append('''
330 <table border="1">
331 
332 ''')
333  for i in range(0,len(plots)):
334  plot = plots[i]
335  plot = os.path.basename(plot)
336  if i%2 == 0:
337  lines.append("<tr>"+end)
338  lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
339  if i%2 == 1:
340  lines.append("</tr>"+end)
341 
342  lines.append('</table>'+end)
343 
344 #________________________________
345 def get_productionFiles( directory ):
346 
347  list = commands.getstatusoutput('ls -t '+directory)
348  list = list[1].split()
349  newlist = []
350  for i in list:
351  if i.find('BeamFit_')!=-1:
352  newlist.append(i)
353 
354  return newlist
355 #_______________________________
356 def get_productionIOVs( directory ):
357 
358  files = get_productionFiles( directory )
359  listofruns = []
360  for f in files:
361  ii = f.find('Run')
362  arun = f[ii:len(f)-4]
363  listofruns.append(arun)
364  return listofruns
365 
366 #______________________________
367 if __name__ == '__main__':
368 
369 
370  # COMMAND LINE OPTIONS
371  #################################
372  option,args = parse(__doc__)
373  if not args and not option: exit()
374 
375  #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
376  htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
377  if option.web: htmlwebsite = option.web
378 
379  ## Get the latest tags
380  #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
381  dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
382  auth = "/afs/cern.ch/cms/DB/conddb"
383  #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb -a
384 
385  list_tags = get_listoftags( dest, auth)
386 
387  # Get the latest IOVs from last tag
388  list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
389 
390  # get latest processed runs
391  processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
392 
393  # create web page
394  lines = []
395  end = '\n'
396  br = '<BR>'+end
397 
398  dump_header(lines)
399 
400  lines.append('<h2>The three latest IOVs</h2>'+end)
401  lines.append(br)
402  lines.append('''
403 <table border="1">
404 ''')
405  write_iovs( list_lastIOVs, lines )
406  lines.append('</table>'+end)
407 
408  lines.append('<h2>Latest data processed</h2>'+end)
409  lines.append(br)
410  #lines.append('to be written'+end)
411  lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
412  print processedruns
413  lines.append(br)
414 
415  lines.append('<h2>The Latest Tags</h2>'+end)
416  lines.append(br)
417  lines.append('''
418 <table border="1">
419 ''')
420  write_tags( list_tags, lines)
421  lines.append('</table>'+end)
422 
423  lasttag = list_lastIOVs.keys()[0]
424  lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
425  lines.append(br)
426 
427  pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
428  write_plots( lines, pngfiles, htmlwebsite )
429 
430  dump_footer(lines)
431 
432  outfile = open(option.output,'w')
433  #print lines
434  outfile.writelines( lines )
435 
436 
437 
438 
439 
440 
441 
442 
beamvalidation.parse
def parse(docstring, arglist=None)
Definition: beamvalidation.py:54
beamvalidation.get_productionIOVs
def get_productionIOVs(directory)
Definition: beamvalidation.py:356
beamvalidation.write_plots
def write_plots(lines, plots, web)
Definition: beamvalidation.py:323
beamvalidation.get_productionFiles
def get_productionFiles(directory)
Definition: beamvalidation.py:345
beamvalidation.dump_header
def dump_header(lines)
Definition: beamvalidation.py:106
mathSSE::return
return((rh^lh)&mask)
beamvalidation.get_listoftags
def get_listoftags(dest, auth)
Definition: beamvalidation.py:185
beamvalidation.write_iovs
def write_iovs(iovs, lines)
Definition: beamvalidation.py:154
beamvalidation.get_plots
def get_plots(path, output, iovs, tag)
Definition: beamvalidation.py:294
beamvalidation.write_tags
def write_tags(tags, lines)
Definition: beamvalidation.py:131
beamvalidation.get_lastIOVs
def get_lastIOVs(listoftags, dest, auth)
Definition: beamvalidation.py:229
beamvalidation.dump_footer
def dump_footer(lines)
Definition: beamvalidation.py:126
beamvalidation.nonzero
def nonzero(self)
Definition: beamvalidation.py:39
beamvalidation.pack
def pack(high, low)
Definition: beamvalidation.py:273
str
#define str(s)
Definition: TestProcessor.cc:47
split
double split
Definition: MVATrainer.cc:139
beamvalidation.exit
def exit(msg="")
Definition: beamvalidation.py:51
beamvalidation.cmp_tags
def cmp_tags(a, b)
Definition: beamvalidation.py:78
Generated for CMSSW Reference Manual by doxygen 1.8.11