CMS 3D CMS Logo
beamvalidation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #____________________________________________________________
3 #
4 #
5 # A very simple
6 #
7 # Francisco Yumiceva
8 # yumiceva@fnal.gov
9 #
10 # Fermilab, 2010
11 #
12 #____________________________________________________________
13 
14 """
15  beam spot validation
16 
17  A very simple script
18 
19  usage: %prog -t <tag name>
20  -o, --output = OUTPUT: filename of output html file.
21  -p, --path = PATH: path to beam spot scripts.
22  -w, --web = WEB: html path to website.
23 
24  Francisco Yumiceva (yumiceva@fnal.gov)
25  Fermilab 2010
26 
27 """
28 from __future__ import print_function
29 
30 
31 import os, string, re, sys, math
32 import commands, time
33 import six
34 
35 #_______________OPTIONS________________
36 import optparse
37 
38 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
39 
40 def nonzero(self): # will become the nonzero method of optparse.Values
41  "True if options were given"
42  for v in six.itervalues(self.__dict__):
43  if v is not None: return True
44  return False
45 
46 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
47 
48 class ParsingError(Exception): pass
49 
50 optionstring=""
51 
52 def exit(msg=""):
53  raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
54 
55 def parse(docstring, arglist=None):
56  global optionstring
57  optionstring = docstring
58  match = USAGE.search(optionstring)
59  if not match: raise ParsingError("Cannot find the option string")
60  optlines = match.group(1).splitlines()
61  try:
62  p = optparse.OptionParser(optlines[0])
63  for line in optlines[1:]:
64  opt, help=line.split(':')[:2]
65  short,long=opt.split(',')[:2]
66  if '=' in opt:
67  action='store'
68  long=long.split('=')[0]
69  else:
70  action='store_true'
71  p.add_option(short.strip(),long.strip(),
72  action = action, help = help.strip())
73  except (IndexError,ValueError):
74  raise ParsingError("Cannot parse the option string correctly")
75  return p.parse_args(arglist)
76 
77 #_______________________________
78 
79 def cmp_tags(a,b):
80 
81  yeara = int(a.split('_')[1])
82  yearb = int(b.split('_')[1])
83 
84  if yeara < yearb: return -1
85  if yeara > yearb: return 1
86 
87  suffix = "_offline"
88  if a.find("_express") != -1:
89  suffix = "_express"
90  if a.find("_prompt") != -1:
91  suffix = "_prompt"
92 
93  tmpa = a.replace("BeamSpotObjects_2009_v","")
94  tmpa = tmpa.replace(suffix,"")
95 
96  tmpb = b.replace("BeamSpotObjects_2009_v","")
97  tmpb = tmpb.replace(suffix,"")
98 
99  na = int(tmpa)
100  nb = int(tmpb)
101  if na < nb: return -1
102  if na == nb: return 0
103  if na > nb: return 1
104 
105 #___
106 
107 def dump_header(lines):
108 
109  lines.append('''
110 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
111 <head><title>Beam Spot Calibration Status</title></head>
112 
113 <body>
114 
115 <H1>Status of the Beam Spot Calibration</H1>
116 <BR>
117 <BR>
118 This page is updated every 10 minutes:
119 <BR>
120 <strong><script src="datemod.js"
121 type="text/javascript"></script></strong>
122 
123 ''')
124 
125 #____
126 
127 def dump_footer(lines):
128 
129  lines.append('</body>\n</html>\n')
130 
131 #______________
132 def write_tags(tags, lines):
133 
134  end = '\n'
135  br = '<BR>'+end
136 
137  lines.append('<tr>'+end)
138  for i in tags.keys():
139 
140  lines.append('<th>'+i)
141  lines.append('</th>'+end)
142  lines.append('</tr>'+end)
143 
144  for ntags in range(0,len(tags['offline'])):
145  lines.append('<tr>'+end)
146  for i in tags.keys():
147  alist = tags[i]
148  if ntags < len(tags[i]):
149  lines.append('<td> '+alist[ntags]+' </td>'+end)
150  else:
151  lines.append('<td> </td>')
152  lines.append('</tr>'+end)
153 
154 #______________
155 def write_iovs(iovs, lines):
156 
157  end = '\n'
158  br = '<BR>'+end
159  # hearder
160  lines.append('<tr>'+end)
161  for i in iovs.keys():
162  lines.append('<th>'+i)
163  lines.append('</th>'+end)
164  lines.append('</tr>'+end)
165  # lumi type
166  lines.append('<tr>'+end)
167  for i in iovs.keys():
168  aIOVlist = iovs[i]
169  aIOV = IOV()
170  if len(aIOVlist) > 0:
171  aIOV = aIOVlist[0]
172  lines.append('<td> '+aIOV.type+' </td>'+end)
173  lines.append('</tr>'+end)
174  # print iovs
175  for niovs in range(0,len(iovs[iovs.keys()[0]])):
176  lines.append('<tr>'+end)
177  for i in iovs.keys():
178  aIOVlist = iovs[i]
179  aIOV = IOV()
180  if len(aIOVlist) > niovs:
181  aIOV = aIOVlist[niovs]
182  lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
183  lines.append('</tr>'+end)
184 
185 #______________
186 def get_listoftags(dest, auth,):
187 
188  queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
189  print(queryTags_cmd)
190  outcmd = commands.getstatusoutput( queryTags_cmd )
191  print(outcmd[1])
192 
193  listtags = outcmd[1].split()
194 
195  listtags_offline = []
196  for itag in listtags:
197  if itag[len(itag)-7:len(itag)] == "offline":
198  listtags_offline.append(itag)
199  listtags_express = []
200  for itag in listtags:
201  if itag[len(itag)-7:len(itag)] == "express":
202  listtags_express.append(itag)
203  listtags_prompt = []
204  for itag in listtags:
205  if itag[len(itag)-6:len(itag)] == "prompt":
206  listtags_prompt.append(itag)
207 
208  listtags_offline.sort( cmp = cmp_tags )
209  listtags_offline.reverse()
210  listtags_express.sort( cmp = cmp_tags )
211  listtags_express.reverse()
212  listtags_prompt.sort( cmp = cmp_tags )
213  listtags_prompt.reverse()
214 
215  result = {}
216  result['offline'] = listtags_offline
217  result['express'] = listtags_express
218  result['prompt'] = listtags_prompt
219 
220  return result
221 
222 #______________________
223 class IOV:
224  def __init__(self):
225  self.type = ''
226  self.IOVfirst = ''
227  self.IOVlast = ''
228 
229 #_________________
230 def get_lastIOVs( listoftags, dest, auth ):
231 
232  dbtags = ['offline','express','prompt']
233 
234  results = {}
235  for itag in dbtags:
236 
237  lasttag = listoftags[itag][0]
238  #fix for the moment to read old tags
239  if itag != "offline":
240  lasttag = listoftags[itag][1]
241 
242  queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
243  print(queryIOVs_cmd)
244 
245  outcmd = commands.getstatusoutput( queryIOVs_cmd )
246 
247  tmparr = outcmd[1].split('\n')
248 
249  TimeType = tmparr[1].split()[1]
250  listIOVs = []
251 
252  # look at number of payloads
253  lastline = tmparr[len(tmparr)-1].split()
254  npayloads = int( lastline[len(lastline)-1] )
255 
256  maxIOVs = 3
257  if npayloads < 3:
258  maxIOVs = npayloads
259  # pick the last three IOVs
260  for i in range(0,maxIOVs):
261  tmpline = tmparr[len(tmparr) -2 -i]
262  aIOV = IOV()
263  aIOV.IOVfirst = tmpline.split()[0]
264  aIOV.IOVlast = tmpline.split()[1]
265  aIOV.type = TimeType
266  listIOVs.append( aIOV )
267 
268  results[lasttag] = listIOVs
269 
270  return results
271 
272 #
273 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
274 def pack(high,low):
275  """pack high,low 32bit unsigned int to one unsigned 64bit long long
276  Note:the print value of result number may appear signed, if the sign bit is used.
277  """
278  h=high<<32
279  return (h|low)
280 
281 def unpack(i):
282  """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
283  """
284  high=i>>32
285  low=i&0xFFFFFFFF
286  return(high,low)
287 
288 def unpackLumiid(i):
289  """unpack 64bit lumiid to dictionary {'run','lumisection'}
290  """
291  j=unpack(i)
292  return {'run':j[0],'lumisection':j[1]}
293 
294 #____________________________
295 def get_plots(path,output, iovs, tag):
296 
297  initial=iovs[len(iovs)-1].IOVfirst
298  final =iovs[0].IOVfirst
299  if iovs[0].type == "lumiid":
300  initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
301  final = str(unpack(final)[0])+":"+str(unpack(final)[1])
302 
303  initial = str(int(initial) -100 )
304  cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
305  print(cmd)
306  outcmd = commands.getstatusoutput( cmd )
307  print(outcmd[1])
308 
309  cmd = "ls *.png"
310  outcmd = commands.getstatusoutput( cmd )
311 
312  pngfiles = outcmd[1].split('\n')
313  print(pngfiles)
314 
315  cmd = "cp *.png "+os.path.dirname(output)
316  outcmd = commands.getstatusoutput( cmd )
317  cmd = "rm *.png"
318  outcmd = commands.getstatusoutput( cmd )
319 
320  pngfiles.sort()
321  return pngfiles
322 
323 #_______________________________
324 def write_plots(lines, plots,web):
325 
326  end = '\n'
327  br = '<BR>'+end
328 
329  lines.append(br)
330  lines.append('''
331 <table border="1">
332 
333 ''')
334  for i in range(0,len(plots)):
335  plot = plots[i]
336  plot = os.path.basename(plot)
337  if i%2 == 0:
338  lines.append("<tr>"+end)
339  lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
340  if i%2 == 1:
341  lines.append("</tr>"+end)
342 
343  lines.append('</table>'+end)
344 
345 #________________________________
346 def get_productionFiles( directory ):
347 
348  list = commands.getstatusoutput('ls -t '+directory)
349  list = list[1].split()
350  newlist = []
351  for i in list:
352  if i.find('BeamFit_')!=-1:
353  newlist.append(i)
354 
355  return newlist
356 #_______________________________
357 def get_productionIOVs( directory ):
358 
359  files = get_productionFiles( directory )
360  listofruns = []
361  for f in files:
362  ii = f.find('Run')
363  arun = f[ii:len(f)-4]
364  listofruns.append(arun)
365  return listofruns
366 
367 #______________________________
368 if __name__ == '__main__':
369 
370 
371  # COMMAND LINE OPTIONS
372  #################################
373  option,args = parse(__doc__)
374  if not args and not option: exit()
375 
376  #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
377  htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
378  if option.web: htmlwebsite = option.web
379 
380  ## Get the latest tags
381  #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
382  dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
383  auth = "/afs/cern.ch/cms/DB/conddb"
384  #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb -a
385 
386  list_tags = get_listoftags( dest, auth)
387 
388  # Get the latest IOVs from last tag
389  list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
390 
391  # get latest processed runs
392  processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
393 
394  # create web page
395  lines = []
396  end = '\n'
397  br = '<BR>'+end
398 
399  dump_header(lines)
400 
401  lines.append('<h2>The three latest IOVs</h2>'+end)
402  lines.append(br)
403  lines.append('''
404 <table border="1">
405 ''')
406  write_iovs( list_lastIOVs, lines )
407  lines.append('</table>'+end)
408 
409  lines.append('<h2>Latest data processed</h2>'+end)
410  lines.append(br)
411  #lines.append('to be written'+end)
412  lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
413  print(processedruns)
414  lines.append(br)
415 
416  lines.append('<h2>The Latest Tags</h2>'+end)
417  lines.append(br)
418  lines.append('''
419 <table border="1">
420 ''')
421  write_tags( list_tags, lines)
422  lines.append('</table>'+end)
423 
424  lasttag = list_lastIOVs.keys()[0]
425  lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
426  lines.append(br)
427 
428  pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
429  write_plots( lines, pngfiles, htmlwebsite )
430 
431  dump_footer(lines)
432 
433  outfile = open(option.output,'w')
434  #print lines
435  outfile.writelines( lines )
436 
437 
438 
439 
440 
441 
442 
443 
beamvalidation.parse
def parse(docstring, arglist=None)
Definition: beamvalidation.py:55
beamvalidation.get_productionIOVs
def get_productionIOVs(directory)
Definition: beamvalidation.py:357
beamvalidation.write_plots
def write_plots(lines, plots, web)
Definition: beamvalidation.py:324
beamvalidation.get_productionFiles
def get_productionFiles(directory)
Definition: beamvalidation.py:346
edm::print
S & print(S &os, JobReport::InputFile const &f)
Definition: JobReport.cc:66
beamvalidation.dump_header
def dump_header(lines)
Definition: beamvalidation.py:107
mathSSE::return
return((rh^lh)&mask)
beamvalidation.get_listoftags
def get_listoftags(dest, auth)
Definition: beamvalidation.py:186
beamvalidation.write_iovs
def write_iovs(iovs, lines)
Definition: beamvalidation.py:155
beamvalidation.get_plots
def get_plots(path, output, iovs, tag)
Definition: beamvalidation.py:295
beamvalidation.write_tags
def write_tags(tags, lines)
Definition: beamvalidation.py:132
beamvalidation.get_lastIOVs
def get_lastIOVs(listoftags, dest, auth)
Definition: beamvalidation.py:230
beamvalidation.dump_footer
def dump_footer(lines)
Definition: beamvalidation.py:127
beamvalidation.nonzero
def nonzero(self)
Definition: beamvalidation.py:40
beamvalidation.pack
def pack(high, low)
Definition: beamvalidation.py:274
str
#define str(s)
Definition: TestProcessor.cc:47
split
double split
Definition: MVATrainer.cc:139
beamvalidation.exit
def exit(msg="")
Definition: beamvalidation.py:52
beamvalidation.cmp_tags
def cmp_tags(a, b)
Definition: beamvalidation.py:79
Generated for CMSSW Reference Manual by doxygen 1.8.11