CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
beamvalidation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #____________________________________________________________
3 #
4 #
5 # A very simple
6 #
7 # Francisco Yumiceva
8 # yumiceva@fnal.gov
9 #
10 # Fermilab, 2010
11 #
12 #____________________________________________________________
13 
14 """
15  beam spot validation
16 
17  A very simple script
18 
19  usage: %prog -t <tag name>
20  -o, --output = OUTPUT: filename of output html file.
21  -p, --path = PATH: path to beam spot scripts.
22  -w, --web = WEB: html path to website.
23 
24  Francisco Yumiceva (yumiceva@fnal.gov)
25  Fermilab 2010
26 
27 """
28 
29 
30 import os, string, re, sys, math
31 import commands, time
32 
33 #_______________OPTIONS________________
34 import optparse
35 
36 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
37 
38 def nonzero(self): # will become the nonzero method of optparse.Values
39  "True if options were given"
40  for v in self.__dict__.itervalues():
41  if v is not None: return True
42  return False
43 
44 optparse.Values.__nonzero__ = nonzero # dynamically fix optparse.Values
45 
46 class ParsingError(Exception): pass
47 
48 optionstring=""
49 
50 def exit(msg=""):
51  raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
52 
53 def parse(docstring, arglist=None):
54  global optionstring
55  optionstring = docstring
56  match = USAGE.search(optionstring)
57  if not match: raise ParsingError("Cannot find the option string")
58  optlines = match.group(1).splitlines()
59  try:
60  p = optparse.OptionParser(optlines[0])
61  for line in optlines[1:]:
62  opt, help=line.split(':')[:2]
63  short,long=opt.split(',')[:2]
64  if '=' in opt:
65  action='store'
66  long=long.split('=')[0]
67  else:
68  action='store_true'
69  p.add_option(short.strip(),long.strip(),
70  action = action, help = help.strip())
71  except (IndexError,ValueError):
72  raise ParsingError("Cannot parse the option string correctly")
73  return p.parse_args(arglist)
74 
75 #_______________________________
76 
77 def cmp_tags(a,b):
78 
79  yeara = int(a.split('_')[1])
80  yearb = int(b.split('_')[1])
81 
82  if yeara < yearb: return -1
83  if yeara > yearb: return 1
84 
85  suffix = "_offline"
86  if a.find("_express") != -1:
87  suffix = "_express"
88  if a.find("_prompt") != -1:
89  suffix = "_prompt"
90 
91  tmpa = a.replace("BeamSpotObjects_2009_v","")
92  tmpa = tmpa.replace(suffix,"")
93 
94  tmpb = b.replace("BeamSpotObjects_2009_v","")
95  tmpb = tmpb.replace(suffix,"")
96 
97  na = int(tmpa)
98  nb = int(tmpb)
99  if na < nb: return -1
100  if na == nb: return 0
101  if na > nb: return 1
102 
103 #___
104 
105 def dump_header(lines):
106 
107  lines.append('''
108 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"><html>
109 <head><title>Beam Spot Calibration Status</title></head>
110 
111 <body>
112 
113 <H1>Status of the Beam Spot Calibration</H1>
114 <BR>
115 <BR>
116 This page is updated every 10 minutes:
117 <BR>
118 <strong><script src="datemod.js"
119 type="text/javascript"></script></strong>
120 
121 ''')
122 
123 #____
124 
125 def dump_footer(lines):
126 
127  lines.append('</body>\n</html>\n')
128 
129 #______________
130 def write_tags(tags, lines):
131 
132  end = '\n'
133  br = '<BR>'+end
134 
135  lines.append('<tr>'+end)
136  for i in tags.keys():
137 
138  lines.append('<th>'+i)
139  lines.append('</th>'+end)
140  lines.append('</tr>'+end)
141 
142  for ntags in range(0,len(tags['offline'])):
143  lines.append('<tr>'+end)
144  for i in tags.keys():
145  alist = tags[i]
146  if ntags < len(tags[i]):
147  lines.append('<td> '+alist[ntags]+' </td>'+end)
148  else:
149  lines.append('<td> </td>')
150  lines.append('</tr>'+end)
151 
152 #______________
153 def write_iovs(iovs, lines):
154 
155  end = '\n'
156  br = '<BR>'+end
157  # hearder
158  lines.append('<tr>'+end)
159  for i in iovs.keys():
160  lines.append('<th>'+i)
161  lines.append('</th>'+end)
162  lines.append('</tr>'+end)
163  # lumi type
164  lines.append('<tr>'+end)
165  for i in iovs.keys():
166  aIOVlist = iovs[i]
167  aIOV = IOV()
168  if len(aIOVlist) > 0:
169  aIOV = aIOVlist[0]
170  lines.append('<td> '+aIOV.type+' </td>'+end)
171  lines.append('</tr>'+end)
172  # print iovs
173  for niovs in range(0,len(iovs[iovs.keys()[0]])):
174  lines.append('<tr>'+end)
175  for i in iovs.keys():
176  aIOVlist = iovs[i]
177  aIOV = IOV()
178  if len(aIOVlist) > niovs:
179  aIOV = aIOVlist[niovs]
180  lines.append('<td> '+aIOV.IOVfirst +' - '+aIOV.IOVlast+' </td>'+end)
181  lines.append('</tr>'+end)
182 
183 #______________
184 def get_listoftags(dest, auth,):
185 
186  queryTags_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -a | grep BeamSpotObjects"
187  print queryTags_cmd
188  outcmd = commands.getstatusoutput( queryTags_cmd )
189  print outcmd[1]
190 
191  listtags = outcmd[1].split()
192 
193  listtags_offline = []
194  for itag in listtags:
195  if itag[len(itag)-7:len(itag)] == "offline":
196  listtags_offline.append(itag)
197  listtags_express = []
198  for itag in listtags:
199  if itag[len(itag)-7:len(itag)] == "express":
200  listtags_express.append(itag)
201  listtags_prompt = []
202  for itag in listtags:
203  if itag[len(itag)-6:len(itag)] == "prompt":
204  listtags_prompt.append(itag)
205 
206  listtags_offline.sort( cmp = cmp_tags )
207  listtags_offline.reverse()
208  listtags_express.sort( cmp = cmp_tags )
209  listtags_express.reverse()
210  listtags_prompt.sort( cmp = cmp_tags )
211  listtags_prompt.reverse()
212 
213  result = {}
214  result['offline'] = listtags_offline
215  result['express'] = listtags_express
216  result['prompt'] = listtags_prompt
217 
218  return result
219 
220 #______________________
221 class IOV:
222  def __init__(self):
223  self.type = ''
224  self.IOVfirst = ''
225  self.IOVlast = ''
226 
227 #_________________
228 def get_lastIOVs( listoftags, dest, auth ):
229 
230  dbtags = ['offline','express','prompt']
231 
232  results = {}
233  for itag in dbtags:
234 
235  lasttag = listoftags[itag][0]
236  #fix for the moment to read old tags
237  if itag != "offline":
238  lasttag = listoftags[itag][1]
239 
240  queryIOVs_cmd = "cmscond_list_iov -c "+dest+" -P "+auth+" -t "+ lasttag
241  print queryIOVs_cmd
242 
243  outcmd = commands.getstatusoutput( queryIOVs_cmd )
244 
245  tmparr = outcmd[1].split('\n')
246 
247  TimeType = tmparr[1].split()[1]
248  listIOVs = []
249 
250  # look at number of payloads
251  lastline = tmparr[len(tmparr)-1].split()
252  npayloads = int( lastline[len(lastline)-1] )
253 
254  maxIOVs = 3
255  if npayloads < 3:
256  maxIOVs = npayloads
257  # pick the last three IOVs
258  for i in range(0,maxIOVs):
259  tmpline = tmparr[len(tmparr) -2 -i]
260  aIOV = IOV()
261  aIOV.IOVfirst = tmpline.split()[0]
262  aIOV.IOVlast = tmpline.split()[1]
263  aIOV.type = TimeType
264  listIOVs.append( aIOV )
265 
266  results[lasttag] = listIOVs
267 
268  return results
269 
270 #
271 # lumi tools CondCore/Utilities/python/timeUnitHelper.py
272 def pack(high,low):
273  """pack high,low 32bit unsigned int to one unsigned 64bit long long
274  Note:the print value of result number may appear signed, if the sign bit is used.
275  """
276  h=high<<32
277  return (h|low)
278 
279 def unpack(i):
280  """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
281  """
282  high=i>>32
283  low=i&0xFFFFFFFF
284  return(high,low)
285 
286 def unpackLumiid(i):
287  """unpack 64bit lumiid to dictionary {'run','lumisection'}
288  """
289  j=unpack(i)
290  return {'run':j[0],'lumisection':j[1]}
291 
292 #____________________________
293 def get_plots(path,output, iovs, tag):
294 
295  initial=iovs[len(iovs)-1].IOVfirst
296  final =iovs[0].IOVfirst
297  if iovs[0].type == "lumiid":
298  initial = str(unpack(initial)[0])+":"+str(unpack(initial)[1])
299  final = str(unpack(final)[0])+":"+str(unpack(final)[1])
300 
301  initial = str(int(initial) -100 )
302  cmd = path+"/plotBeamSpotDB.py -b -P -t "+tag+" -i "+initial +" -f "+final
303  print cmd
304  outcmd = commands.getstatusoutput( cmd )
305  print outcmd[1]
306 
307  cmd = "ls *.png"
308  outcmd = commands.getstatusoutput( cmd )
309 
310  pngfiles = outcmd[1].split('\n')
311  print pngfiles
312 
313  cmd = "cp *.png "+os.path.dirname(output)
314  outcmd = commands.getstatusoutput( cmd )
315  cmd = "rm *.png"
316  outcmd = commands.getstatusoutput( cmd )
317 
318  pngfiles.sort()
319  return pngfiles
320 
321 #_______________________________
322 def write_plots(lines, plots,web):
323 
324  end = '\n'
325  br = '<BR>'+end
326 
327  lines.append(br)
328  lines.append('''
329 <table border="1">
330 
331 ''')
332  for i in range(0,len(plots)):
333  plot = plots[i]
334  plot = os.path.basename(plot)
335  if i%2 == 0:
336  lines.append("<tr>"+end)
337  lines.append("<td> <a href=\""+web+"/"+plot+"\"> <img src="+plot+" alt="+plot+" width='700' height='250' /> </a> </td>"+end)
338  if i%2 == 1:
339  lines.append("</tr>"+end)
340 
341  lines.append('</table>'+end)
342 
343 #________________________________
344 def get_productionFiles( directory ):
345 
346  list = commands.getstatusoutput('ls -t '+directory)
347  list = list[1].split()
348  newlist = []
349  for i in list:
350  if i.find('BeamFit_')!=-1:
351  newlist.append(i)
352 
353  return newlist
354 #_______________________________
355 def get_productionIOVs( directory ):
356 
357  files = get_productionFiles( directory )
358  listofruns = []
359  for f in files:
360  ii = f.find('Run')
361  arun = f[ii:len(f)-4]
362  listofruns.append(arun)
363  return listofruns
364 
365 #______________________________
366 if __name__ == '__main__':
367 
368 
369  # COMMAND LINE OPTIONS
370  #################################
371  option,args = parse(__doc__)
372  if not args and not option: exit()
373 
374  #htmlwebsite = "http://cmsrocstor.fnal.gov/lpc1/cmsroc/yumiceva/tmp/"
375  htmlwebsite = "https://yumiceva.web.cern.ch/yumiceva/beamspot/"
376  if option.web: htmlwebsite = option.web
377 
378  ## Get the latest tags
379  #dest = "frontier://cmsfrontier.cern.ch:8000/Frontier/CMS_COND_31X_BEAMSPOT"
380  dest = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
381  auth = "/afs/cern.ch/cms/DB/conddb"
382  #cmscond_list_iov -c oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT -P /afs/cern.ch/cms/DB/conddb -a
383 
384  list_tags = get_listoftags( dest, auth)
385 
386  # Get the latest IOVs from last tag
387  list_lastIOVs = get_lastIOVs( list_tags, dest, auth)
388 
389  # get latest processed runs
390  processedruns = get_productionIOVs('/afs/cern.ch/cms/CAF/CMSCOMM/COMM_BSPOT/yumiceva/tmp_lumi_workflow/')
391 
392  # create web page
393  lines = []
394  end = '\n'
395  br = '<BR>'+end
396 
397  dump_header(lines)
398 
399  lines.append('<h2>The three latest IOVs</h2>'+end)
400  lines.append(br)
401  lines.append('''
402 <table border="1">
403 ''')
404  write_iovs( list_lastIOVs, lines )
405  lines.append('</table>'+end)
406 
407  lines.append('<h2>Latest data processed</h2>'+end)
408  lines.append(br)
409  #lines.append('to be written'+end)
410  lines.append(processedruns[0]+', '+processedruns[1]+', '+processedruns[2])
411  print processedruns
412  lines.append(br)
413 
414  lines.append('<h2>The Latest Tags</h2>'+end)
415  lines.append(br)
416  lines.append('''
417 <table border="1">
418 ''')
419  write_tags( list_tags, lines)
420  lines.append('</table>'+end)
421 
422  lasttag = list_lastIOVs.keys()[0]
423  lines.append('<h2>Plots of the latest IOVs from condDB tag: '+lasttag+' </h2>'+end)
424  lines.append(br)
425 
426  pngfiles = get_plots(option.path,option.output, list_lastIOVs[lasttag], lasttag)
427  write_plots( lines, pngfiles, htmlwebsite )
428 
429  dump_footer(lines)
430 
431  outfile = open(option.output,'w')
432  #print lines
433  outfile.writelines( lines )
434 
435 
436 
437 
438 
439 
440 
441 
reco::return
return(e1-e2)*(e1-e2)+dp *dp
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5