00001 import math, re, optparse, commands, os, sys, time, datetime
00002 from BeamSpotObj import BeamSpot
00003 from IOVObj import IOV
00004
00005 lockFile = ".lock"
00006
00007
00008 def timeoutManager(type,timeout=-1,fileName=".timeout"):
00009 if timeout == 0:
00010 return 1
00011 timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
00012 currentTime = time.gmtime()
00013 timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
00014 isTimeout = False
00015 alreadyThere = False
00016 timeoutType = -1;
00017 fileExist = os.path.isfile(fileName)
00018 text = ''
00019 fields = []
00020 reset = False
00021 if timeout == -1:
00022 reset = True
00023 if fileExist:
00024 file = open(fileName)
00025 for line in file:
00026 text += line
00027 fields = line.strip('\n').split(' ')
00028 if fields[0] == type:
00029 alreadyThere = True
00030 if reset:
00031 text = text.replace(line,'')
00032 continue
00033
00034 fileTime = time.strptime(fields[1],timeFormat)
00035 myTime = time.mktime(fileTime)
00036 referenceTime = time.mktime(time.gmtime())
00037 daylight = 0
00038 if currentTime.tm_isdst == 0:
00039 daylight = 3600
00040 elapsedTime = referenceTime-myTime-daylight
00041 if elapsedTime > timeout:
00042 isTimeout = True
00043 timeoutType = 1
00044 print "Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!"
00045 else:
00046 timeoutType = 0
00047 print "Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1]
00048
00049 file.close()
00050
00051 if not fileExist or not alreadyThere and not reset:
00052 timeoutType = -1
00053 text += timeoutLine
00054
00055 if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
00056 if fileExist:
00057 commands.getstatusoutput("rm -rf " + fileName)
00058 file = open(fileName,'w')
00059 file.write(text)
00060 file.close()
00061
00062 return timeoutType
00063
00064
00065
00066 def setLockName(name):
00067 global lockFile
00068 lockFile = name
00069
00070
00071 def checkLock():
00072 global lockFile
00073 if os.path.isfile(lockFile):
00074 return True
00075 else:
00076 return False
00077
00078
00079 def lock():
00080 global lockFile
00081 commands.getstatusoutput( "touch " + lockFile)
00082
00083
00084 def rmLock():
00085 global lockFile
00086 if checkLock():
00087 commands.getstatusoutput( "rm " + lockFile)
00088
00089
00090 def exit(msg=""):
00091 rmLock()
00092 raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
00093
00094
00095 def isnan(num):
00096 fnum = float(num)
00097 return fnum != fnum
00098
00099
00100
00101
00102 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
00103
00104
00105 def parse(docstring, arglist=None):
00106 global optionstring
00107 global tagType
00108 optionstring = docstring
00109 match = USAGE.search(optionstring)
00110 if not match: raise ParsingError("Cannot find the option string")
00111 optlines = match.group(1).splitlines()
00112 try:
00113 p = optparse.OptionParser(optlines[0])
00114 for line in optlines[1:]:
00115 opt, help=line.split(':')[:2]
00116 short,long=opt.split(',')[:2]
00117 if '=' in opt:
00118 action='store'
00119 long=long.split('=')[0]
00120 else:
00121 action='store_true'
00122 p.add_option(short.strip(),long.strip(),
00123 action = action, help = help.strip())
00124 except (IndexError,ValueError):
00125 raise ParsingError("Cannot parse the option string correctly")
00126 return p.parse_args(arglist)
00127
00128
00129 def nonzero(self):
00130 "True if options were given"
00131 for v in self.__dict__.itervalues():
00132 if v is not None: return True
00133 return False
00134
00135
00136 optparse.Values.__nonzero__ = nonzero
00137
00138
00139
00140
00141
00142 class ParsingError(Exception): pass
00143
00144
00145
00146
00147
00148 def sendEmail(mailList,error):
00149 print "Sending email to " + mailList + " with body: " + error
00150 list = mailList.split(',')
00151 for email in list:
00152 p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
00153 p.write(error)
00154 status = p.close()
00155
00156
00157 def dirExists(dir):
00158 if dir.find("castor") != -1:
00159 lsCommand = "nsls " + dir
00160 output = commands.getstatusoutput( lsCommand )
00161 return not output[0]
00162 else:
00163 return os.path.exists(dir)
00164
00165
00166 def ls(dir,filter=""):
00167 lsCommand = ''
00168 listOfFiles = []
00169 if dir.find('castor') != -1:
00170 lsCommand = 'ns'
00171 elif not os.path.exists(dir):
00172 print "ERROR: File or directory " + dir + " doesn't exist"
00173 return listOfFiles
00174
00175 aCommand = lsCommand + 'ls '+ dir
00176
00177 if filter != "":
00178 aCommand += " | grep " + filter
00179
00180 tmpStatus = commands.getstatusoutput( aCommand )
00181 listOfFiles = tmpStatus[1].split('\n')
00182 if len(listOfFiles) == 1:
00183 if listOfFiles[0].find('No such file or directory') != -1:
00184 exit("ERROR: File or directory " + dir + " doesn't exist")
00185
00186 return listOfFiles
00187
00188
00189 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
00190 cpCommand = ''
00191 copiedFiles = []
00192 if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
00193 cpCommand = 'rf'
00194 elif fromDir.find('resilient') != -1:
00195 cpCommand = 'dc'
00196 if fromDir[len(fromDir)-1] != '/':
00197 fromDir += '/'
00198
00199 if toDir[len(toDir)-1] != '/':
00200 toDir += '/'
00201
00202 for file in listOfFiles:
00203 if os.path.isfile(toDir+file):
00204 if overwrite:
00205 print "File " + file + " already exists in destination directory. We will overwrite it."
00206 else:
00207 print "File " + file + " already exists in destination directory. We will Keep original file."
00208 if not smallList:
00209 copiedFiles.append(file)
00210 continue
00211
00212 aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
00213 print " >> " + aCommand
00214 tmpStatus = commands.getstatusoutput( aCommand )
00215 if tmpStatus[0] == 0:
00216 copiedFiles.append(file)
00217 else:
00218 print "[cp()]\tERROR: Can't copy file " + file
00219 return copiedFiles
00220
00221
00222
00223
00224
00225
00226
00227 def pack(high,low):
00228 """pack high,low 32bit unsigned int to one unsigned 64bit long long
00229 Note:the print value of result number may appear signed, if the sign bit is used.
00230 """
00231 h=high<<32
00232 return (h|low)
00233
00234
00235 def unpack(i):
00236 """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
00237 """
00238 high=i>>32
00239 low=i&0xFFFFFFFF
00240 return(high,low)
00241
00242
00243 def unpackLumiid(i):
00244 """unpack 64bit lumiid to dictionary {'run','lumisection'}
00245 """
00246 j=unpack(i)
00247 return {'run':j[0],'lumisection':j[1]}
00248
00249
00250
00251
00252
00253 def cmp_list_run(a,b):
00254 if int(a.IOVfirst) < int(b.IOVfirst): return -1
00255 if int(a.IOVfirst) == int(b.IOVfirst): return 0
00256 if int(a.IOVfirst) > int(b.IOVfirst): return 1
00257
00258
00259 def cmp_list_lumi(a,b):
00260 if int(a.Run) < int(b.Run): return -1
00261 if int(a.Run) == int(b.Run):
00262 if int(a.IOVfirst) < int(b.IOVfirst): return -1
00263 if int(a.IOVfirst) == int(b.IOVfirst): return 0
00264 if int(a.IOVfirst) > int(b.IOVfirst): return 1
00265 if int(a.Run) > int(b.Run) : return 1
00266
00267
00268 def weight(x1, x1err,x2,x2err):
00269
00270 x1 = float(x1)
00271 x1err = float(x1err)
00272 x2 = float(x2)
00273 x2err = float(x2err)
00274 tmperr = 0.
00275 if x2err < 1e-6 :
00276 x2err = 1e-6
00277 if x1err < 1e-6:
00278 x1 = x2/(x2err * x2err)
00279 tmperr = 1/(x2err*x2err)
00280 else:
00281 x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
00282 tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
00283 x1 = x1/tmperr
00284 x1err = 1/tmperr
00285 x1err = math.sqrt(x1err)
00286 return (str(x1), str(x1err))
00287
00288
00289 def dump( beam, file):
00290 end = "\n"
00291 file.write("Runnumber "+beam.Run+end)
00292 file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
00293 file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
00294 file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
00295 dumpValues(beam, file)
00296
00297
00298 def dumpValues( beam, file):
00299 end = "\n"
00300 file.write("Type "+str(beam.Type)+end)
00301 file.write("X0 "+str(beam.X)+end)
00302 file.write("Y0 "+str(beam.Y)+end)
00303 file.write("Z0 "+str(beam.Z)+end)
00304 file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
00305 file.write("dxdz "+str(beam.dxdz)+end)
00306 file.write("dydz "+str(beam.dydz)+end)
00307 file.write("BeamWidthX "+beam.beamWidthX+end)
00308 file.write("BeamWidthY "+beam.beamWidthY+end)
00309 file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0" +end)
00310 file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0" +end)
00311 file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0" +end)
00312 file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0" +end)
00313 file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0" +end)
00314 file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0" +end)
00315 file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2)) +end)
00316 file.write("EmittanceX 0"+end)
00317 file.write("EmittanceY 0"+end)
00318 file.write("BetaStar 0"+end)
00319
00320
00321 def delta(x,xerr,nextx,nextxerr):
00322
00323 return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
00324
00325 def deltaSig( x ):
00326 return math.fabs(x[0])/x[1]
00327
00328
00329 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
00330 tmpbeam = BeamSpot()
00331 tmpbeamsize = 0
00332
00333
00334
00335 if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
00336 firstRun = "1:1"
00337 lastRun = "4999999999:4999999999"
00338
00339 inputfiletype = 0
00340
00341
00342
00343
00344 maplist = {}
00345 hasBX = False
00346
00347 tmpfile = open(fileName)
00348 atmpline = tmpfile.readline()
00349 if atmpline.find('Runnumber') != -1:
00350 inputfiletype = 1
00351 if len(atmpline.split()) > 2:
00352 hasBX = True
00353 print " Input data has been calculated as function of BUNCH CROSSINGS."
00354 tmpfile.seek(0)
00355
00356
00357 if inputfiletype ==1:
00358
00359 tmpBX = 0
00360 for line in tmpfile:
00361
00362 if line.find('Type') != -1:
00363 tmpbeam.Type = int(line.split()[1])
00364 tmpbeamsize += 1
00365 if line.find('X0') != -1:
00366 tmpbeam.X = line.split()[1]
00367
00368 tmpbeamsize += 1
00369
00370 if line.find('Y0') != -1:
00371 tmpbeam.Y = line.split()[1]
00372
00373 tmpbeamsize += 1
00374
00375 if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
00376 tmpbeam.Z = line.split()[1]
00377
00378 tmpbeamsize += 1
00379 if line.find('sigmaZ0') !=-1:
00380 tmpbeam.sigmaZ = line.split()[1]
00381
00382 tmpbeamsize += 1
00383 if line.find('dxdz') != -1:
00384 tmpbeam.dxdz = line.split()[1]
00385
00386 tmpbeamsize += 1
00387 if line.find('dydz') != -1:
00388 tmpbeam.dydz = line.split()[1]
00389
00390 tmpbeamsize += 1
00391 if line.find('BeamWidthX') != -1:
00392 tmpbeam.beamWidthX = line.split()[1]
00393
00394 tmpbeamsize += 1
00395 if line.find('BeamWidthY') != -1:
00396 tmpbeam.beamWidthY = line.split()[1]
00397
00398 tmpbeamsize += 1
00399 if line.find('Cov(0,j)') != -1:
00400 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
00401 tmpbeamsize += 1
00402 if line.find('Cov(1,j)') != -1:
00403 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
00404 tmpbeamsize += 1
00405 if line.find('Cov(2,j)') != -1:
00406 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
00407 tmpbeamsize += 1
00408 if line.find('Cov(3,j)') != -1:
00409 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
00410 tmpbeamsize += 1
00411 if line.find('Cov(4,j)') != -1:
00412 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
00413 tmpbeamsize += 1
00414 if line.find('Cov(5,j)') != -1:
00415 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
00416 tmpbeamsize += 1
00417 if line.find('Cov(6,j)') != -1:
00418 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
00419 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
00420 tmpbeamsize += 1
00421 if line.find('LumiRange') != -1:
00422 if IOVbase=="lumibase":
00423 tmpbeam.IOVfirst = line.split()[1]
00424 tmpbeam.IOVlast = line.split()[3]
00425 tmpbeamsize += 1
00426 if line.find('Runnumber') != -1:
00427 tmpbeam.Run = line.split()[1]
00428 if IOVbase == "runbase":
00429 tmpbeam.IOVfirst = line.split()[1]
00430 tmpbeam.IOVlast = line.split()[1]
00431 if hasBX:
00432 tmpBX = line.split()[3]
00433 tmpbeamsize += 1
00434 if line.find('BeginTimeOfFit') != -1:
00435 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
00436 if IOVbase =="timebase":
00437 tmpbeam.IOVfirst = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
00438 tmpbeamsize += 1
00439 if line.find('EndTimeOfFit') != -1:
00440 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
00441 if IOVbase =="timebase":
00442 tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
00443 tmpbeamsize += 1
00444 if tmpbeamsize == 20:
00445 if IOVbase=="lumibase":
00446 tmprunfirst = int(firstRun.split(":")[0])
00447 tmprunlast = int(lastRun.split(":")[0])
00448 tmplumifirst = int(firstRun.split(":")[1])
00449 tmplumilast = int(lastRun.split(":")[1])
00450 acceptiov1 = acceptiov2 = False
00451
00452 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
00453 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
00454 acceptiov1 = acceptiov2 = True
00455
00456 elif int(tmpbeam.Run) == tmprunfirst:
00457 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
00458 elif int(tmpbeam.Run) == tmprunlast:
00459 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
00460 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
00461 acceptiov1 = acceptiov2 = True
00462
00463 if acceptiov1 and acceptiov2:
00464 if tmpbeam.Type != 2:
00465 print "invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00466 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00467 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00468 elif hasBX:
00469 if maplist.has_key(tmpBX) == False:
00470 maplist[tmpBX] = [tmpbeam]
00471 else:
00472 maplist[tmpBX].append(tmpbeam)
00473 else:
00474 listbeam.append(tmpbeam)
00475
00476 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
00477 if tmpbeam.Type != 2:
00478 print "invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00479 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00480 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00481 else:
00482 listbeam.append(tmpbeam)
00483
00484 tmpbeamsize = 0
00485 tmpbeam = BeamSpot()
00486 tmpBX = 0
00487 else:
00488
00489 for line in tmpfile:
00490
00491 if line.find('X0') != -1:
00492 tmpbeam.X = line.split()[2]
00493 tmpbeam.Xerr = line.split()[4]
00494 tmpbeamsize += 1
00495
00496 if line.find('Y0') != -1:
00497 tmpbeam.Y = line.split()[2]
00498 tmpbeam.Yerr = line.split()[4]
00499 tmpbeamsize += 1
00500
00501 if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
00502 tmpbeam.Z = line.split()[2]
00503 tmpbeam.Zerr = line.split()[4]
00504 tmpbeamsize += 1
00505
00506 if line.find('Sigma Z0') !=-1:
00507 tmpbeam.sigmaZ = line.split()[3]
00508 tmpbeam.sigmaZerr = line.split()[5]
00509 tmpbeamsize += 1
00510 if line.find('dxdz') != -1:
00511 tmpbeam.dxdz = line.split()[2]
00512 tmpbeam.dxdzerr = line.split()[4]
00513 tmpbeamsize += 1
00514 if line.find('dydz') != -1:
00515 tmpbeam.dydz = line.split()[2]
00516 tmpbeam.dydzerr = line.split()[4]
00517 tmpbeamsize += 1
00518 if line.find('Beam Width X') != -1:
00519 tmpbeam.beamWidthX = line.split()[4]
00520 tmpbeam.beamWidthXerr = line.split()[6]
00521 tmpbeamsize += 1
00522 if line.find('Beam Width Y') != -1:
00523 tmpbeam.beamWidthY = line.split()[4]
00524 tmpbeam.beamWidthYerr = line.split()[6]
00525 tmpbeamsize += 1
00526
00527 if line.find('for runs') != -1:
00528
00529 tmpbeam.Run = line.split()[2]
00530 if IOVbase == "runbase":
00531 tmpbeam.IOVfirst = line.split()[2]
00532 tmpbeam.IOVlast = line.split()[4]
00533 tmpbeamsize += 1
00534 if line.find('LumiSection') != -1:
00535 if IOVbase=="lumibase":
00536 tmpbeam.IOVfirst = line.split()[10]
00537 tmpbeam.IOVlast = line.split()[10]
00538 tmpbeamsize += 1
00539 if tmpbeamsize == 10:
00540
00541 if IOVbase=="lumibase":
00542 tmprunfirst = int(firstRun.split(":")[0])
00543 tmprunlast = int(lastRun.split(":")[0])
00544 tmplumifirst = int(firstRun.split(":")[1])
00545 tmplumilast = int(lastRun.split(":")[1])
00546 acceptiov1 = acceptiov2 = False
00547
00548 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
00549 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
00550 acceptiov1 = acceptiov2 = True
00551
00552 elif int(tmpbeam.Run) == tmprunfirst:
00553 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
00554 elif int(tmpbeam.Run) == tmprunlast:
00555 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
00556 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
00557 acceptiov1 = acceptiov2 = True
00558
00559 if acceptiov1 and acceptiov2:
00560 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00561 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00562 elif hasBX:
00563 if maplist.has_key(tmpBX) == False:
00564 maplist[tmpBX] = [tmpbeam]
00565 else:
00566 maplist[tmpBX].append(tmpbeam)
00567 else:
00568 listbeam.append(tmpbeam)
00569
00570 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
00571 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00572 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00573 else:
00574 listbeam.append(tmpbeam)
00575
00576 tmpbeamsize = 0
00577 tmpbeam = BeamSpot()
00578 tmpBX = 0
00579
00580 tmpfile.close()
00581 print " got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName
00582
00583 if hasBX:
00584 return maplist
00585 else:
00586 return listbeam
00587
00588
00589
00590 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
00591
00592 if IOVbase == "lumibase":
00593 listbeam.sort( cmp = cmp_list_lumi )
00594 else:
00595 listbeam.sort( cmp = cmp_list_run )
00596
00597
00598 tmpremovelist = []
00599 for ii in range(0,len(listbeam)):
00600 ibeam = listbeam[ii]
00601 datax = ibeam.IOVfirst
00602
00603 if datax == '0' and IOVbase =="runbase":
00604 print " iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast)
00605 tmpremovelist.append(ibeam)
00606
00607 if ii < len(listbeam) -1:
00608
00609 if IOVbase =="lumibase":
00610 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
00611 print " duplicate IOV = "+datax+", keep only last duplicate entry"
00612 tmpremovelist.append(ibeam)
00613 elif datax == listbeam[ii+1].IOVfirst:
00614 print " duplicate IOV = "+datax+", keep only last duplicate entry"
00615 tmpremovelist.append(ibeam)
00616
00617 for itmp in tmpremovelist:
00618 listbeam.remove(itmp)
00619
00620
00621
00622 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
00623 newlistbeam = []
00624 tmpbeam = BeamSpot()
00625 docreate = True
00626 countlumi = 0
00627 tmprun = ""
00628 maxNlumis = 60
00629 if weighted:
00630 maxNlumis = 999999999
00631 for ii in range(0,len(listbeam)):
00632
00633 ibeam = listbeam[ii]
00634 inextbeam = BeamSpot()
00635 iNNbeam = BeamSpot()
00636 if docreate:
00637 tmpbeam.IOVfirst = ibeam.IOVfirst
00638 tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
00639 tmpbeam.Run = ibeam.Run
00640 tmpbeam.Type = 2
00641 docheck = False
00642 docreate = False
00643
00644
00645 if ii < len(listbeam) - 1:
00646 inextbeam = listbeam[ii+1]
00647 docheck = True
00648 if ii < len(listbeam) -2:
00649 iNNbeam = listbeam[ii+2]
00650 else:
00651 print "close payload because end of data has been reached. Run "+ibeam.Run
00652 docreate = True
00653
00654 if ibeam.Run != inextbeam.Run:
00655 print "close payload because end of run "+ibeam.Run
00656 docreate = True
00657
00658 if countlumi == maxNlumis -1:
00659 print "close payload because maximum lumi sections accumulated within run "+ibeam.Run
00660 docreate = True
00661 countlumi = 0
00662
00663 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
00664 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
00665 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
00666 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
00667 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
00668 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
00669
00670 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
00671 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
00672
00673 if weighted:
00674 docheck = False
00675
00676
00677 if docheck:
00678
00679
00680 min_limit = 0.0025
00681
00682
00683 limit = float(ibeam.beamWidthX)/2.
00684 if limit < min_limit: limit = min_limit
00685
00686
00687 adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
00688 adelta2 = (0.,1.e9)
00689 adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
00690 adelta2dxdz = (0.,1.e9)
00691 adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
00692 adelta2dydz = (0.,1.e9)
00693 adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
00694 adelta2widthx = (0.,1.e9)
00695 adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
00696 adelta2widthy = (0.,1.e9)
00697
00698 if iNNbeam.Type != -1:
00699 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
00700 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
00701 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
00702 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
00703 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
00704
00705 deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
00706 if ii < len(listbeam) -2:
00707 if deltaX==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
00708
00709 deltaX = True
00710 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
00711 deltaX = False
00712
00713
00714
00715
00716
00717 adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
00718 adelta2 = (0.,1.e9)
00719 if iNNbeam.Type != -1:
00720 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
00721
00722 deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
00723 if ii < len(listbeam) -2:
00724 if deltaY==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
00725 deltaY = True
00726 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
00727 deltaY = False
00728
00729 adelta = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
00730
00731
00732 limit = float(ibeam.sigmaZ)/2.
00733 deltaZ = deltaSig(adelta) > 3.5 and adelta[0] >= limit
00734
00735 adelta = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
00736 deltasigmaZ = deltaSig(adelta) > 5.0
00737
00738
00739 adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
00740 deltadxdz = deltaSig(adelta) > 5.0
00741 if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
00742 deltadxdz = False
00743
00744 adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
00745 deltadydz = deltaSig(adelta) > 5.0
00746 if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
00747 deltadydz = False
00748
00749 adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
00750 deltawidthX = deltaSig(adelta) > 5
00751 if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
00752 deltawidthX = False
00753
00754 adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
00755 deltawidthY = deltaSig(adelta) > 5
00756 if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
00757 deltawidthY = False
00758
00759
00760
00761
00762
00763
00764
00765
00766
00767
00768
00769
00770 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
00771 docreate = True
00772
00773
00774
00775
00776
00777
00778 print "close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY)
00779 if docreate:
00780
00781 tmpbeam.IOVlast = ibeam.IOVlast
00782 tmpbeam.IOVEndTime = ibeam.IOVEndTime
00783 print " Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast)
00784 newlistbeam.append(tmpbeam)
00785 tmpbeam = BeamSpot()
00786 countlumi = 0
00787 tmprun = ibeam.Run
00788 countlumi += 1
00789
00790 payloadfile = open(fileName,"w")
00791 for iload in newlistbeam:
00792 dump( iload, payloadfile )
00793 payloadfile.close()
00794 return newlistbeam
00795
00796 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
00797 newlistbeam = []
00798 docreate = False
00799 docheck = False
00800 lastPayload = listbeam[0]
00801
00802 firstToUse = 0
00803 lastToUse = 0
00804 for ii in range(0,len(listbeam)):
00805 docreate = False
00806 if docheck:
00807 deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
00808 deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
00809 deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
00810
00811 deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
00812 deltadxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
00813 deltadydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
00814
00815 deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
00816 deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
00817
00818
00819
00820
00821
00822
00823
00824
00825
00826
00827
00828
00829
00830 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
00831 if ii != 0:
00832 docreate = True
00833 lastToUse = ii-1
00834
00835
00836
00837
00838
00839
00840 print "close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY)
00841
00842
00843 if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
00844 print "close payload because end of run has been reached. Run " + listbeam[ii].Run
00845 docreate = True
00846 lastToUse = ii
00847
00848
00849
00850
00851
00852
00853
00854 if docreate:
00855 tmpbeam = BeamSpot()
00856 for ibeam in listbeam[firstToUse:lastToUse]:
00857 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
00858 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
00859 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
00860 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
00861 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
00862 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
00863
00864 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
00865 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
00866 tmpbeam.IOVfirst = listbeam[firstToUse].IOVfirst
00867 tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
00868 tmpbeam.Run = listbeam[firstToUse].Run
00869 tmpbeam.Type = 2
00870 tmpbeam.IOVlast = listbeam[lastToUse].IOVlast
00871 tmpbeam.IOVEndTime = listbeam[lastToUse].IOVEndTime
00872 newlistbeam.append(tmpbeam)
00873 firstToUse = lastToUse+1
00874 print "Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast)
00875
00876 payloadfile = open(fileName,"w")
00877 for iload in newlistbeam:
00878 dump( iload, payloadfile )
00879 payloadfile.close()
00880
00881
00882 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
00883 writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
00884 wFile = open(sqliteTemplateFile)
00885 wNewFile = open(writeDBOut,'w')
00886
00887 writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
00888 ('TAGNAME',tagName),
00889 ('TIMETYPE',timeType),
00890 ('BEAMSPOTFILE',beamSpotFile)]
00891
00892 for line in wFile:
00893 for itag in writeDBTags:
00894 line = line.replace(itag[0],itag[1])
00895 wNewFile.write(line)
00896
00897 wNewFile.close()
00898 print "writing sqlite file ..."
00899 status_wDB = commands.getstatusoutput('cmsRun '+ writeDBOut)
00900 print status_wDB[1]
00901
00902 os.system("rm -f " + writeDBOut)
00903 return not status_wDB[0]
00904
00905
00906 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
00907 readDBOut = tmpDir + "readDB_" + tagName + ".py"
00908
00909 rFile = open(sqliteTemplateFile)
00910 rNewFile = open(readDBOut,'w')
00911
00912 readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
00913 ('TAGNAME',tagName)]
00914
00915 for line in rFile:
00916 for itag in readDBTags:
00917 line = line.replace(itag[0],itag[1])
00918 rNewFile.write(line)
00919
00920 rNewFile.close()
00921 status_rDB = commands.getstatusoutput('cmsRun '+ readDBOut)
00922
00923 outtext = status_rDB[1]
00924 print outtext
00925 os.system("rm -f " + readDBOut)
00926 return not status_rDB[0]
00927
00928
00929 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
00930 aCommand = "cmscond_export_iov -d sqlite_file:" + tmpDir + combinedSqliteFileName + " -s sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -l sqlite_file:" + tmpDir + "log.db" + " -b " + IOVSince + " -e " + IOVTill
00931 print aCommand
00932 std = commands.getstatusoutput(aCommand)
00933 print std[1]
00934 return not std[0]
00935
00936
00937 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
00938
00939 acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
00940 outcmd = commands.getstatusoutput(acmd)
00941 print acmd
00942
00943 if outcmd[0]:
00944 print "Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt"
00945 return False
00946
00947 acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
00948 print acmd
00949 outcmd = commands.getstatusoutput(acmd)
00950 print outcmd[1]
00951 if outcmd[0]:
00952 print "Couldn't cd to " + sqliteFileDirName
00953 return False
00954
00955 acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
00956 print acmd
00957 outcmd = commands.getstatusoutput(acmd)
00958 print outcmd[1]
00959 if outcmd[0]:
00960 print "Couldn't zip the files!"
00961 return False
00962
00963 acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
00964 outcmd = commands.getstatusoutput(acmd)
00965 print acmd
00966
00967 if outcmd[0]:
00968 print "Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2"
00969 return False
00970
00971 acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
00972 print acmd
00973 outcmd = commands.getstatusoutput(acmd)
00974 print outcmd[1]
00975 if outcmd[0]:
00976 print "Couldn't scp the files to DropBox!"
00977 return False
00978
00979
00980 acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
00981 print acmd
00982 outcmd = commands.getstatusoutput(acmd)
00983 print outcmd[1]
00984 if outcmd[0]:
00985 print "Couldn't mv the file to " + sqliteFileDirName
00986 return False
00987
00988 acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
00989 print acmd
00990 outcmd = commands.getstatusoutput(acmd)
00991 print outcmd[1]
00992 if outcmd[0]:
00993 print "Couldn't rm the db and txt files"
00994 return False
00995
00996
00997
00998
00999
01000
01001
01002
01003
01004
01005
01006
01007
01008
01009
01010 return False
01011
01012
01013
01014
01015
01016
01017
01018
01019
01020 return True
01021
1.7.3