00001 import math, re, optparse, commands, os, sys, time, datetime
00002 from BeamSpotObj import BeamSpot
00003 from IOVObj import IOV
00004
00005 lockFile = ".lock"
00006
00007
00008 def timeoutManager(type,timeout=-1,fileName=".timeout"):
00009 if timeout == 0:
00010 return 1
00011 timeFormat = "%a,%Y/%m/%d,%H:%M:%S"
00012 currentTime = time.gmtime()
00013 timeoutLine = type + ' ' + time.strftime(timeFormat, currentTime) + '\n'
00014 isTimeout = False
00015 alreadyThere = False
00016 timeoutType = -1;
00017 fileExist = os.path.isfile(fileName)
00018 text = ''
00019 fields = []
00020 reset = False
00021 if timeout == -1:
00022 reset = True
00023 if fileExist:
00024 file = open(fileName)
00025 for line in file:
00026 text += line
00027 fields = line.strip('\n').split(' ')
00028 if fields[0] == type:
00029 alreadyThere = True
00030 if reset:
00031 text = text.replace(line,'')
00032 continue
00033
00034 fileTime = time.strptime(fields[1],timeFormat)
00035 myTime = time.mktime(fileTime)
00036 referenceTime = time.mktime(time.gmtime())
00037 daylight = 0
00038 if currentTime.tm_isdst == 0:
00039 daylight = 3600
00040 elapsedTime = referenceTime-myTime-daylight
00041 if elapsedTime > timeout:
00042 isTimeout = True
00043 timeoutType = 1
00044 print "Timeout! " + str(elapsedTime) + " seconds passed since the " + type + " timeout was set and you can't tolerate more than " + str(timeout) + " seconds!"
00045 else:
00046 timeoutType = 0
00047 print "Timeout of type " + type + " already exist and was generated " + str(elapsedTime) + " seconds ago at " + fields[1]
00048
00049 file.close()
00050
00051 if not fileExist or not alreadyThere and not reset:
00052 timeoutType = -1
00053 text += timeoutLine
00054
00055 if not fileExist or not alreadyThere or isTimeout or (reset and alreadyThere):
00056 if fileExist:
00057 commands.getstatusoutput("rm -rf " + fileName)
00058 file = open(fileName,'w')
00059 file.write(text)
00060 file.close()
00061
00062 return timeoutType
00063
00064
00065
00066 def setLockName(name):
00067 global lockFile
00068 lockFile = name
00069
00070
00071 def checkLock():
00072 global lockFile
00073 if os.path.isfile(lockFile):
00074 return True
00075 else:
00076 return False
00077
00078
00079 def lock():
00080 global lockFile
00081 commands.getstatusoutput( "touch " + lockFile)
00082
00083
00084 def rmLock():
00085 global lockFile
00086 if checkLock():
00087 commands.getstatusoutput( "rm " + lockFile)
00088
00089
00090 def exit(msg=""):
00091 rmLock()
00092 raise SystemExit(msg or optionstring.replace("%prog",sys.argv[0]))
00093
00094
00095 def isnan(num):
00096 fnum = float(num)
00097 return fnum != fnum
00098
00099
00100
00101
00102 USAGE = re.compile(r'(?s)\s*usage: (.*?)(\n[ \t]*\n|$)')
00103
00104
00105 def parse(docstring, arglist=None):
00106 global optionstring
00107 global tagType
00108 optionstring = docstring
00109 match = USAGE.search(optionstring)
00110 if not match: raise ParsingError("Cannot find the option string")
00111 optlines = match.group(1).splitlines()
00112 try:
00113 p = optparse.OptionParser(optlines[0])
00114 for line in optlines[1:]:
00115 opt, help=line.split(':')[:2]
00116 short,long=opt.split(',')[:2]
00117 if '=' in opt:
00118 action='store'
00119 long=long.split('=')[0]
00120 else:
00121 action='store_true'
00122 p.add_option(short.strip(),long.strip(),
00123 action = action, help = help.strip())
00124 except (IndexError,ValueError):
00125 raise ParsingError("Cannot parse the option string correctly")
00126 return p.parse_args(arglist)
00127
00128
00129 def nonzero(self):
00130 "True if options were given"
00131 for v in self.__dict__.itervalues():
00132 if v is not None: return True
00133 return False
00134
00135
00136 optparse.Values.__nonzero__ = nonzero
00137
00138
00139
00140
00141
00142 class ParsingError(Exception): pass
00143
00144
00145
00146
00147
00148 def sendEmail(mailList,error):
00149 print "Sending email to " + mailList + " with body: " + error
00150 list = mailList.split(',')
00151 for email in list:
00152 p = os.popen("mail -s \"Automatic workflow error\" " + email ,"w")
00153 p.write(error)
00154 status = p.close()
00155
00156
00157 def dirExists(dir):
00158 if dir.find("castor") != -1:
00159 lsCommand = "nsls " + dir
00160 output = commands.getstatusoutput( lsCommand )
00161 return not output[0]
00162 else:
00163 return os.path.exists(dir)
00164
00165
00166 def ls(dir,filter=""):
00167 lsCommand = ''
00168 listOfFiles = []
00169 if dir.find('castor') != -1:
00170 lsCommand = 'ns'
00171 elif not os.path.exists(dir):
00172 print "ERROR: File or directory " + dir + " doesn't exist"
00173 return listOfFiles
00174
00175 aCommand = lsCommand + 'ls '+ dir
00176
00177 if filter != "":
00178 aCommand += " | grep " + filter
00179
00180 tmpStatus = commands.getstatusoutput( aCommand )
00181 listOfFiles = tmpStatus[1].split('\n')
00182 if len(listOfFiles) == 1:
00183 if listOfFiles[0].find('No such file or directory') != -1:
00184 exit("ERROR: File or directory " + dir + " doesn't exist")
00185
00186 return listOfFiles
00187
00188
00189 def cp(fromDir,toDir,listOfFiles,overwrite=False,smallList=False):
00190 cpCommand = ''
00191 copiedFiles = []
00192 if fromDir.find('castor') != -1 or toDir.find('castor') != -1 :
00193 cpCommand = 'rf'
00194 elif fromDir.find('resilient') != -1:
00195 cpCommand = 'dc'
00196 if fromDir[len(fromDir)-1] != '/':
00197 fromDir += '/'
00198
00199 if toDir[len(toDir)-1] != '/':
00200 toDir += '/'
00201
00202 for file in listOfFiles:
00203 if os.path.isfile(toDir+file):
00204 if overwrite:
00205 print "File " + file + " already exists in destination directory. We will overwrite it."
00206 else:
00207 print "File " + file + " already exists in destination directory. We will Keep original file."
00208 if not smallList:
00209 copiedFiles.append(file)
00210 continue
00211
00212 aCommand = cpCommand + 'cp '+ fromDir + file + " " + toDir
00213 print " >> " + aCommand
00214 tmpStatus = commands.getstatusoutput( aCommand )
00215 if tmpStatus[0] == 0:
00216 copiedFiles.append(file)
00217 else:
00218 print "[cp()]\tERROR: Can't copy file " + file
00219 return copiedFiles
00220
00221
00222
00223
00224
00225
00226
00227 def pack(high,low):
00228 """pack high,low 32bit unsigned int to one unsigned 64bit long long
00229 Note:the print value of result number may appear signed, if the sign bit is used.
00230 """
00231 h=high<<32
00232 return (h|low)
00233
00234
00235 def unpack(i):
00236 """unpack 64bit unsigned long long into 2 32bit unsigned int, return tuple (high,low)
00237 """
00238 high=i>>32
00239 low=i&0xFFFFFFFF
00240 return(high,low)
00241
00242
00243 def unpackLumiid(i):
00244 """unpack 64bit lumiid to dictionary {'run','lumisection'}
00245 """
00246 j=unpack(i)
00247 return {'run':j[0],'lumisection':j[1]}
00248
00249
00250
00251
00252
00253 def cmp_list_run(a,b):
00254 if int(a.IOVfirst) < int(b.IOVfirst): return -1
00255 if int(a.IOVfirst) == int(b.IOVfirst): return 0
00256 if int(a.IOVfirst) > int(b.IOVfirst): return 1
00257
00258
00259 def cmp_list_lumi(a,b):
00260 if int(a.Run) < int(b.Run): return -1
00261 if int(a.Run) == int(b.Run):
00262 if int(a.IOVfirst) < int(b.IOVfirst): return -1
00263 if int(a.IOVfirst) == int(b.IOVfirst): return 0
00264 if int(a.IOVfirst) > int(b.IOVfirst): return 1
00265 if int(a.Run) > int(b.Run) : return 1
00266
00267
00268 def weight(x1, x1err,x2,x2err):
00269
00270 x1 = float(x1)
00271 x1err = float(x1err)
00272 x2 = float(x2)
00273 x2err = float(x2err)
00274 tmperr = 0.
00275 if x2err < 1e-6 :
00276 x2err = 1e-6
00277 if x1err < 1e-6:
00278 x1 = x2/(x2err * x2err)
00279 tmperr = 1/(x2err*x2err)
00280 else:
00281 x1 = x1/(x1err*x1err) + x2/(x2err * x2err)
00282 tmperr = 1/(x1err*x1err) + 1/(x2err*x2err)
00283 x1 = x1/tmperr
00284 x1err = 1/tmperr
00285 x1err = math.sqrt(x1err)
00286 return (str(x1), str(x1err))
00287
00288
00289 def dump( beam, file):
00290 end = "\n"
00291 file.write("Runnumber "+beam.Run+end)
00292 file.write("BeginTimeOfFit "+str(beam.IOVBeginTime)+end)
00293 file.write("EndTimeOfFit "+str(beam.IOVEndTime)+end)
00294 file.write("LumiRange "+str(beam.IOVfirst)+" - "+str(beam.IOVlast)+end)
00295 dumpValues(beam, file)
00296
00297
00298 def dumpValues( beam, file):
00299 end = "\n"
00300 file.write("Type "+str(beam.Type)+end)
00301 file.write("X0 "+str(beam.X)+end)
00302 file.write("Y0 "+str(beam.Y)+end)
00303 file.write("Z0 "+str(beam.Z)+end)
00304 file.write("sigmaZ0 "+str(beam.sigmaZ)+end)
00305 file.write("dxdz "+str(beam.dxdz)+end)
00306 file.write("dydz "+str(beam.dydz)+end)
00307 file.write("BeamWidthX "+beam.beamWidthX+end)
00308 file.write("BeamWidthY "+beam.beamWidthY+end)
00309 file.write("Cov(0,j) "+str(math.pow(float(beam.Xerr),2))+" 0 0 0 0 0 0" +end)
00310 file.write("Cov(1,j) 0 "+str(math.pow(float(beam.Yerr),2))+" 0 0 0 0 0" +end)
00311 file.write("Cov(2,j) 0 0 "+str(math.pow(float(beam.Zerr),2))+" 0 0 0 0" +end)
00312 file.write("Cov(3,j) 0 0 0 "+str(math.pow(float(beam.sigmaZerr),2))+" 0 0 0" +end)
00313 file.write("Cov(4,j) 0 0 0 0 "+str(math.pow(float(beam.dxdzerr),2))+" 0 0" +end)
00314 file.write("Cov(5,j) 0 0 0 0 0 "+str(math.pow(float(beam.dydzerr),2))+" 0" +end)
00315 file.write("Cov(6,j) 0 0 0 0 0 0 "+str(math.pow(float(beam.beamWidthXerr),2)) +end)
00316 file.write("EmittanceX 0"+end)
00317 file.write("EmittanceY 0"+end)
00318 file.write("BetaStar 0"+end)
00319
00320
00321 def delta(x,xerr,nextx,nextxerr):
00322
00323 return ( float(x) - float(nextx), math.sqrt(math.pow(float(xerr),2) + math.pow(float(nextxerr),2)) )
00324
00325 def deltaSig( x ):
00326 return math.fabs(x[0])/x[1]
00327
00328
00329 def readBeamSpotFile(fileName,listbeam=[],IOVbase="runbase", firstRun='1',lastRun='4999999999'):
00330 tmpbeam = BeamSpot()
00331 tmpbeamsize = 0
00332
00333
00334
00335 if IOVbase == "lumibase" and firstRun=='1' and lastRun=='4999999999' :
00336 firstRun = "1:1"
00337 lastRun = "4999999999:4999999999"
00338
00339 inputfiletype = 0
00340
00341
00342
00343
00344 maplist = {}
00345 hasBX = False
00346
00347 tmpfile = open(fileName)
00348 atmpline = tmpfile.readline()
00349 if atmpline.find('Runnumber') != -1:
00350 inputfiletype = 1
00351 if len(atmpline.split()) > 2:
00352 hasBX = True
00353 print " Input data has been calculated as function of BUNCH CROSSINGS."
00354 tmpfile.seek(0)
00355
00356
00357 if inputfiletype ==1:
00358
00359 tmpBX = 0
00360 for line in tmpfile:
00361
00362 if line.find('Type') != -1:
00363 tmpbeam.Type = int(line.split()[1])
00364 tmpbeamsize += 1
00365 if line.find('X0') != -1:
00366 tmpbeam.X = line.split()[1]
00367
00368 tmpbeamsize += 1
00369
00370 if line.find('Y0') != -1:
00371 tmpbeam.Y = line.split()[1]
00372
00373 tmpbeamsize += 1
00374
00375 if line.find('Z0') != -1 and line.find('sigmaZ0') == -1:
00376 tmpbeam.Z = line.split()[1]
00377
00378 tmpbeamsize += 1
00379 if line.find('sigmaZ0') !=-1:
00380 tmpbeam.sigmaZ = line.split()[1]
00381
00382 tmpbeamsize += 1
00383 if line.find('dxdz') != -1:
00384 tmpbeam.dxdz = line.split()[1]
00385
00386 tmpbeamsize += 1
00387 if line.find('dydz') != -1:
00388 tmpbeam.dydz = line.split()[1]
00389
00390 tmpbeamsize += 1
00391 if line.find('BeamWidthX') != -1:
00392 tmpbeam.beamWidthX = line.split()[1]
00393
00394 tmpbeamsize += 1
00395 if line.find('BeamWidthY') != -1:
00396 tmpbeam.beamWidthY = line.split()[1]
00397
00398 tmpbeamsize += 1
00399 if line.find('Cov(0,j)') != -1:
00400 tmpbeam.Xerr = str(math.sqrt( float( line.split()[1] ) ) )
00401 tmpbeamsize += 1
00402 if line.find('Cov(1,j)') != -1:
00403 tmpbeam.Yerr = str(math.sqrt( float( line.split()[2] ) ) )
00404 tmpbeamsize += 1
00405 if line.find('Cov(2,j)') != -1:
00406 tmpbeam.Zerr = str(math.sqrt( float( line.split()[3] ) ) )
00407 tmpbeamsize += 1
00408 if line.find('Cov(3,j)') != -1:
00409 tmpbeam.sigmaZerr = str(math.sqrt( float( line.split()[4] ) ) )
00410 tmpbeamsize += 1
00411 if line.find('Cov(4,j)') != -1:
00412 tmpbeam.dxdzerr = str(math.sqrt( float( line.split()[5] ) ) )
00413 tmpbeamsize += 1
00414 if line.find('Cov(5,j)') != -1:
00415 tmpbeam.dydzerr = str(math.sqrt( float( line.split()[6] ) ) )
00416 tmpbeamsize += 1
00417 if line.find('Cov(6,j)') != -1:
00418 tmpbeam.beamWidthXerr = str(math.sqrt( float( line.split()[7] ) ) )
00419 tmpbeam.beamWidthYerr = tmpbeam.beamWidthXerr
00420 tmpbeamsize += 1
00421 if line.find('LumiRange') != -1:
00422 if IOVbase=="lumibase":
00423 tmpbeam.IOVfirst = line.split()[1]
00424 tmpbeam.IOVlast = line.split()[3]
00425 tmpbeamsize += 1
00426 if line.find('Runnumber') != -1:
00427 tmpbeam.Run = line.split()[1]
00428 if IOVbase == "runbase":
00429 tmpbeam.IOVfirst = line.split()[1]
00430 tmpbeam.IOVlast = line.split()[1]
00431 if hasBX:
00432 tmpBX = line.split()[3]
00433 tmpbeamsize += 1
00434 if line.find('BeginTimeOfFit') != -1:
00435 tmpbeam.IOVBeginTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
00436 if IOVbase =="timebase":
00437 tmpbeam.IOVfirst = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
00438 tmpbeamsize += 1
00439 if line.find('EndTimeOfFit') != -1:
00440 tmpbeam.IOVEndTime = line.split()[1] +" "+line.split()[2] +" "+line.split()[3]
00441 if IOVbase =="timebase":
00442 tmpbeam.IOVlast = time.mktime( time.strptime(line.split()[1] + " " + line.split()[2] + " " + line.split()[3],"%Y.%m.%d %H:%M:%S %Z") )
00443 tmpbeamsize += 1
00444 if tmpbeamsize == 20:
00445 if IOVbase=="lumibase":
00446 tmprunfirst = int(firstRun.split(":")[0])
00447 tmprunlast = int(lastRun.split(":")[0])
00448 tmplumifirst = int(firstRun.split(":")[1])
00449 tmplumilast = int(lastRun.split(":")[1])
00450 acceptiov1 = acceptiov2 = False
00451
00452 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
00453 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
00454 acceptiov1 = acceptiov2 = True
00455
00456 elif int(tmpbeam.Run) == tmprunfirst:
00457 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
00458 elif int(tmpbeam.Run) == tmprunlast:
00459 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
00460 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
00461 acceptiov1 = acceptiov2 = True
00462
00463 if acceptiov1 and acceptiov2:
00464 if tmpbeam.Type != 2:
00465 print "invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00466 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00467 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00468 elif hasBX:
00469 if maplist.has_key(tmpBX) == False:
00470 maplist[tmpBX] = [tmpbeam]
00471 else:
00472 maplist[tmpBX].append(tmpbeam)
00473 else:
00474 listbeam.append(tmpbeam)
00475
00476 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
00477 if tmpbeam.Type != 2:
00478 print "invalid fit, skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00479 elif isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00480 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00481 else:
00482 listbeam.append(tmpbeam)
00483
00484 tmpbeamsize = 0
00485 tmpbeam = BeamSpot()
00486 tmpBX = 0
00487 else:
00488
00489 for line in tmpfile:
00490
00491 if line.find('X0') != -1:
00492 tmpbeam.X = line.split()[2]
00493 tmpbeam.Xerr = line.split()[4]
00494 tmpbeamsize += 1
00495
00496 if line.find('Y0') != -1:
00497 tmpbeam.Y = line.split()[2]
00498 tmpbeam.Yerr = line.split()[4]
00499 tmpbeamsize += 1
00500
00501 if line.find('Z0') != -1 and line.find('Sigma Z0') == -1:
00502 tmpbeam.Z = line.split()[2]
00503 tmpbeam.Zerr = line.split()[4]
00504 tmpbeamsize += 1
00505
00506 if line.find('Sigma Z0') !=-1:
00507 tmpbeam.sigmaZ = line.split()[3]
00508 tmpbeam.sigmaZerr = line.split()[5]
00509 tmpbeamsize += 1
00510 if line.find('dxdz') != -1:
00511 tmpbeam.dxdz = line.split()[2]
00512 tmpbeam.dxdzerr = line.split()[4]
00513 tmpbeamsize += 1
00514 if line.find('dydz') != -1:
00515 tmpbeam.dydz = line.split()[2]
00516 tmpbeam.dydzerr = line.split()[4]
00517 tmpbeamsize += 1
00518 if line.find('Beam Width X') != -1:
00519 tmpbeam.beamWidthX = line.split()[4]
00520 tmpbeam.beamWidthXerr = line.split()[6]
00521 tmpbeamsize += 1
00522 if line.find('Beam Width Y') != -1:
00523 tmpbeam.beamWidthY = line.split()[4]
00524 tmpbeam.beamWidthYerr = line.split()[6]
00525 tmpbeamsize += 1
00526
00527 if line.find('for runs') != -1:
00528
00529 tmpbeam.Run = line.split()[2]
00530 if IOVbase == "runbase":
00531 tmpbeam.IOVfirst = line.split()[2]
00532 tmpbeam.IOVlast = line.split()[4]
00533 tmpbeamsize += 1
00534 if line.find('LumiSection') != -1:
00535 if IOVbase=="lumibase":
00536 tmpbeam.IOVfirst = line.split()[10]
00537 tmpbeam.IOVlast = line.split()[10]
00538 tmpbeamsize += 1
00539 if tmpbeamsize == 10:
00540
00541 if IOVbase=="lumibase":
00542 tmprunfirst = int(firstRun.split(":")[0])
00543 tmprunlast = int(lastRun.split(":")[0])
00544 tmplumifirst = int(firstRun.split(":")[1])
00545 tmplumilast = int(lastRun.split(":")[1])
00546 acceptiov1 = acceptiov2 = False
00547
00548 if tmprunfirst == tmprunlast and int(tmpbeam.Run)==tmprunfirst:
00549 if int(tmpbeam.IOVfirst) >= tmplumifirst and int(tmpbeam.IOVlast)<=tmplumilast:
00550 acceptiov1 = acceptiov2 = True
00551
00552 elif int(tmpbeam.Run) == tmprunfirst:
00553 if int(tmpbeam.IOVfirst) >= tmplumifirst: acceptiov1 = True
00554 elif int(tmpbeam.Run) == tmprunlast:
00555 if int(tmpbeam.IOVlast) <= tmplumilast: acceptiov2 = True
00556 elif tmprunfirst <= int(tmpbeam.Run) and tmprunlast >= int(tmpbeam.Run):
00557 acceptiov1 = acceptiov2 = True
00558
00559 if acceptiov1 and acceptiov2:
00560 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00561 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00562 elif hasBX:
00563 if maplist.has_key(tmpBX) == False:
00564 maplist[tmpBX] = [tmpbeam]
00565 else:
00566 maplist[tmpBX].append(tmpbeam)
00567 else:
00568 listbeam.append(tmpbeam)
00569
00570 elif int(tmpbeam.IOVfirst) >= int(firstRun) and int(tmpbeam.IOVlast) <= int(lastRun):
00571 if isnan(tmpbeam.Z) or isnan(tmpbeam.Zerr) or isnan(tmpbeam.sigmaZerr) or isnan(tmpbeam.beamWidthXerr) or isnan(tmpbeam.beamWidthYerr):
00572 print "invalid fit, NaN values!! skip Run "+str(tmpbeam.Run)+" IOV: "+str(tmpbeam.IOVfirst) + " to "+ str(tmpbeam.IOVlast)
00573 else:
00574 listbeam.append(tmpbeam)
00575
00576 tmpbeamsize = 0
00577 tmpbeam = BeamSpot()
00578 tmpBX = 0
00579
00580 tmpfile.close()
00581 print " got total number of IOVs = " + str(len(listbeam)) + " from file " + fileName
00582
00583 if hasBX:
00584 return maplist
00585 else:
00586 return listbeam
00587
00588
00589
00590 def sortAndCleanBeamList(listbeam=[],IOVbase="lumibase"):
00591
00592 if IOVbase == "lumibase":
00593 listbeam.sort( cmp = cmp_list_lumi )
00594 else:
00595 listbeam.sort( cmp = cmp_list_run )
00596
00597
00598 tmpremovelist = []
00599 for ii in range(0,len(listbeam)):
00600 ibeam = listbeam[ii]
00601 datax = ibeam.IOVfirst
00602
00603 if datax == '0' and IOVbase =="runbase":
00604 print " iov = 0? skip this IOV = "+ str(ibeam.IOVfirst) + " to " + str(ibeam.IOVlast)
00605 tmpremovelist.append(ibeam)
00606
00607 if ii < len(listbeam) -1:
00608
00609 if IOVbase =="lumibase":
00610 if ibeam.Run == listbeam[ii+1].Run and ibeam.IOVfirst == listbeam[ii+1].IOVfirst:
00611 print " duplicate IOV = "+datax+", keep only last duplicate entry"
00612 tmpremovelist.append(ibeam)
00613 elif datax == listbeam[ii+1].IOVfirst:
00614 print " duplicate IOV = "+datax+", keep only last duplicate entry"
00615 tmpremovelist.append(ibeam)
00616
00617 for itmp in tmpremovelist:
00618 listbeam.remove(itmp)
00619
00620
00621
00622 def createWeightedPayloads(fileName,listbeam=[],weighted=True):
00623 newlistbeam = []
00624 tmpbeam = BeamSpot()
00625 docreate = True
00626 countlumi = 0
00627 tmprun = ""
00628 maxNlumis = 60
00629 if weighted:
00630 maxNlumis = 999999999
00631 for ii in range(0,len(listbeam)):
00632 ibeam = listbeam[ii]
00633 inextbeam = BeamSpot()
00634 iNNbeam = BeamSpot()
00635 if docreate:
00636 tmpbeam.IOVfirst = ibeam.IOVfirst
00637 tmpbeam.IOVBeginTime = ibeam.IOVBeginTime
00638 tmpbeam.Run = ibeam.Run
00639 tmpbeam.Type = 2
00640 docheck = False
00641 docreate = False
00642
00643
00644
00645 if ii < len(listbeam) - 1:
00646 inextbeam = listbeam[ii+1]
00647 docheck = True
00648 if ii < len(listbeam) -2:
00649 iNNbeam = listbeam[ii+2]
00650 else:
00651 print "close payload because end of data has been reached. Run "+ibeam.Run
00652 docreate = True
00653
00654 if ibeam.Run != inextbeam.Run:
00655 print "close payload because end of run "+ibeam.Run
00656 docreate = True
00657
00658 if countlumi == maxNlumis -1:
00659 print "close payload because maximum lumi sections accumulated within run "+ibeam.Run
00660 docreate = True
00661 countlumi = 0
00662
00663 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
00664 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
00665 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
00666 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
00667 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
00668 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
00669
00670 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
00671 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
00672
00673 if weighted:
00674 docheck = False
00675
00676
00677 if docheck:
00678
00679
00680 min_limit = 0.0025
00681
00682
00683 limit = float(ibeam.beamWidthX)/2.
00684 if limit < min_limit: limit = min_limit
00685
00686
00687 adelta1 = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr)
00688 adelta2 = (0.,1.e9)
00689 adelta1dxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
00690 adelta2dxdz = (0.,1.e9)
00691 adelta1dydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
00692 adelta2dydz = (0.,1.e9)
00693 adelta1widthx = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
00694 adelta2widthx = (0.,1.e9)
00695 adelta1widthy = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
00696 adelta2widthy = (0.,1.e9)
00697 adelta1z0 = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr)
00698 adelta1sigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr)
00699
00700 if iNNbeam.Type != -1:
00701 adelta2 = delta(inextbeam.X, inextbeam.Xerr, iNNbeam.X, iNNbeam.Xerr)
00702 adelta2dxdz = delta(inextbeam.dxdz, inextbeam.dxdzerr, iNNbeam.dxdz, iNNbeam.dxdzerr)
00703 adelta2dydz = delta(inextbeam.dydz, inextbeam.dydzerr, iNNbeam.dydz, iNNbeam.dydzerr)
00704 adelta2widthx = delta(inextbeam.beamWidthX, inextbeam.beamWidthXerr, iNNbeam.beamWidthX, iNNbeam.beamWidthXerr)
00705 adelta2widthy = delta(inextbeam.beamWidthY, inextbeam.beamWidthYerr, iNNbeam.beamWidthY, iNNbeam.beamWidthYerr)
00706
00707 deltaX = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
00708 if ii < len(listbeam) -2:
00709 if deltaX==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
00710
00711 deltaX = True
00712 elif deltaX==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
00713 deltaX = False
00714
00715
00716
00717
00718
00719 adelta1 = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr)
00720 adelta2 = (0.,1.e9)
00721 if iNNbeam.Type != -1:
00722 adelta2 = delta(inextbeam.Y, inextbeam.Yerr, iNNbeam.Y, iNNbeam.Yerr)
00723
00724 deltaY = deltaSig(adelta1) > 3.5 and adelta1[0] >= limit
00725 if ii < len(listbeam) -2:
00726 if deltaY==False and adelta1[0]*adelta2[0] > 0. and math.fabs(adelta1[0]+adelta2[0]) >= limit:
00727 deltaY = True
00728 elif deltaY==True and adelta1[0]*adelta2[0]<=0 and adelta2[0] != 0 and math.fabs(adelta1[0]/adelta2[0]) > 0.33 and math.fabs(adelta1[0]/adelta2[0]) < 3:
00729 deltaY = False
00730
00731
00732 limit = float(ibeam.sigmaZ)/2.
00733 deltaZ = deltaSig(adelta1z0) > 3.5 and math.fabs(adelta1z0[0]) >= limit
00734
00735 deltasigmaZ = deltaSig(adelta1sigmaZ) > 5.0
00736
00737
00738 adelta = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr)
00739 deltadxdz = deltaSig(adelta) > 5.0
00740 if deltadxdz and adelta1dxdz[0]*adelta2dxdz[0]<=0 and adelta2dxdz[0] != 0 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) > 0.33 and math.fabs(adelta1dxdz[0]/adelta2dxdz[0]) < 3:
00741 deltadxdz = False
00742
00743 adelta = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr)
00744 deltadydz = deltaSig(adelta) > 5.0
00745 if deltadydz and adelta1dydz[0]*adelta2dydz[0]<=0 and adelta2dydz[0] != 0 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) > 0.33 and math.fabs(adelta1dydz[0]/adelta2dydz[0]) < 3:
00746 deltadydz = False
00747
00748 adelta = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr)
00749 deltawidthX = deltaSig(adelta) > 5
00750 if deltawidthX and adelta1widthx[0]*adelta2widthx[0]<=0 and adelta2widthx[0] != 0 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) > 0.33 and math.fabs(adelta1widthx[0]/adelta2widthx[0]) < 3:
00751 deltawidthX = False
00752
00753 adelta = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr)
00754 deltawidthY = deltaSig(adelta) > 5
00755 if deltawidthY and adelta1widthy[0]*adelta2widthy[0]<=0 and adelta2widthy[0] != 0 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) > 0.33 and math.fabs(adelta1widthy[0]/adelta2widthy[0]) < 3:
00756 deltawidthY = False
00757
00758
00759
00760
00761
00762
00763
00764
00765
00766
00767
00768
00769 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
00770 docreate = True
00771
00772
00773
00774
00775
00776
00777 print "close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY)
00778 if docreate:
00779
00780 tmpbeam.IOVlast = ibeam.IOVlast
00781 tmpbeam.IOVEndTime = ibeam.IOVEndTime
00782 print " Run: "+tmpbeam.Run +" Lumi1: "+str(tmpbeam.IOVfirst) + " Lumi2: "+str(tmpbeam.IOVlast)
00783 newlistbeam.append(tmpbeam)
00784 tmpbeam = BeamSpot()
00785 countlumi = 0
00786 tmprun = ibeam.Run
00787 countlumi += 1
00788
00789 payloadfile = open(fileName,"w")
00790 for iload in newlistbeam:
00791 dump( iload, payloadfile )
00792 payloadfile.close()
00793 return newlistbeam
00794
00795 def createWeightedPayloadsNew(fileName,listbeam=[],weighted=True):
00796 newlistbeam = []
00797 docreate = False
00798 docheck = False
00799 lastPayload = listbeam[0]
00800
00801 firstToUse = 0
00802 lastToUse = 0
00803 for ii in range(0,len(listbeam)):
00804 docreate = False
00805 if docheck:
00806 deltaX = delta(ibeam.X, ibeam.Xerr, inextbeam.X, inextbeam.Xerr) > 1.5
00807 deltaY = delta(ibeam.Y, ibeam.Yerr, inextbeam.Y, inextbeam.Yerr) > 1.5
00808 deltaZ = delta(ibeam.Z, ibeam.Zerr, inextbeam.Z, inextbeam.Zerr) > 2.5
00809
00810 deltasigmaZ = delta(ibeam.sigmaZ, ibeam.sigmaZerr, inextbeam.sigmaZ, inextbeam.sigmaZerr) > 2.5
00811 deltadxdz = delta(ibeam.dxdz, ibeam.dxdzerr, inextbeam.dxdz, inextbeam.dxdzerr) > 2.5
00812 deltadydz = delta(ibeam.dydz, ibeam.dydzerr, inextbeam.dydz, inextbeam.dydzerr) > 2.5
00813
00814 deltawidthX = delta(ibeam.beamWidthX, ibeam.beamWidthXerr, inextbeam.beamWidthX, inextbeam.beamWidthXerr) > 3
00815 deltawidthY = delta(ibeam.beamWidthY, ibeam.beamWidthYerr, inextbeam.beamWidthY, inextbeam.beamWidthYerr) > 3
00816
00817
00818
00819
00820
00821
00822
00823
00824
00825
00826
00827
00828
00829 if deltaX or deltaY or deltaZ or deltasigmaZ or deltadxdz or deltadydz or deltawidthX or deltawidthY:
00830 if ii != 0:
00831 docreate = True
00832 lastToUse = ii-1
00833
00834
00835
00836
00837
00838
00839 print "close payload because of movement in X= "+str(deltaX)+", Y= "+str(deltaY) + ", Z= "+str(deltaZ)+", sigmaZ= "+str(deltasigmaZ)+", dxdz= "+str(deltadxdz)+", dydz= "+str(deltadydz)+", widthX= "+str(deltawidthX)+", widthY= "+str(deltawidthY)
00840
00841
00842 if ii >= len(listbeam) - 1 or listbeam[ii].Run != listbeam[ii+1].Run :
00843 print "close payload because end of run has been reached. Run " + listbeam[ii].Run
00844 docreate = True
00845 lastToUse = ii
00846
00847
00848
00849
00850
00851
00852
00853 if docreate:
00854 tmpbeam = BeamSpot()
00855 for ibeam in listbeam[firstToUse:lastToUse]:
00856 (tmpbeam.X, tmpbeam.Xerr) = weight(tmpbeam.X, tmpbeam.Xerr, ibeam.X, ibeam.Xerr)
00857 (tmpbeam.Y, tmpbeam.Yerr) = weight(tmpbeam.Y, tmpbeam.Yerr, ibeam.Y, ibeam.Yerr)
00858 (tmpbeam.Z, tmpbeam.Zerr) = weight(tmpbeam.Z, tmpbeam.Zerr, ibeam.Z, ibeam.Zerr)
00859 (tmpbeam.sigmaZ, tmpbeam.sigmaZerr) = weight(tmpbeam.sigmaZ, tmpbeam.sigmaZerr, ibeam.sigmaZ, ibeam.sigmaZerr)
00860 (tmpbeam.dxdz, tmpbeam.dxdzerr) = weight(tmpbeam.dxdz, tmpbeam.dxdzerr, ibeam.dxdz, ibeam.dxdzerr)
00861 (tmpbeam.dydz, tmpbeam.dydzerr) = weight(tmpbeam.dydz, tmpbeam.dydzerr, ibeam.dydz, ibeam.dydzerr)
00862
00863 (tmpbeam.beamWidthX, tmpbeam.beamWidthXerr) = weight(tmpbeam.beamWidthX, tmpbeam.beamWidthXerr, ibeam.beamWidthX, ibeam.beamWidthXerr)
00864 (tmpbeam.beamWidthY, tmpbeam.beamWidthYerr) = weight(tmpbeam.beamWidthY, tmpbeam.beamWidthYerr, ibeam.beamWidthY, ibeam.beamWidthYerr)
00865 tmpbeam.IOVfirst = listbeam[firstToUse].IOVfirst
00866 tmpbeam.IOVBeginTime = listbeam[firstToUse].IOVBeginTime
00867 tmpbeam.Run = listbeam[firstToUse].Run
00868 tmpbeam.Type = 2
00869 tmpbeam.IOVlast = listbeam[lastToUse].IOVlast
00870 tmpbeam.IOVEndTime = listbeam[lastToUse].IOVEndTime
00871 newlistbeam.append(tmpbeam)
00872 firstToUse = lastToUse+1
00873 print "Run: " + tmpbeam.Run + " Lumi1: " + str(tmpbeam.IOVfirst) + " Lumi2: " + str(tmpbeam.IOVlast)
00874
00875 payloadfile = open(fileName,"w")
00876 for iload in newlistbeam:
00877 dump( iload, payloadfile )
00878 payloadfile.close()
00879
00880
00881 def writeSqliteFile(sqliteFileName,tagName,timeType,beamSpotFile,sqliteTemplateFile,tmpDir="/tmp/"):
00882 writeDBOut = tmpDir + "write2DB_" + tagName + ".py"
00883 wFile = open(sqliteTemplateFile)
00884 wNewFile = open(writeDBOut,'w')
00885
00886 writeDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
00887 ('TAGNAME',tagName),
00888 ('TIMETYPE',timeType),
00889 ('BEAMSPOTFILE',beamSpotFile)]
00890
00891 for line in wFile:
00892 for itag in writeDBTags:
00893 line = line.replace(itag[0],itag[1])
00894 wNewFile.write(line)
00895
00896 wNewFile.close()
00897 print "writing sqlite file ..."
00898 status_wDB = commands.getstatusoutput('cmsRun '+ writeDBOut)
00899 print status_wDB[1]
00900
00901 os.system("rm -f " + writeDBOut)
00902 return not status_wDB[0]
00903
00904
00905 def readSqliteFile(sqliteFileName,tagName,sqliteTemplateFile,tmpDir="/tmp/"):
00906 readDBOut = tmpDir + "readDB_" + tagName + ".py"
00907
00908 rFile = open(sqliteTemplateFile)
00909 rNewFile = open(readDBOut,'w')
00910
00911 readDBTags = [('SQLITEFILE','sqlite_file:' + sqliteFileName),
00912 ('TAGNAME',tagName)]
00913
00914 for line in rFile:
00915 for itag in readDBTags:
00916 line = line.replace(itag[0],itag[1])
00917 rNewFile.write(line)
00918
00919 rNewFile.close()
00920 status_rDB = commands.getstatusoutput('cmsRun '+ readDBOut)
00921
00922 outtext = status_rDB[1]
00923 print outtext
00924 os.system("rm -f " + readDBOut)
00925 return not status_rDB[0]
00926
00927
00928 def appendSqliteFile(combinedSqliteFileName, sqliteFileName, tagName, IOVSince, IOVTill ,tmpDir="/tmp/"):
00929 aCommand = "cmscond_export_iov -d sqlite_file:" + tmpDir + combinedSqliteFileName + " -s sqlite_file:" + sqliteFileName + " -i " + tagName + " -t " + tagName + " -l sqlite_file:" + tmpDir + "log.db" + " -b " + IOVSince + " -e " + IOVTill
00930 print aCommand
00931 std = commands.getstatusoutput(aCommand)
00932 print std[1]
00933 return not std[0]
00934
00935
00936 def uploadSqliteFile(sqliteFileDirName, sqliteFileName, dropbox="/DropBox"):
00937
00938 acmd = "chmod a+w " + sqliteFileDirName + sqliteFileName + ".txt"
00939 outcmd = commands.getstatusoutput(acmd)
00940 print acmd
00941
00942 if outcmd[0]:
00943 print "Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".txt"
00944 return False
00945
00946 acmd = "cp " + sqliteFileDirName + sqliteFileName + ".db " + sqliteFileDirName + sqliteFileName + ".txt ."
00947 print acmd
00948 outcmd = commands.getstatusoutput(acmd)
00949 print outcmd[1]
00950 if outcmd[0]:
00951 print "Couldn't cd to " + sqliteFileDirName
00952 return False
00953
00954 acmd = "tar -cvjf " + sqliteFileName + ".tar.bz2 " + sqliteFileName + ".db " + sqliteFileName + ".txt"
00955 print acmd
00956 outcmd = commands.getstatusoutput(acmd)
00957 print outcmd[1]
00958 if outcmd[0]:
00959 print "Couldn't zip the files!"
00960 return False
00961
00962 acmd = "chmod a+w " + sqliteFileName + ".tar.bz2"
00963 outcmd = commands.getstatusoutput(acmd)
00964 print acmd
00965
00966 if outcmd[0]:
00967 print "Can't change permission to file: " + sqliteFileDirName + sqliteFileName + ".tar.bz2"
00968 return False
00969
00970 acmd = "scp -p " + sqliteFileName + ".tar.bz2" + " webcondvm.cern.ch:" + dropbox
00971 print acmd
00972 outcmd = commands.getstatusoutput(acmd)
00973 print outcmd[1]
00974 if outcmd[0]:
00975 print "Couldn't scp the files to DropBox!"
00976 return False
00977
00978
00979 acmd = "mv " + sqliteFileName + ".tar.bz2 " + sqliteFileDirName
00980 print acmd
00981 outcmd = commands.getstatusoutput(acmd)
00982 print outcmd[1]
00983 if outcmd[0]:
00984 print "Couldn't mv the file to " + sqliteFileDirName
00985 return False
00986
00987 acmd = "rm " + sqliteFileName + ".db " + sqliteFileName + ".txt"
00988 print acmd
00989 outcmd = commands.getstatusoutput(acmd)
00990 print outcmd[1]
00991 if outcmd[0]:
00992 print "Couldn't rm the db and txt files"
00993 return False
00994
00995
00996
00997
00998
00999
01000
01001
01002
01003
01004
01005
01006
01007
01008
01009 return False
01010
01011
01012
01013
01014
01015
01016
01017
01018
01019 return True
01020
1.7.3