BeamSpotWorkflow.py

Go to the documentation of this file.

#!/usr/bin/env python
#____________________________________________________________
#
#  BeamSpotWorkflow
#
# A very complicate way to automatize the beam spot workflow
#
# Francisco Yumiceva, Lorenzo Uplegger
# yumiceva@fnal.gov, uplegger@fnal.gov
#
# Fermilab, 2010
#
#____________________________________________________________

"""
   BeamSpotWorkflow.py

   A very complicate script to upload the results into the DB

   usage: %prog -d <data file/directory> -t <tag name>
   -c, --cfg = CFGFILE : Use a different configuration file than the default
   -l, --lock = LOCK   : Create a lock file to have just one script running 
   -o, --overwrite     : Overwrite results files when copying.
   -T, --Test          : Upload files to Test dropbox for data validation.   
   -u, --upload        : Upload files to offline drop box via scp.
   -z, --zlarge        : Enlarge sigmaZ to 10 +/- 0.005 cm.

   Francisco Yumiceva (yumiceva@fnal.gov)
   Lorenzo Uplegger   (send an email to Francisco)
   Fermilab 2010
   
"""


import sys,os
import commands, re, time
import datetime
import ConfigParser
import xmlrpclib
from BeamSpotObj import BeamSpot
from IOVObj import IOV
from CommonMethods import *

try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
    import json
except:
    try:
        import simplejson as json
    except:
        error = "Please set a crab environment in order to get the proper JSON lib"
        exit(error)

#####################################################################################
# General functions
#####################################################################################
def getLastUploadedIOV(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
    #return 582088327592295
    listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName 
    dbError = commands.getstatusoutput( listIOVCommand )
    if dbError[0] != 0 :
        if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
            print "Creating a new tag because I got the following error contacting the DB"
            print dbError[1]
            return 1
            #return 133928
        else:
            exit("ERROR: Can\'t connect to db because:\n" + dbError[1])


    aCommand = listIOVCommand + " | grep DB= | tail -1 | awk \'{print $1}\'"
    output = commands.getstatusoutput( aCommand )
    
    #WARNING when we pass to lumi IOV this should be long long
    if output[1] == '':
      exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
      
    return long(output[1])

########################################################################
def getListOfFilesToProcess(dataSet,lastRun=-1):
    queryCommand = "dbs --search --query \"find file where dataset=" + dataSet
    if lastRun != -1:
        queryCommand = queryCommand + " and run > " + str(lastRun)
    queryCommand = queryCommand + "\" | grep .root"    
#    print " >> " + queryCommand
    output = commands.getstatusoutput( queryCommand )
    return output[1].split('\n')

########################################################################
def getNumberOfFilesToProcessForRun(dataSet,run):
    queryCommand = "dbs --search --query \"find file where dataset=" + dataSet + " and run = " + str(run) + "\" | grep .root"
    #print " >> " + queryCommand
    output = commands.getstatusoutput( queryCommand )
    if output[0] != 0:
        return 0
    else:
        return len(output[1].split('\n'))

########################################################################
def getListOfRunsAndLumiFromDBS(dataSet,lastRun=-1):
    datasetList = dataSet.split(',')
    outputList = []
    for data in datasetList:
        queryCommand = "dbs --search --query \"find run,lumi where dataset=" + data
        if lastRun != -1:
            queryCommand = queryCommand + " and run > " + str(lastRun)
        queryCommand += "\""
        print " >> " + queryCommand
        output = []
        for i in range(0,3):
        output = commands.getstatusoutput( queryCommand )
            if output[0] == 0 and not (output[1].find("ERROR") != -1 or output[1].find("Error") != -1) :
                break
        if output[0] != 0:
            exit("ERROR: I can't contact DBS for the following reason:\n" + output[1])
        #print output[1]
        tmpList = output[1].split('\n')
        for file in tmpList:
            outputList.append(file)
    runsAndLumis = {}
    for out in outputList:
        regExp = re.search('(\d+)\s+(\d+)',out)
        if regExp:
            run  = long(regExp.group(1))
            lumi = long(regExp.group(2))
            if not run in runsAndLumis:
                runsAndLumis[run] = []
            runsAndLumis[run].append(lumi)

#    print runsAndLumis
#    exit("ok")
    return runsAndLumis

#####################################################################################
def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
    file = open(fileName);
    jsonFile = file.read();
    file.close()
    jsonList=json.loads(jsonFile);

    selected_dcs = {};
    for element in jsonList:
        selected_dcs[long(element)]=jsonList[element]
    return selected_dcs

########################################################################
def getListOfRunsAndLumiFromRR(firstRun=-1):
    RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
    #RunReg  = "http://localhost:40010/runregistry"
    #Dataset=%Online%
    Group   = "Collisions10"

    # get handler to RR XML-RPC server
    FULLADDRESS=RunReg + "/xmlrpc"
    #print "RunRegistry from: ",FULLADDRESS
    server = xmlrpclib.ServerProxy(FULLADDRESS)
    #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
    sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) 
    #sel_dcstable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"

    maxAttempts = 3;
    tries = 0;
    while tries<maxAttempts:
        try:
            run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
            #dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
            break
        except:
            print "Something wrong in accessing runregistry, retrying in 2s....", tries, "/", maxAttempts
            tries += 1
            time.sleep(2)
        if tries==maxAttempts:
            error = "Run registry unaccessible.....exiting now"
            return {};
    

    listOfRuns=[]
    for line in run_data.split("\n"):
        run=line.split(',')[0]
        if run.isdigit():
            listOfRuns.append(run)

    
    firstRun = listOfRuns[len(listOfRuns)-1];
    lastRun  = listOfRuns[0];
    sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"

    tries = 0;
    while tries<maxAttempts:
        try:
            #run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
            dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
            break
        except:
            print "I was able to get the list of runs and now I am trying to access the detector status, retrying in 2s....", tries, "/", maxAttempts
            tries += 1
            time.sleep(2)
        if tries==maxAttempts:
            error = "Run registry unaccessible.....exiting now"
            return {};

    selected_dcs={}
    jsonList=json.loads(dcs_data)

    #for element in jsonList:
    for element in listOfRuns:
        #if element in listOfRuns:
        if element in jsonList:
            selected_dcs[long(element)]=jsonList[element]
        else:
            print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!" 
            selected_dcs[long(element)]= [[]] 
    #print selected_dcs        
    return selected_dcs

########################################################################
def getLastClosedRun(DBSListOfFiles):
    runs = []
    for file in DBSListOfFiles:
        runNumber = getRunNumberFromDBSName(file)
        if runs.count(runNumber) == 0: 
            runs.append(runNumber)

    if len(runs) <= 1: #No closed run
        return -1
    else:
        runs.sort()
        return long(runs[len(runs)-2])
    
########################################################################
def getRunNumberFromFileName(fileName):
#    regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
    regExp = re.search('(\D+)_(\d+)_(\d+)_',fileName)
    if not regExp:
        return -1
    return long(regExp.group(3))

########################################################################
def getRunNumberFromDBSName(fileName):
    regExp = re.search('(\D+)/(\d+)/(\d+)/(\d+)/(\D+)',fileName)
    if not regExp:
        return -1
    return long(regExp.group(3)+regExp.group(4))
    
########################################################################
def getNewRunList(fromDir,lastUploadedIOV):
    newRunList = []
    listOfFiles = ls(fromDir,".txt")
    runFileMap = {}
    for fileName in listOfFiles:
        runNumber = getRunNumberFromFileName(fileName) 
        if runNumber > lastUploadedIOV:
            newRunList.append(fileName)
    return newRunList        

########################################################################
def selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,newRunList,runListDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout):
    runsAndLumisProcessed = {}
    runsAndFiles = {}
    for fileName in newRunList:
        file = open(runListDir+fileName)
        for line in file:
            if line.find("Runnumber") != -1:
                run = long(line.replace('\n','').split(' ')[1])
            elif line.find("LumiRange") != -1:
                lumiLine = line.replace('\n','').split(' ')
                begLumi = long(lumiLine[1])
                endLumi = long(lumiLine[3])
                if begLumi != endLumi:
                    error = "The lumi range is greater than 1 for run " + str(run) + " " + line + " in file: " + runListDir + fileName
                    exit(error)
                else:
                    if not run in runsAndLumisProcessed:
                        runsAndLumisProcessed[run] = []
                    if begLumi in runsAndLumisProcessed[run]:
                        print "Lumi " + str(begLumi) + " in event " + str(run) + " already exist. This MUST not happen but right now I will ignore this lumi!"
                    else:    
                        runsAndLumisProcessed[run].append(begLumi)
        if not run in runsAndFiles:
            runsAndFiles[run] = []
        runsAndFiles[run].append(fileName)    
        file.close()

    rrKeys = sorted(listOfRunsAndLumiFromRR.keys())
    dbsKeys = listOfRunsAndLumiFromDBS.keys()
    dbsKeys.sort()
    #I remove the last entry from DBS since I am not sure it is an already closed run!
    lastUnclosedRun = dbsKeys.pop()
    #print "Last unclosed run: " + str(lastUnclosedRun)
    procKeys = runsAndLumisProcessed.keys()
    procKeys.sort()
    #print "Run Registry:"    
    #print rrKeys
    #print "DBS:"    
    #print dbsKeys
    #print "List:"    
    #print procKeys
    #print lastUnclosedRun
    filesToProcess = []
    for run in rrKeys:
        RRList = []
        for lumiRange in listOfRunsAndLumiFromRR[run]:
            if lumiRange != []: 
                for l in range(lumiRange[0],lumiRange[1]+1):
                    RRList.append(long(l))
        if run in procKeys and run < lastUnclosedRun:
            #print "run " + str(run) + " is in procKeys"
            if not run in dbsKeys and run != lastUnclosedRun:
                error = "Impossible but run " + str(run) + " has been processed and it is also in the run registry but it is not in DBS!" 
                exit(error)
            print "Working on run " + str(run)
            nFiles = 0
            for data in dataSet.split(','):
                nFiles = getNumberOfFilesToProcessForRun(data,run)
                if nFiles != 0:
                    break
            if len(runsAndFiles[run]) < nFiles:
                print "I haven't processed all files yet : " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " for run: " + str(run) 
                if nFiles - len(runsAndFiles[run]) <= missingFilesTolerance:
                    timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run)) # resetting this timeout
                    timeoutType = timeoutManager("DBS_MISMATCH_Run"+str(run),missingLumisTimeout)
                    if timeoutType == 1:
                        print "WARNING: I previously set a timeout that expired...I'll continue with the script even if I didn't process all the lumis!"
                    else:
                        if timeoutType == -1:
                            print "WARNING: Setting the DBS_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!"
                        else:
                            print "WARNING: Timeout DBS_MISMATCH_Run" + str(run) + " is in progress."
                        return filesToProcess
                else:
                    timeoutType = timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run),missingLumisTimeout)
                    if timeoutType == 1:
                        error = "ERROR: I previously set a timeout that expired...I can't continue with the script because there are too many (" + str(nFiles - len(runsAndFiles[run])) + " files missing) and for too long " + str(missingLumisTimeout/3600) + " hours! I will process anyway the runs before this one (" + str(run) + ")"
                        sendEmail(mailList,error)
                        return filesToProcess
                        #exit(error)
                    else:
                        if timeoutType == -1:
                            print "WARNING: Setting the DBS_VERY_BIG_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!"
                        else:
                            print "WARNING: Timeout DBS_VERY_BIG_MISMATCH_Run" + str(run) + " is in progress."
                        return filesToProcess
                    
            else:
                timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run))
                timeoutManager("DBS_MISMATCH_Run"+str(run))
                print "I have processed " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " files that are in DBS. So I should have all the lumis!" 
            errors          = []
            badProcessed    = []
            badDBSProcessed = []
            badDBS          = []
            badRRProcessed  = []
            badRR           = []
            #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
            badDBSProcessed,badDBS = compareLumiLists(runsAndLumisProcessed[run],listOfRunsAndLumiFromDBS[run],errors)
            for i in range(0,len(errors)):
                errors[i] = errors[i].replace("listA","the processed lumis")
                errors[i] = errors[i].replace("listB","DBS")
            #print errors
            #print badProcessed
            #print badDBS
            #exit("ciao")
            if len(badDBS) != 0:
                print "This is weird because I processed more lumis than the ones that are in DBS!"
            if len(badDBSProcessed) != 0 and run in rrKeys:
                lastError = len(errors)
                #print RRList            
                #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
                badRRProcessed,badRR = compareLumiLists(runsAndLumisProcessed[run],RRList,errors)
                for i in range(0,len(errors)):
                    errors[i] = errors[i].replace("listA","the processed lumis")
                    errors[i] = errors[i].replace("listB","Run Registry")
                #print errors
                #print badProcessed
                #print badRunRegistry
                    
                if len(badRRProcessed) != 0:    
                    print "I have not processed some of the lumis that are in the run registry for run: " + str(run)
                    for lumi in badDBSProcessed:
                        if lumi in badRRProcessed:
                            badProcessed.append(lumi)
                    lenA = len(badProcessed)
                    lenB = len(RRList)
                    if 100.*lenA/lenB <= dbsTolerancePercent:
                        print "WARNING: I didn't process " + str(100.*lenA/lenB) + "% of the lumis but I am within the " + str(dbsTolerancePercent) + "% set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis"
                        #print errors
                        badProcessed = []
                    elif lenA <= dbsTolerance:
                        print "WARNING: I didn't process " + str(lenA) + " lumis but I am within the " + str(dbsTolerance) + " lumis set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis"
                        #print errors
                        badProcessed = []
                    else:    
                        error = "ERROR: For run " + str(run) + " I didn't process " + str(100.*lenA/lenB) + "% of the lumis and I am not within the " + str(dbsTolerancePercent) + "% set in the configuration. The number of lumis that I didn't process (" + str(lenA) + " out of " + str(lenB) + ") is greater also than the " + str(dbsTolerance) + " lumis that I can tolerate. I can't process runs >= " + str(run) + " but I'll process the runs before!"
                        sendEmail(mailList,error)
                        print error
                        return filesToProcess
                        #exit(errors)
                    #return filesToProcess
                elif len(errors) != 0:
                    print "The number of lumi sections processed didn't match the one in DBS but they cover all the ones in the Run Registry, so it is ok!"
                    #print errors

            #If I get here it means that I passed or the DBS or the RR test            
            if len(badProcessed) == 0:
                for file in runsAndFiles[run]:
                    filesToProcess.append(file)
            else:
                #print errors
                print "This should never happen because if I have errors I return or exit! Run: " + str(run)
        else:
            error = "Run " + str(run) + " is in the run registry but it has not been processed yet!"
            print error
            timeoutType = timeoutManager("MISSING_RUNREGRUN_Run"+str(run),missingLumisTimeout)
            if timeoutType == 1:
                if len(RRList) <= rrTolerance:
                    error = "WARNING: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " but it only had " + str(len(RRList)) + " <= " + str(rrTolerance) + " lumis. So I will continue and ignore it... "
                    #print listOfRunsAndLumiFromRR[run]
                    print error
                    #sendEmail(mailList,error)
                else:
                    error = "ERROR: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " which has " + str(len(RRList)) + " > " + str(rrTolerance) + " lumis. I can't continue but I'll process the runs before this one"
                    sendEmail(mailList,error)
                    return filesToProcess
                    #exit(error)
            else:
                if timeoutType == -1:
                    print "WARNING: Setting the MISSING_RUNREGRUN_Run" + str(run) + " timeout because I haven't processed a run!"
                else:
                    print "WARNING: Timeout MISSING_RUNREGRUN_Run" + str(run) + " is in progress."
                return filesToProcess
                    
    return filesToProcess
########################################################################
def compareLumiLists(listA,listB,errors=[],tolerance=0):
    lenA = len(listA)
    lenB = len(listB)
    if lenA < lenB-(lenB*float(tolerance)/100):
        errors.append("ERROR: The number of lumi sections is different: listA(" + str(lenA) + ")!=(" + str(lenB) + ")listB")
    #else:
        #errors.append("Lumi check ok!listA(" + str(lenA) + ")-(" + str(lenB) + ")listB")
    #print errors
    listA.sort()
    listB.sort()
    #shorter = lenA
    #if lenB < shorter:
    #    shorter = lenB
    #a = 0
    #b = 0
    badA = []
    badB = []
    #print listB
    #print listA
    #print len(listA)
    #print len(listB)
    #counter = 1
    for lumi in listA:
        #print str(counter) + "->" + str(lumi)
        #counter += 1
        if not lumi in listB:
            errors.append("Lumi (" + str(lumi) + ") is in listA but not in listB")
            badB.append(lumi)
            #print "Bad B: " + str(lumi)
    #exit("hola")
    for lumi in listB:
        if not lumi in listA:
            errors.append("Lumi (" + str(lumi) + ") is in listB but not in listA")
            badA.append(lumi)
            #print "Bad A: " + str(lumi)
            
    return badA,badB

########################################################################
def removeUncompleteRuns(newRunList,dataSet):
    processedRuns = {}
    for fileName in newRunList:
        run = getRunNumberFromFileName(fileName)
        if not run in processedRuns:
            processedRuns[run] = 0
        processedRuns[run] += 1

    for run in processedRuns.keys():   
        nFiles = getNumberOfFilesToProcessForRun(dataSet,run)
        if processedRuns[run] < nFiles:
            print "I haven't processed all files yet : " + str(processedRuns[run]) + " out of " + str(nFiles) + " for run: " + str(run)
        else:
            print "All files have been processed for run: " + str(run) + " (" + str(processedRuns[run]) + " out of " + str(nFiles) + ")"
            
########################################################################
def aselectFilesToProcess(listOfFilesToProcess,newRunList):
    selectedFiles = []
    runsToProcess = {}
    processedRuns = {}
    for file in listOfFilesToProcess:
        run = getRunNumberFromDBSName(file)
#        print "To process: " + str(run) 
        if run not in runsToProcess:
            runsToProcess[run] = 1
        else:
            runsToProcess[run] = runsToProcess[run] + 1 

    for file in newRunList:
        run = getRunNumberFromFileName(file)
#        print "Processed: " + str(run)
        if run not in processedRuns:
            processedRuns[run] = 1
        else:
            processedRuns[run] = processedRuns[run] + 1 

    #WARNING: getLastClosedRun MUST also have a timeout otherwise the last run will not be considered
    lastClosedRun = getLastClosedRun(listOfFilesToProcess)
#    print "LastClosedRun:-" + str(lastClosedRun) + "-"

    processedRunsKeys = sorted(processedRuns.keys())

    for run in processedRunsKeys:
        if run <= lastClosedRun :
            print "For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files and in DBS there are " + str(runsToProcess[run]) + " files!"
            if not run in runsToProcess:
                exit("ERROR: I have a result file for run " + str(run) + " but it doesn't exist in DBS. Impossible but it happened!")
            lumiList = getDBSLumiListForRun(run)
            if processedRuns[run] == runsToProcess[run]:
                for file in newRunList:
                    if run == getRunNumberFromFileName(file):
                        selectedFiles.append(file)
            else:
                exit("ERROR: For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files but in DBS there are " + str(runsToProcess[run]) + " files!")
    return selectedFiles            

########################################################################
def main():
    ######### COMMAND LINE OPTIONS ##############
    option,args = parse(__doc__)

    ######### Check if there is already a megascript running ########
    if option.lock:
        setLockName('.' + option.lock)
        if checkLock():
            print "There is already a megascript runnning...exiting"
            return
        else:
            lock()
            

    destDB = 'oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT'
    if option.Test:
        destDB = 'oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT'

    ######### CONFIGURATION FILE ################
    cfgFile = "BeamSpotWorkflow.cfg"    
    if option.cfg:
        cfgFile = option.cfg
    configurationFile = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/scripts/" + cfgFile
    configuration     = ConfigParser.ConfigParser()
    print 'Reading configuration from ', configurationFile
    configuration.read(configurationFile)

    sourceDir             = configuration.get('Common','SOURCE_DIR')
    archiveDir            = configuration.get('Common','ARCHIVE_DIR')
    workingDir            = configuration.get('Common','WORKING_DIR')
    databaseTag           = configuration.get('Common','DBTAG')
    dataSet               = configuration.get('Common','DATASET')
    fileIOVBase           = configuration.get('Common','FILE_IOV_BASE')
    dbIOVBase             = configuration.get('Common','DB_IOV_BASE')
    dbsTolerance          = float(configuration.get('Common','DBS_TOLERANCE'))
    dbsTolerancePercent   = float(configuration.get('Common','DBS_TOLERANCE_PERCENT'))
    rrTolerance           = float(configuration.get('Common','RR_TOLERANCE'))
    missingFilesTolerance = float(configuration.get('Common','MISSING_FILES_TOLERANCE'))
    missingLumisTimeout   = float(configuration.get('Common','MISSING_LUMIS_TIMEOUT'))
    jsonFileName          = configuration.get('Common','JSON_FILE')
    mailList              = configuration.get('Common','EMAIL')

    ######### DIRECTORIES SETUP #################
    if sourceDir[len(sourceDir)-1] != '/':
        sourceDir = sourceDir + '/'
    if not dirExists(sourceDir):
        error = "ERROR: The source directory " + sourceDir + " doesn't exist!"
        sendEmail(mailList,error)
        exit(error)

    if archiveDir[len(archiveDir)-1] != '/':
        archiveDir = archiveDir + '/'
    if not os.path.isdir(archiveDir):
    os.mkdir(archiveDir)

    if workingDir[len(workingDir)-1] != '/':
        workingDir = workingDir + '/'
    if not os.path.isdir(workingDir):
    os.mkdir(workingDir)
    else:
        os.system("rm -f "+ workingDir + "*") 


    print "Getting last IOV for tag: " + databaseTag
    lastUploadedIOV = 1
    if destDB == "oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT": 
        lastUploadedIOV = getLastUploadedIOV(databaseTag)
    else:
        lastUploadedIOV = getLastUploadedIOV(databaseTag,destDB)
        
    #lastUploadedIOV = 133885
    #lastUploadedIOV = 575216380019329
    if dbIOVBase == "lumiid":
        lastUploadedIOV = unpackLumiid(lastUploadedIOV)["run"]

    ######### Get list of files processed after the last IOV  
    print "Getting list of files processed after IOV " + str(lastUploadedIOV)
    newProcessedRunList      = getNewRunList(sourceDir,lastUploadedIOV)
    if len(newProcessedRunList) == 0:
        exit("There are no new runs after " + str(lastUploadedIOV))

    ######### Copy files to archive directory
    print "Copying files to archive directory"
    copiedFiles = []
    for i in range(3):
        copiedFiles = cp(sourceDir,archiveDir,newProcessedRunList)    
        if len(copiedFiles) == len(newProcessedRunList):
            break;
    if len(copiedFiles) != len(newProcessedRunList):
        error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(newProcessedRunList)) 
        sendEmail(mailList,error)
        exit(error)


    ######### Get from DBS the list of files after last IOV    
    #listOfFilesToProcess = getListOfFilesToProcess(dataSet,lastUploadedIOV) 
    print "Getting list of files from DBS"
    listOfRunsAndLumiFromDBS = getListOfRunsAndLumiFromDBS(dataSet,lastUploadedIOV)
    if len(listOfRunsAndLumiFromDBS) == 0:
       exit("There are no files in DBS to process") 
    print "Getting list of files from RR"
    listOfRunsAndLumiFromRR  = getListOfRunsAndLumiFromRR(lastUploadedIOV) 
    if(not listOfRunsAndLumiFromRR):
        print "Looks like I can't get anything from the run registry so I'll get the data from the json file " + jsonFileName
        listOfRunsAndLumiFromRR  = getListOfRunsAndLumiFromFile(lastUploadedIOV,jsonFileName) 
    ######### Get list of files to process for DB
    #selectedFilesToProcess = selectFilesToProcess(listOfFilesToProcess,copiedFiles)
    #completeProcessedRuns = removeUncompleteRuns(copiedFiles,dataSet)
    #print copiedFiles
    #print completeProcessedRuns
    #exit("complete")
    print "Getting list of files to process"
    selectedFilesToProcess = selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,copiedFiles,archiveDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout)
    if len(selectedFilesToProcess) == 0:
       exit("There are no files to process")
        
    #print selectedFilesToProcess
    ######### Copy files to working directory
    print "Copying files from archive to working directory"
    copiedFiles = []
    for i in range(3):
        copiedFiles = cp(archiveDir,workingDir,selectedFilesToProcess)    
        if len(copiedFiles) == len(selectedFilesToProcess):
            break;
        else:
            commands.getstatusoutput("rm -rf " + workingDir)
    if len(copiedFiles) != len(selectedFilesToProcess):
        error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(selectedFilesToProcess)) + " from " + archiveDir + " to " + workingDir 
        sendEmail(mailList,error)
        exit(error)

    print "Sorting and cleaning beamlist"
    beamSpotObjList = []
    for fileName in copiedFiles:
        readBeamSpotFile(workingDir+fileName,beamSpotObjList,fileIOVBase)

    sortAndCleanBeamList(beamSpotObjList,fileIOVBase)

    if len(beamSpotObjList) == 0:
        error = "WARNING: None of the processed and copied payloads has a valid fit so there are no results. This shouldn't happen since we are filtering using the run register, so there should be at least one good run."
        exit(error)

    payloadFileName = "PayloadFile.txt"

    runBased = False
    if dbIOVBase == "runnumber":
        runBased = True
        
    payloadList = createWeightedPayloads(workingDir+payloadFileName,beamSpotObjList,runBased)
    if len(payloadList) == 0:
        error = "WARNING: I wasn't able to create any payload even if I have some BeamSpot objects."
        exit(error)
       

    tmpPayloadFileName = workingDir + "SingleTmpPayloadFile.txt"
    tmpSqliteFileName  = workingDir + "SingleTmpSqliteFile.db"

    writeDBTemplate = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/write2DB_template.py"
    readDBTemplate  = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/readDB_template.py"
    payloadNumber = -1
    iovSinceFirst = '0';
    iovTillLast   = '0';

    #Creating the final name for the combined sqlite file
    uuid = commands.getstatusoutput('uuidgen -t')[1]
    final_sqlite_file_name = databaseTag + '@' + uuid
    sqlite_file     = workingDir + final_sqlite_file_name + ".db"
    metadata_file   = workingDir + final_sqlite_file_name + ".txt"

    for payload in payloadList:
        payloadNumber += 1
        if option.zlarge:
            payload.sigmaZ = 10
            payload.sigmaZerr = 2.5e-05
        tmpFile = file(tmpPayloadFileName,'w')
        dumpValues(payload,tmpFile)
        tmpFile.close()
        if not writeSqliteFile(tmpSqliteFileName,databaseTag,dbIOVBase,tmpPayloadFileName,writeDBTemplate,workingDir):
            error = "An error occurred while writing the sqlite file: " + tmpSqliteFileName
            exit(error)
        readSqliteFile(tmpSqliteFileName,databaseTag,readDBTemplate,workingDir)
        
        ##############################################################
        #WARNING I am not sure if I am packing the right values
        if dbIOVBase == "runnumber":
            iov_since = str(payload.Run)
            iov_till  = iov_since
        elif dbIOVBase == "lumiid":
        iov_since = str( pack(int(payload.Run), int(payload.IOVfirst)) )
            iov_till  = str( pack(int(payload.Run), int(payload.IOVlast)) )
        elif dbIOVBase == "timestamp":
            error = "ERROR: IOV " + dbIOVBase + " still not implemented."
            exit(error)
        else:
            error = "ERROR: IOV " + dbIOVBase + " unrecognized!"
            exit(error)

        if payloadNumber == 0:
            iovSinceFirst = iov_since
        if payloadNumber == len(payloadList)-1:
            iovTillLast   = iov_till
            
        appendSqliteFile(final_sqlite_file_name + ".db", tmpSqliteFileName, databaseTag, iov_since, iov_till ,workingDir)
        os.system("rm -f " + tmpPayloadFileName + " " + tmpSqliteFileName)

        
    #### CREATE payload for merged output

    print " create MERGED payload card for dropbox ..."

    dfile = open(metadata_file,'w')

    dfile.write('destDB '  + destDB        +'\n')
    dfile.write('tag '     + databaseTag   +'\n')
    dfile.write('inputtag'                 +'\n')
    dfile.write('since '   + iovSinceFirst +'\n')
    #dfile.write('till '    + iov_till      +'\n')
    dfile.write('Timetype '+ dbIOVBase     +'\n')

    ###################################################
    # WARNING tagType forced to offline
    print "WARNING TAG TYPE forced to be just offline"
    tagType = "offline"
    checkType = tagType
    if tagType == "express":
        checkType = "hlt"
    dfile.write('IOVCheck ' + checkType + '\n')
    dfile.write('usertext Beam spot position\n')
            
    dfile.close()

                                                                                                

    if option.upload:
        print " scp files to offline Drop Box"
        dropbox = "/DropBox"
        if option.Test:
            dropbox = "/DropBox_test"
        print "UPLOADING TO TEST DB"
        uploadSqliteFile(workingDir, final_sqlite_file_name, dropbox)
                   
    archive_sqlite_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + final_sqlite_file_name
    archive_results_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + databaseTag + ".txt"
    if not os.path.isdir(archiveDir + 'payloads'):
        os.mkdir(archiveDir + 'payloads')
    commands.getstatusoutput('mv ' + sqlite_file   + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.db')
    commands.getstatusoutput('mv ' + metadata_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.txt')
    commands.getstatusoutput('cp ' + workingDir + payloadFileName + ' ' + archiveDir + 'payloads/' + archive_results_file_name)
  
    print archiveDir + "payloads/" + archive_sqlite_file_name + '.db'
    print archiveDir + "payloads/" + archive_sqlite_file_name + '.txt'

    rmLock()
    
if __name__ == '__main__':
    main()