BeamSpotWorkflow Namespace Reference

Functions
def	aselectFilesToProcess (listOfFilesToProcess, newRunList)
def	compareLumiLists (listA, listB, errors=[], tolerance=0)
def	getLastClosedRun (DBSListOfFiles)
def	getLastUploadedIOV (tagName, destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT")
	General functions. More...
def	getListOfFilesToProcess (dataSet, lastRun=-1)
def	getListOfRunsAndLumiFromDBS (dataSet, lastRun=-1)
def	getListOfRunsAndLumiFromFile (firstRun=-1, fileName="")
def	getListOfRunsAndLumiFromRR (firstRun=-1)
def	getNewRunList (fromDir, lastUploadedIOV)
def	getNumberOfFilesToProcessForRun (dataSet, run)
def	getRunNumberFromDBSName (fileName)
def	getRunNumberFromFileName (fileName)
def	main ()
def	removeUncompleteRuns (newRunList, dataSet)
def	selectFilesToProcess (listOfRunsAndLumiFromDBS, listOfRunsAndLumiFromRR, newRunList, runListDir, dataSet, mailList, dbsTolerance, dbsTolerancePercent, rrTolerance, missingFilesTolerance, missingLumisTimeout)

Variables
	error

Function Documentation

aselectFilesToProcess()

def BeamSpotWorkflow.aselectFilesToProcess	(		listOfFilesToProcess,
			newRunList
	)

Definition at line 491 of file BeamSpotWorkflow.py.

def aselectFilesToProcess(listOfFilesToProcess,newRunList):
    selectedFiles = []
    runsToProcess = {}
    processedRuns = {}
    for file in listOfFilesToProcess:
        run = getRunNumberFromDBSName(file)
#        print "To process: " + str(run) 
        if run not in runsToProcess:
            runsToProcess[run] = 1
        else:
            runsToProcess[run] = runsToProcess[run] + 1 
 
    for file in newRunList:
        run = getRunNumberFromFileName(file)
#        print "Processed: " + str(run)
        if run not in processedRuns:
            processedRuns[run] = 1
        else:
            processedRuns[run] = processedRuns[run] + 1 
 
    #WARNING: getLastClosedRun MUST also have a timeout otherwise the last run will not be considered
    lastClosedRun = getLastClosedRun(listOfFilesToProcess)
#    print "LastClosedRun:-" + str(lastClosedRun) + "-"
 
    processedRunsKeys = sorted(processedRuns.keys())
 
    for run in processedRunsKeys:
        if run <= lastClosedRun :
            print("For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files and in DBS there are " + str(runsToProcess[run]) + " files!")
            if not run in runsToProcess:
                exit("ERROR: I have a result file for run " + str(run) + " but it doesn't exist in DBS. Impossible but it happened!")
            lumiList = getDBSLumiListForRun(run)
            if processedRuns[run] == runsToProcess[run]:
                for file in newRunList:
                    if run == getRunNumberFromFileName(file):
                        selectedFiles.append(file)
            else:
                exit("ERROR: For run " + str(run) + " I have processed " + str(processedRuns[run]) + " files but in DBS there are " + str(runsToProcess[run]) + " files!")
    return selectedFiles            
 

References beamvalidation.exit(), getLastClosedRun(), getRunNumberFromDBSName(), getRunNumberFromFileName(), print(), and str.

compareLumiLists()

def BeamSpotWorkflow.compareLumiLists	(		listA,
			listB,
			errors = [],
			tolerance = 0
	)

Definition at line 436 of file BeamSpotWorkflow.py.

def compareLumiLists(listA,listB,errors=[],tolerance=0):
    lenA = len(listA)
    lenB = len(listB)
    if lenA < lenB-(lenB*float(tolerance)/100):
        errors.append("ERROR: The number of lumi sections is different: listA(" + str(lenA) + ")!=(" + str(lenB) + ")listB")
    #else:
        #errors.append("Lumi check ok!listA(" + str(lenA) + ")-(" + str(lenB) + ")listB")
    #print errors
    listA.sort()
    listB.sort()
    #shorter = lenA
    #if lenB < shorter:
    #    shorter = lenB
    #a = 0
    #b = 0
    badA = []
    badB = []
    #print listB
    #print listA
    #print len(listA)
    #print len(listB)
    #counter = 1
    for lumi in listA:
        #print str(counter) + "->" + str(lumi)
        #counter += 1
        if not lumi in listB:
            errors.append("Lumi (" + str(lumi) + ") is in listA but not in listB")
            badB.append(lumi)
            #print "Bad B: " + str(lumi)
    #exit("hola")
    for lumi in listB:
        if not lumi in listA:
            errors.append("Lumi (" + str(lumi) + ") is in listB but not in listA")
            badA.append(lumi)
            #print "Bad A: " + str(lumi)
 
    return badA,badB
 

References dqmMemoryStats.float, and str.

Referenced by selectFilesToProcess().

getLastClosedRun()

def BeamSpotWorkflow.getLastClosedRun

(

DBSListOfFiles

)

Definition at line 219 of file BeamSpotWorkflow.py.

def getLastClosedRun(DBSListOfFiles):
    runs = []
    for file in DBSListOfFiles:
        runNumber = getRunNumberFromDBSName(file)
        if runs.count(runNumber) == 0: 
            runs.append(runNumber)
 
    if len(runs) <= 1: #No closed run
        return -1
    else:
        runs.sort()
        return long(runs[len(runs)-2])
 

References getRunNumberFromDBSName().

Referenced by aselectFilesToProcess().

getLastUploadedIOV()

def BeamSpotWorkflow.getLastUploadedIOV	(		tagName,
			destDB = "oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"
	)

General functions.

Definition at line 58 of file BeamSpotWorkflow.py.

def getLastUploadedIOV(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):    #return 582088327592295
    listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName 
    dbError = commands.getstatusoutput( listIOVCommand )
    if dbError[0] != 0 :
        if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
            print("Creating a new tag because I got the following error contacting the DB")
            print(dbError[1])
            return 1
            #return 133928
        else:
            exit("ERROR: Can\'t connect to db because:\n" + dbError[1])
 
 
    aCommand = listIOVCommand + " | grep DB= | tail -1 | awk \'{print $1}\'"
    output = commands.getstatusoutput( aCommand )
 
    #WARNING when we pass to lumi IOV this should be long long
    if output[1] == '':
        exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
 
    return long(output[1])
 
 

References beamvalidation.exit(), spr.find(), and print().

Referenced by main().

getListOfFilesToProcess()

def BeamSpotWorkflow.getListOfFilesToProcess	(		dataSet,
			lastRun = -1
	)

Definition at line 82 of file BeamSpotWorkflow.py.

def getListOfFilesToProcess(dataSet,lastRun=-1):
    queryCommand = "dbs --search --query \"find file where dataset=" + dataSet
    if lastRun != -1:
        queryCommand = queryCommand + " and run > " + str(lastRun)
    queryCommand = queryCommand + "\" | grep .root"    
#    print " >> " + queryCommand
    output = commands.getstatusoutput( queryCommand )
    return output[1].split('\n')
 

References submitPVValidationJobs.split(), and str.

getListOfRunsAndLumiFromDBS()

def BeamSpotWorkflow.getListOfRunsAndLumiFromDBS	(		dataSet,
			lastRun = -1
	)

Definition at line 102 of file BeamSpotWorkflow.py.

def getListOfRunsAndLumiFromDBS(dataSet,lastRun=-1):
    datasetList = dataSet.split(',')
    outputList = []
    for data in datasetList:
        queryCommand = "dbs --search --query \"find run,lumi where dataset=" + data
        if lastRun != -1:
            queryCommand = queryCommand + " and run > " + str(lastRun)
        queryCommand += "\""
        print(" >> " + queryCommand)
        output = []
        for i in range(0,3):
            output = commands.getstatusoutput( queryCommand )
            if output[0] == 0 and not (output[1].find("ERROR") != -1 or output[1].find("Error") != -1) :
                break
        if output[0] != 0:
            exit("ERROR: I can't contact DBS for the following reason:\n" + output[1])
        #print output[1]
        tmpList = output[1].split('\n')
        for file in tmpList:
            outputList.append(file)
    runsAndLumis = {}
    for out in outputList:
        regExp = re.search('(\d+)\s+(\d+)',out)
        if regExp:
            run  = long(regExp.group(1))
            lumi = long(regExp.group(2))
            if not run in runsAndLumis:
                runsAndLumis[run] = []
            runsAndLumis[run].append(lumi)
 
#    print runsAndLumis
#    exit("ok")
    return runsAndLumis
 

References mps_setup.append, beamvalidation.exit(), spr.find(), print(), FastTimerService_cff.range, submitPVValidationJobs.split(), and str.

Referenced by main().

getListOfRunsAndLumiFromFile()

def BeamSpotWorkflow.getListOfRunsAndLumiFromFile	(		firstRun = -1,
			fileName = ""
	)

Definition at line 137 of file BeamSpotWorkflow.py.

def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
    file = open(fileName);
    jsonFile = file.read();
    file.close()
    jsonList=json.loads(jsonFile);
 
    selected_dcs = {};
    for element in jsonList:
        selected_dcs[long(element)]=jsonList[element]
    return selected_dcs
 

Referenced by main().

getListOfRunsAndLumiFromRR()

def BeamSpotWorkflow.getListOfRunsAndLumiFromRR

(

firstRun = -1

)

Definition at line 149 of file BeamSpotWorkflow.py.

def getListOfRunsAndLumiFromRR(firstRun=-1):
    RunReg  ="http://pccmsdqm04.cern.ch/runregistry"
    #RunReg  = "http://localhost:40010/runregistry"
    #Dataset=%Online%
    Group   = "Collisions10"
 
    # get handler to RR XML-RPC server
    FULLADDRESS=RunReg + "/xmlrpc"
    #print "RunRegistry from: ",FULLADDRESS
    server = xmlrpclib.ServerProxy(FULLADDRESS)
    #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
    sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) 
    #sel_dcstable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
 
    maxAttempts = 3;
    tries = 0;
    while tries<maxAttempts:
        try:
            run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
            #dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
            break
        except:
            print("Something wrong in accessing runregistry, retrying in 2s....", tries, "/", maxAttempts)
            tries += 1
            time.sleep(2)
        if tries==maxAttempts:
            error = "Run registry unaccessible.....exiting now"
            return {};
 
 
    listOfRuns=[]
    for line in run_data.split("\n"):
        run=line.split(',')[0]
        if run.isdigit():
            listOfRuns.append(run)
 
 
    firstRun = listOfRuns[len(listOfRuns)-1];
    lastRun  = listOfRuns[0];
    sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
 
    tries = 0;
    while tries<maxAttempts:
        try:
            #run_data = server.DataExporter.export('RUN'           , 'GLOBAL', 'csv_runs', sel_runtable)
            dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json'    , sel_dcstable)
            break
        except:
            print("I was able to get the list of runs and now I am trying to access the detector status, retrying in 2s....", tries, "/", maxAttempts)
            tries += 1
            time.sleep(2)
        if tries==maxAttempts:
            error = "Run registry unaccessible.....exiting now"
            return {};
 
    selected_dcs={}
    jsonList=json.loads(dcs_data)
 
    #for element in jsonList:
    for element in listOfRuns:
        #if element in listOfRuns:
        if element in jsonList:
            selected_dcs[long(element)]=jsonList[element]
        else:
            print("WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!") 
            selected_dcs[long(element)]= [[]] 
    #print selected_dcs        
    return selected_dcs
 

References print(), and str.

Referenced by main().

getNewRunList()

def BeamSpotWorkflow.getNewRunList	(		fromDir,
			lastUploadedIOV
	)

Definition at line 248 of file BeamSpotWorkflow.py.

def getNewRunList(fromDir,lastUploadedIOV):
    newRunList = []
    listOfFiles = ls(fromDir,".txt")
    runFileMap = {}
    for fileName in listOfFiles:
        runNumber = getRunNumberFromFileName(fileName) 
        if runNumber > lastUploadedIOV:
            newRunList.append(fileName)
    return newRunList        
 

References getRunNumberFromFileName(), and eostools.ls().

Referenced by main().

getNumberOfFilesToProcessForRun()

def BeamSpotWorkflow.getNumberOfFilesToProcessForRun	(		dataSet,
			run
	)

Definition at line 92 of file BeamSpotWorkflow.py.

def getNumberOfFilesToProcessForRun(dataSet,run):
    queryCommand = "dbs --search --query \"find file where dataset=" + dataSet + " and run = " + str(run) + "\" | grep .root"
    #print " >> " + queryCommand
    output = commands.getstatusoutput( queryCommand )
    if output[0] != 0:
        return 0
    else:
        return len(output[1].split('\n'))
 

References submitPVValidationJobs.split(), and str.

Referenced by removeUncompleteRuns(), and selectFilesToProcess().

getRunNumberFromDBSName()

def BeamSpotWorkflow.getRunNumberFromDBSName

(

fileName

)

Definition at line 241 of file BeamSpotWorkflow.py.

def getRunNumberFromDBSName(fileName):
    regExp = re.search('(\D+)/(\d+)/(\d+)/(\d+)/(\D+)',fileName)
    if not regExp:
        return -1
    return long(regExp.group(3)+regExp.group(4))
 

Referenced by aselectFilesToProcess(), and getLastClosedRun().

getRunNumberFromFileName()

def BeamSpotWorkflow.getRunNumberFromFileName

(

fileName

)

Definition at line 233 of file BeamSpotWorkflow.py.

def getRunNumberFromFileName(fileName):
#    regExp = re.search('(\D+)_(\d+)_(\d+)_(\d+)',fileName)
    regExp = re.search('(\D+)_(\d+)_(\d+)_',fileName)
    if not regExp:
        return -1
    return long(regExp.group(3))
 

Referenced by aselectFilesToProcess(), getNewRunList(), and removeUncompleteRuns().

main()

def BeamSpotWorkflow.main

(

)

Definition at line 532 of file BeamSpotWorkflow.py.

def main():
    
    option,args = parse(__doc__)
 
    
    if option.lock:
        setLockName('.' + option.lock)
        if checkLock():
            print("There is already a megascript runnning...exiting")
            return
        else:
            lock()
 
 
    destDB = 'oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT'
    if option.Test:
        destDB = 'oracle://cms_orcoff_prep/CMS_COND_BEAMSPOT'
 
    
    cfgFile = "BeamSpotWorkflow.cfg"    
    if option.cfg:
        cfgFile = option.cfg
    configurationFile = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/scripts/" + cfgFile
    configuration     = ConfigParser.ConfigParser()
    print('Reading configuration from ', configurationFile)
    configuration.read(configurationFile)
 
    sourceDir             = configuration.get('Common','SOURCE_DIR')
    archiveDir            = configuration.get('Common','ARCHIVE_DIR')
    workingDir            = configuration.get('Common','WORKING_DIR')
    databaseTag           = configuration.get('Common','DBTAG')
    dataSet               = configuration.get('Common','DATASET')
    fileIOVBase           = configuration.get('Common','FILE_IOV_BASE')
    dbIOVBase             = configuration.get('Common','DB_IOV_BASE')
    dbsTolerance          = float(configuration.get('Common','DBS_TOLERANCE'))
    dbsTolerancePercent   = float(configuration.get('Common','DBS_TOLERANCE_PERCENT'))
    rrTolerance           = float(configuration.get('Common','RR_TOLERANCE'))
    missingFilesTolerance = float(configuration.get('Common','MISSING_FILES_TOLERANCE'))
    missingLumisTimeout   = float(configuration.get('Common','MISSING_LUMIS_TIMEOUT'))
    jsonFileName          = configuration.get('Common','JSON_FILE')
    mailList              = configuration.get('Common','EMAIL')
 
    
    if sourceDir[len(sourceDir)-1] != '/':
        sourceDir = sourceDir + '/'
    if not dirExists(sourceDir):
        error = "ERROR: The source directory " + sourceDir + " doesn't exist!"
        sendEmail(mailList,error)
        exit(error)
 
    if archiveDir[len(archiveDir)-1] != '/':
        archiveDir = archiveDir + '/'
    if not os.path.isdir(archiveDir):
        os.mkdir(archiveDir)
 
    if workingDir[len(workingDir)-1] != '/':
        workingDir = workingDir + '/'
    if not os.path.isdir(workingDir):
        os.mkdir(workingDir)
    else:
        os.system("rm -f "+ workingDir + "*") 
 
 
    print("Getting last IOV for tag: " + databaseTag)
    lastUploadedIOV = 1
    if destDB == "oracle://cms_orcon_prod/CMS_COND_31X_BEAMSPOT": 
        lastUploadedIOV = getLastUploadedIOV(databaseTag)
    else:
        lastUploadedIOV = getLastUploadedIOV(databaseTag,destDB)
 
    #lastUploadedIOV = 133885
    #lastUploadedIOV = 575216380019329
    if dbIOVBase == "lumiid":
        lastUploadedIOV = unpackLumiid(lastUploadedIOV)["run"]
 
    
    print("Getting list of files processed after IOV " + str(lastUploadedIOV))
    newProcessedRunList      = getNewRunList(sourceDir,lastUploadedIOV)
    if len(newProcessedRunList) == 0:
        exit("There are no new runs after " + str(lastUploadedIOV))
 
    
    print("Copying files to archive directory")
    copiedFiles = []
    for i in range(3):
        copiedFiles = cp(sourceDir,archiveDir,newProcessedRunList)    
        if len(copiedFiles) == len(newProcessedRunList):
            break;
    if len(copiedFiles) != len(newProcessedRunList):
        error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(newProcessedRunList)) 
        sendEmail(mailList,error)
        exit(error)
 
 
    
    print("Getting list of files from DBS")
    listOfRunsAndLumiFromDBS = getListOfRunsAndLumiFromDBS(dataSet,lastUploadedIOV)
    if len(listOfRunsAndLumiFromDBS) == 0:
        exit("There are no files in DBS to process") 
    print("Getting list of files from RR")
    listOfRunsAndLumiFromRR  = getListOfRunsAndLumiFromRR(lastUploadedIOV) 
    if(not listOfRunsAndLumiFromRR):
        print("Looks like I can't get anything from the run registry so I'll get the data from the json file " + jsonFileName)
        listOfRunsAndLumiFromRR  = getListOfRunsAndLumiFromFile(lastUploadedIOV,jsonFileName) 
    
    print("Getting list of files to process")
    selectedFilesToProcess = selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,copiedFiles,archiveDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout)
    if len(selectedFilesToProcess) == 0:
        exit("There are no files to process")
 
    #print selectedFilesToProcess
    
    print("Copying files from archive to working directory")
    copiedFiles = []
    for i in range(3):
        copiedFiles = cp(archiveDir,workingDir,selectedFilesToProcess)    
        if len(copiedFiles) == len(selectedFilesToProcess):
            break;
        else:
            commands.getstatusoutput("rm -rf " + workingDir)
    if len(copiedFiles) != len(selectedFilesToProcess):
        error = "ERROR: I can't copy more than " + str(len(copiedFiles)) + " files out of " + str(len(selectedFilesToProcess)) + " from " + archiveDir + " to " + workingDir 
        sendEmail(mailList,error)
        exit(error)
 
    print("Sorting and cleaning beamlist")
    beamSpotObjList = []
    for fileName in copiedFiles:
        readBeamSpotFile(workingDir+fileName,beamSpotObjList,fileIOVBase)
 
    sortAndCleanBeamList(beamSpotObjList,fileIOVBase)
 
    if len(beamSpotObjList) == 0:
        error = "WARNING: None of the processed and copied payloads has a valid fit so there are no results. This shouldn't happen since we are filtering using the run register, so there should be at least one good run."
        exit(error)
 
    payloadFileName = "PayloadFile.txt"
 
    runBased = False
    if dbIOVBase == "runnumber":
        runBased = True
 
    payloadList = createWeightedPayloads(workingDir+payloadFileName,beamSpotObjList,runBased)
    if len(payloadList) == 0:
        error = "WARNING: I wasn't able to create any payload even if I have some BeamSpot objects."
        exit(error)
 
 
    tmpPayloadFileName = workingDir + "SingleTmpPayloadFile.txt"
    tmpSqliteFileName  = workingDir + "SingleTmpSqliteFile.db"
 
    writeDBTemplate = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/write2DB_template.py"
    readDBTemplate  = os.getenv("CMSSW_BASE") + "/src/RecoVertex/BeamSpotProducer/test/readDB_template.py"
    payloadNumber = -1
    iovSinceFirst = '0';
    iovTillLast   = '0';
 
    #Creating the final name for the combined sqlite file
    uuid = commands.getstatusoutput('uuidgen -t')[1]
    final_sqlite_file_name = databaseTag + '@' + uuid
    sqlite_file     = workingDir + final_sqlite_file_name + ".db"
    metadata_file   = workingDir + final_sqlite_file_name + ".txt"
 
    for payload in payloadList:
        payloadNumber += 1
        if option.zlarge:
            payload.sigmaZ = 10
            payload.sigmaZerr = 2.5e-05
        tmpFile = file(tmpPayloadFileName,'w')
        dumpValues(payload,tmpFile)
        tmpFile.close()
        if not writeSqliteFile(tmpSqliteFileName,databaseTag,dbIOVBase,tmpPayloadFileName,writeDBTemplate,workingDir):
            error = "An error occurred while writing the sqlite file: " + tmpSqliteFileName
            exit(error)
        readSqliteFile(tmpSqliteFileName,databaseTag,readDBTemplate,workingDir)
 
        
        if dbIOVBase == "runnumber":
            iov_since = str(payload.Run)
            iov_till  = iov_since
        elif dbIOVBase == "lumiid":
            iov_since = str( pack(int(payload.Run), int(payload.IOVfirst)) )
            iov_till  = str( pack(int(payload.Run), int(payload.IOVlast)) )
        elif dbIOVBase == "timestamp":
            error = "ERROR: IOV " + dbIOVBase + " still not implemented."
            exit(error)
        else:
            error = "ERROR: IOV " + dbIOVBase + " unrecognized!"
            exit(error)
 
        if payloadNumber == 0:
            iovSinceFirst = iov_since
        if payloadNumber == len(payloadList)-1:
            iovTillLast   = iov_till
 
        appendSqliteFile(final_sqlite_file_name + ".db", tmpSqliteFileName, databaseTag, iov_since, iov_till ,workingDir)
        os.system("rm -f " + tmpPayloadFileName + " " + tmpSqliteFileName)
 
 
    
 
    print(" create MERGED payload card for dropbox ...")
 
    dfile = open(metadata_file,'w')
 
    dfile.write('destDB '  + destDB        +'\n')
    dfile.write('tag '     + databaseTag   +'\n')
    dfile.write('inputtag'                 +'\n')
    dfile.write('since '   + iovSinceFirst +'\n')
    #dfile.write('till '    + iov_till      +'\n')
    dfile.write('Timetype '+ dbIOVBase     +'\n')
 
    
    print("WARNING TAG TYPE forced to be just offline")
    tagType = "offline"
    checkType = tagType
    if tagType == "express":
        checkType = "hlt"
    dfile.write('IOVCheck ' + checkType + '\n')
    dfile.write('usertext Beam spot position\n')
 
    dfile.close()
 
 
 
    if option.upload:
        print(" scp files to offline Drop Box")
        dropbox = "/DropBox"
        if option.Test:
            dropbox = "/DropBox_test"
        print("UPLOADING TO TEST DB")
        uploadSqliteFile(workingDir, final_sqlite_file_name, dropbox)
 
    archive_sqlite_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + final_sqlite_file_name
    archive_results_file_name = "Payloads_" + iovSinceFirst + "_" + iovTillLast + "_" + databaseTag + ".txt"
    if not os.path.isdir(archiveDir + 'payloads'):
        os.mkdir(archiveDir + 'payloads')
    commands.getstatusoutput('mv ' + sqlite_file   + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.db')
    commands.getstatusoutput('mv ' + metadata_file + ' ' + archiveDir + 'payloads/' + archive_sqlite_file_name + '.txt')
    commands.getstatusoutput('cp ' + workingDir + payloadFileName + ' ' + archiveDir + 'payloads/' + archive_results_file_name)
 
    print(archiveDir + "payloads/" + archive_sqlite_file_name + '.db')
    print(archiveDir + "payloads/" + archive_sqlite_file_name + '.txt')
 
    rmLock()
 

References CommonMethods.appendSqliteFile(), CommonMethods.checkLock(), CommonMethods.createWeightedPayloads(), CommonMethods.dirExists(), CommonMethods.dumpValues(), beamvalidation.exit(), FrontierConditions_GlobalTag_cff.file, dqmMemoryStats.float, getLastUploadedIOV(), getListOfRunsAndLumiFromDBS(), getListOfRunsAndLumiFromFile(), getListOfRunsAndLumiFromRR(), getNewRunList(), if(), createfilelist.int, CommonMethods.lock(), timeUnitHelper.pack(), dumpparser.parse(), print(), FastTimerService_cff.range, CommonMethods.readBeamSpotFile(), CommonMethods.readSqliteFile(), CommonMethods.rmLock(), selectFilesToProcess(), CommonMethods.sendEmail(), CommonMethods.setLockName(), CommonMethods.sortAndCleanBeamList(), str, timeUnitHelper.unpackLumiid(), CommonMethods.uploadSqliteFile(), and CommonMethods.writeSqliteFile().

removeUncompleteRuns()

def BeamSpotWorkflow.removeUncompleteRuns	(		newRunList,
			dataSet
	)

Definition at line 475 of file BeamSpotWorkflow.py.

def removeUncompleteRuns(newRunList,dataSet):
    processedRuns = {}
    for fileName in newRunList:
        run = getRunNumberFromFileName(fileName)
        if not run in processedRuns:
            processedRuns[run] = 0
        processedRuns[run] += 1
 
    for run in processedRuns.keys():   
        nFiles = getNumberOfFilesToProcessForRun(dataSet,run)
        if processedRuns[run] < nFiles:
            print("I haven't processed all files yet : " + str(processedRuns[run]) + " out of " + str(nFiles) + " for run: " + str(run))
        else:
            print("All files have been processed for run: " + str(run) + " (" + str(processedRuns[run]) + " out of " + str(nFiles) + ")")
 

References getNumberOfFilesToProcessForRun(), getRunNumberFromFileName(), print(), and str.

selectFilesToProcess()

def BeamSpotWorkflow.selectFilesToProcess	(		listOfRunsAndLumiFromDBS,
			listOfRunsAndLumiFromRR,
			newRunList,
			runListDir,
			dataSet,
			mailList,
			dbsTolerance,
			dbsTolerancePercent,
			rrTolerance,
			missingFilesTolerance,
			missingLumisTimeout
	)

Definition at line 259 of file BeamSpotWorkflow.py.

def selectFilesToProcess(listOfRunsAndLumiFromDBS,listOfRunsAndLumiFromRR,newRunList,runListDir,dataSet,mailList,dbsTolerance,dbsTolerancePercent,rrTolerance,missingFilesTolerance,missingLumisTimeout):
    runsAndLumisProcessed = {}
    runsAndFiles = {}
    for fileName in newRunList:
        file = open(runListDir+fileName)
        for line in file:
            if line.find("Runnumber") != -1:
                run = long(line.replace('\n','').split(' ')[1])
            elif line.find("LumiRange") != -1:
                lumiLine = line.replace('\n','').split(' ')
                begLumi = long(lumiLine[1])
                endLumi = long(lumiLine[3])
                if begLumi != endLumi:
                    error = "The lumi range is greater than 1 for run " + str(run) + " " + line + " in file: " + runListDir + fileName
                    exit(error)
                else:
                    if not run in runsAndLumisProcessed:
                        runsAndLumisProcessed[run] = []
                    if begLumi in runsAndLumisProcessed[run]:
                        print("Lumi " + str(begLumi) + " in event " + str(run) + " already exist. This MUST not happen but right now I will ignore this lumi!")
                    else:    
                        runsAndLumisProcessed[run].append(begLumi)
        if not run in runsAndFiles:
            runsAndFiles[run] = []
        runsAndFiles[run].append(fileName)    
        file.close()
 
    rrKeys = sorted(listOfRunsAndLumiFromRR.keys())
    dbsKeys = listOfRunsAndLumiFromDBS.keys()
    dbsKeys.sort()
    #I remove the last entry from DBS since I am not sure it is an already closed run!
    lastUnclosedRun = dbsKeys.pop()
    #print "Last unclosed run: " + str(lastUnclosedRun)
    procKeys = runsAndLumisProcessed.keys()
    procKeys.sort()
    #print "Run Registry:"    
    #print rrKeys
    #print "DBS:"    
    #print dbsKeys
    #print "List:"    
    #print procKeys
    #print lastUnclosedRun
    filesToProcess = []
    for run in rrKeys:
        RRList = []
        for lumiRange in listOfRunsAndLumiFromRR[run]:
            if lumiRange != []: 
                for l in range(lumiRange[0],lumiRange[1]+1):
                    RRList.append(long(l))
        if run in procKeys and run < lastUnclosedRun:
            #print "run " + str(run) + " is in procKeys"
            if not run in dbsKeys and run != lastUnclosedRun:
                error = "Impossible but run " + str(run) + " has been processed and it is also in the run registry but it is not in DBS!" 
                exit(error)
            print("Working on run " + str(run))
            nFiles = 0
            for data in dataSet.split(','):
                nFiles = getNumberOfFilesToProcessForRun(data,run)
                if nFiles != 0:
                    break
            if len(runsAndFiles[run]) < nFiles:
                print("I haven't processed all files yet : " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " for run: " + str(run)) 
                if nFiles - len(runsAndFiles[run]) <= missingFilesTolerance:
                    timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run)) # resetting this timeout
                    timeoutType = timeoutManager("DBS_MISMATCH_Run"+str(run),missingLumisTimeout)
                    if timeoutType == 1:
                        print("WARNING: I previously set a timeout that expired...I'll continue with the script even if I didn't process all the lumis!")
                    else:
                        if timeoutType == -1:
                            print("WARNING: Setting the DBS_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!")
                        else:
                            print("WARNING: Timeout DBS_MISMATCH_Run" + str(run) + " is in progress.")
                        return filesToProcess
                else:
                    timeoutType = timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run),missingLumisTimeout)
                    if timeoutType == 1:
                        error = "ERROR: I previously set a timeout that expired...I can't continue with the script because there are too many (" + str(nFiles - len(runsAndFiles[run])) + " files missing) and for too long " + str(missingLumisTimeout/3600) + " hours! I will process anyway the runs before this one (" + str(run) + ")"
                        sendEmail(mailList,error)
                        return filesToProcess
                        #exit(error)
                    else:
                        if timeoutType == -1:
                            print("WARNING: Setting the DBS_VERY_BIG_MISMATCH_Run" + str(run) + " timeout because I haven't processed all files!")
                        else:
                            print("WARNING: Timeout DBS_VERY_BIG_MISMATCH_Run" + str(run) + " is in progress.")
                        return filesToProcess
 
            else:
                timeoutManager("DBS_VERY_BIG_MISMATCH_Run"+str(run))
                timeoutManager("DBS_MISMATCH_Run"+str(run))
                print("I have processed " + str(len(runsAndFiles[run])) + " out of " + str(nFiles) + " files that are in DBS. So I should have all the lumis!") 
            errors          = []
            badProcessed    = []
            badDBSProcessed = []
            badDBS          = []
            badRRProcessed  = []
            badRR           = []
            #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
            badDBSProcessed,badDBS = compareLumiLists(runsAndLumisProcessed[run],listOfRunsAndLumiFromDBS[run],errors)
            for i in range(0,len(errors)):
                errors[i] = errors[i].replace("listA","the processed lumis")
                errors[i] = errors[i].replace("listB","DBS")
            #print errors
            #print badProcessed
            #print badDBS
            #exit("ciao")
            if len(badDBS) != 0:
                print("This is weird because I processed more lumis than the ones that are in DBS!")
            if len(badDBSProcessed) != 0 and run in rrKeys:
                lastError = len(errors)
                #print RRList            
                #It is important for runsAndLumisProcessed[run] to be the first because the comparision is not ==
                badRRProcessed,badRR = compareLumiLists(runsAndLumisProcessed[run],RRList,errors)
                for i in range(0,len(errors)):
                    errors[i] = errors[i].replace("listA","the processed lumis")
                    errors[i] = errors[i].replace("listB","Run Registry")
                #print errors
                #print badProcessed
                #print badRunRegistry
 
                if len(badRRProcessed) != 0:    
                    print("I have not processed some of the lumis that are in the run registry for run: " + str(run))
                    for lumi in badDBSProcessed:
                        if lumi in badRRProcessed:
                            badProcessed.append(lumi)
                    lenA = len(badProcessed)
                    lenB = len(RRList)
                    if 100.*lenA/lenB <= dbsTolerancePercent:
                        print("WARNING: I didn't process " + str(100.*lenA/lenB) + "% of the lumis but I am within the " + str(dbsTolerancePercent) + "% set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis")
                        #print errors
                        badProcessed = []
                    elif lenA <= dbsTolerance:
                        print("WARNING: I didn't process " + str(lenA) + " lumis but I am within the " + str(dbsTolerance) + " lumis set in the configuration. Which corrispond to " + str(lenA) + " out of " + str(lenB) + " lumis")
                        #print errors
                        badProcessed = []
                    else:    
                        error = "ERROR: For run " + str(run) + " I didn't process " + str(100.*lenA/lenB) + "% of the lumis and I am not within the " + str(dbsTolerancePercent) + "% set in the configuration. The number of lumis that I didn't process (" + str(lenA) + " out of " + str(lenB) + ") is greater also than the " + str(dbsTolerance) + " lumis that I can tolerate. I can't process runs >= " + str(run) + " but I'll process the runs before!"
                        sendEmail(mailList,error)
                        print(error)
                        return filesToProcess
                        #exit(errors)
                    #return filesToProcess
                elif len(errors) != 0:
                    print("The number of lumi sections processed didn't match the one in DBS but they cover all the ones in the Run Registry, so it is ok!")
                    #print errors
 
            #If I get here it means that I passed or the DBS or the RR test            
            if len(badProcessed) == 0:
                for file in runsAndFiles[run]:
                    filesToProcess.append(file)
            else:
                #print errors
                print("This should never happen because if I have errors I return or exit! Run: " + str(run))
        else:
            error = "Run " + str(run) + " is in the run registry but it has not been processed yet!"
            print(error)
            timeoutType = timeoutManager("MISSING_RUNREGRUN_Run"+str(run),missingLumisTimeout)
            if timeoutType == 1:
                if len(RRList) <= rrTolerance:
                    error = "WARNING: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " but it only had " + str(len(RRList)) + " <= " + str(rrTolerance) + " lumis. So I will continue and ignore it... "
                    #print listOfRunsAndLumiFromRR[run]
                    print(error)
                    #sendEmail(mailList,error)
                else:
                    error = "ERROR: I previously set the MISSING_RUNREGRUN_Run" + str(run) + " timeout that expired...I am missing run " + str(run) + " which has " + str(len(RRList)) + " > " + str(rrTolerance) + " lumis. I can't continue but I'll process the runs before this one"
                    sendEmail(mailList,error)
                    return filesToProcess
                    #exit(error)
            else:
                if timeoutType == -1:
                    print("WARNING: Setting the MISSING_RUNREGRUN_Run" + str(run) + " timeout because I haven't processed a run!")
                else:
                    print("WARNING: Timeout MISSING_RUNREGRUN_Run" + str(run) + " is in progress.")
                return filesToProcess
 
    return filesToProcess

References mps_setup.append, compareLumiLists(), beamvalidation.exit(), getNumberOfFilesToProcessForRun(), print(), FastTimerService_cff.range, python.rootplot.root2matplotlib.replace(), CommonMethods.sendEmail(), submitPVValidationJobs.split(), str, and CommonMethods.timeoutManager().

Referenced by main().

Variable Documentation

error

BeamSpotWorkflow.error

Definition at line 52 of file BeamSpotWorkflow.py.