AlCaHLTBitMon_QueryRunRegistry.py

Go to the documentation of this file.

#!/usr/bin/env python3

#-----------------------------------------------------
# original author: Andrea Lucaroni
# Revision:        $Revision: 1.3 $
# Last update:     $Date: 2012/05/22 08:22:04 $
# by:              $Author: taroni $
#-----------------------------------------------------

from __future__ import print_function
from xml.dom import minidom
import re
import json
import os 
import stat
import sys

#include DBS
from DBSAPI.dbsApiException import DbsException
import DBSAPI.dbsApi
from DBSAPI.dbsApiException import *

# include XML-RPC client library
# RR API uses XML-RPC webservices interface for data access
import xmlrpclib

import array
import pickle as pk

from optparse import OptionParser

DEBUG=0
import inspect

def lineno():
    """Returns the current line number in our program."""
    return inspect.currentframe().f_back.f_lineno

#size file
def filesize1(n):    
    info = os.stat(n)
    sz = info[stat.ST_SIZE]
    return sz


def printLumi(file,namefile):
    if(filesize1(file) != 0):
        string= "lumiCalc2.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
        string1= " --nowarning overview >"
        string2= string + file + string1 + namefile
        data = os.system(string2)
    else:
        data = ""
        print("0 lumi are not avaible")
    return data


def DBSquery(dataset,site,run):

    url = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
    if DEBUG:
        print(lineno())
    args = {}
    args['url']     = url
    args['level']   = 'CRITICAL'
    if DEBUG:
        print(lineno())
    api = DBSAPI.dbsApi.DbsApi(args)
    if DEBUG:
        print(lineno())
    try:
        files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
    except DbsApiException as ex:
        print("Caught API Exception %s: %s "  % (ex.getClassName(), ex.getErrorMessage() ))
        files = ""
        if ex.getErrorCode() not in (None, ""):
            print("DBS Exception Error Code: ", ex.getErrorCode())
    if DEBUG:
        print(lineno())
    return files


def makecff(file_list,namefile):
    file1 = open(namefile ,'w')
    stringS  =           "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
    stringS  = stringS + "readFiles = cms.untracked.vstring()\n"
    stringS  = stringS + "secFiles = cms.untracked.vstring()\n"
    stringS  = stringS + "\n"
    file1.write(stringS)

    filecount = 0
    extendOpen = 0
    for filename in file_list:

        if extendOpen == 0:
            stringS  = "readFiles.extend([\n"
            file1.write(stringS)
            extendOpen = 1

        stringS  =           "     '"
        stringS  = stringS + str(filename)
        stringS  = stringS + "',\n"
        file1.write(stringS)
        filecount = filecount + 1
        if filecount == 50:
            stringS  = "])\n\n"
            file1.write(stringS)
            filecount = 0
            extendOpen = 0

    if extendOpen == 1:
        stringS  =           "])\n\n"
        file1.write(stringS)

    stringS  =           "process.source = cms.Source(\"PoolSource\",\n"
    stringS  = stringS + "         fileNames = readFiles,\n"
    stringS  = stringS + "         secondaryFileNames = secFiles\n"
    stringS  = stringS + ")\n"
    file1.write(stringS)
    file1.close()


def defineOptions():
    parser = OptionParser()
    parser.add_option("-w", "--workspace",
                      dest="workspaceName",
                      default="GLOBAL",
                      help="define workspace: GLOBAL TRACKER ")

    parser.add_option("-r", "--regexp",
                      dest="regexp",
                      type="string",
                      default='groupName : LIKE %Collisions12% , runNumber : = 190000 ',
                      help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")

    parser.add_option("-d", "--datasetPath",
                      dest="dataset", \
                      default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
                      help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")

    parser.add_option("-s", "--site",
                      dest="site",
                      default="T2_CH_CAF",
                      help="For example : site T2_CH_CAF")                 

    parser.add_option("-i", "--info",
                      action="store_true",
                      dest="info",
                      default=False,
                      help="printout the column names on which it's possible to cut")

    (options, args) = parser.parse_args()
    if len(sys.argv) == 1:
        print("\nUsage: %s --help"%sys.argv[0])
        sys.exit(0)

    return options


def serverQuery(workspaceName,regexp):

    # get handler to RR XML-RPC server
    server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
    if DEBUG:
        print(lineno(), regexp)
    data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
    return data

#----------------------------------------------------

def getText(nodelist):
    rc = ""
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            rc = rc + node.data
    return rc

def getElement(obj,name):
    return obj.getElementsByTagName(name)

def printObj(obj,name):
    return getText(getElement(obj,name)[0].childNodes)


def getData(doc,options,dataset,site):
    if DEBUG:
        print(lineno(), 'getData')
    server = xmlrpclib.ServerProxy('http://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
    runs = getElement(doc,'RUN')
    txtLongData=""
    txtkey=""
    lista=[]

    sep="\t"

    for run in runs:
        txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
        txtLongData+= txtrun + sep + "\n" 
    for run in runs:
        test=printObj(run,'HLTKEY')
        if not (test in lista):
            lista.append(test)

        file2=open("lista_key.txt",'w')
        for pkey in range(len(lista)):
            pwkey = lista[pkey] +"\n"
            file2.write(pwkey)
            if DEBUG:
                print(lineno(),  lista[pkey])

        file2.close()

    for i in range(len(lista)):
        if DEBUG:
            print(lineno(), lista[i])
        nameDBS=""
        nameDBS=str(i)+".data"
        name=""
        name=str(i)+".json"
        nameLumi=""
        nameLumi=str(i)+".lumi"
        file1 = open( name ,'w')
        listaDBS = []
        if DEBUG:
            print(lineno(), nameDBS)
        for run in runs:
            key=printObj(run,'HLTKEY')
            if (key == lista[i]):
                print("running......", key)
                if DEBUG:
                    print(lineno(), printObj(run,'NUMBER'))
                txtruns = "{runNumber} >= " + printObj(run,'NUMBER') +  " and {runNumber} <= " + str(int(printObj(run,'NUMBER')))
                txtriv = txtruns + " and {cmpDcsBPix} = 1  and {cmpDcsFPix} = 1 and {cmpDcsTibTid} = 1 and {cmpDcsTob} = 1 and {cmpDcsTecM} = 1 and {cmpDcsTecP} = 1"
                lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtriv)
                if DEBUG:
                    print(lineno(), lumirun)
                  
                if lumirun:
                    file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))                    

                if (file != "") :
                    for uno in file:
                        stringDBS = {}
                        stringDBS = uno['LogicalFileName']
                        listaDBS    += [stringDBS]
                    if DEBUG:
                        print(lineno(), lumirun)
                    comp="{}"
                    if (lumirun == comp):
                        print("LUMI ZERO")
                    else:
                        file1.write(lumirun)

        file1.close()
        string=""
        string="sed -i 's/}{/, /g'"
        string2=""
        string2= string + " " + name
        os.system(string2)
        printLumi(name,nameLumi)
        os.system("sed -i 's/\//_/g' lista_key.txt")
        listaDBS2 =[]
        for rootLSF in listaDBS:
            if not (rootLSF in listaDBS2):
                listaDBS2.append(rootLSF)
        makecff(listaDBS2,nameDBS)

    return txtLongData

#---------------------------------------------

def extractData(mode,reg,dataset,site,options):
    doc = minidom.parseString(serverQuery(mode,reg))
    return getData(doc,options,dataset,site)

def getRegExp(regexp):
    items = regexp.split(',') 
    dd = {}
    for item in items:
        key,value = item.split(':')
        dd[key.replace(' ','')] = value
    return dd


#---------------------------------------------MAIN

options = defineOptions()
data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)