CMS 3D CMS Logo

/data/refman/pasoursint/CMSSW_5_3_3/src/Alignment/CommonAlignmentProducer/scripts/AlCaHLTBitMon_QueryRunRegistry.py

Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 
00003 #-----------------------------------------------------
00004 # original author: Andrea Lucaroni
00005 # Revision:        $Revision: 1.2 $
00006 # Last update:     $Date: 2011/12/14 17:04:53 $
00007 # by:              $Author: taroni $
00008 #-----------------------------------------------------
00009 
00010 from xml.dom import minidom
00011 import re
00012 import json
00013 import os 
00014 import stat
00015 import sys
00016 
00017 #include DBS
00018 from DBSAPI.dbsApiException import DbsException
00019 import DBSAPI.dbsApi
00020 from DBSAPI.dbsApiException import *
00021 
00022 # include XML-RPC client library
00023 # RR API uses XML-RPC webservices interface for data access
00024 import xmlrpclib
00025 
00026 import array
00027 import pickle as pk
00028 
00029 from optparse import OptionParser
00030 #####DEBUG
00031 DEBUG = 0
00032 
00033 #size file
00034 def filesize1(n):    
00035     info = os.stat(n)
00036     sz = info[stat.ST_SIZE]
00037     return sz
00038             
00039 ### lumiCalc
00040 def printLumi(file,namefile):
00041     if(filesize1(file) != 0):
00042         string= "lumiCalc.py -c frontier://LumiCalc/CMS_LUMI_PROD -i "
00043         string1= " --nowarning overview >"
00044         string2= string + file + string1 + namefile
00045         data = os.system(string2)
00046     else:
00047         data = ""
00048         print "0 lumi are not avaible"
00049     return data
00050 
00051 ###file  dbs
00052 def DBSquery(dataset,site,run):
00053 
00054     url = "https://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
00055     args = {}
00056     args['url']     = url
00057     args['level']   = 'CRITICAL'
00058     api = DBSAPI.dbsApi.DbsApi(args)
00059     files = api.listFiles(path=dataset,tier_list =site,runNumber=run)
00060     return files
00061 
00062 ###file cff data
00063 def makecff(file_list,namefile):
00064     file1 = open(namefile ,'w')
00065     stringS  =           "process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n"
00066     stringS  = stringS + "readFiles = cms.untracked.vstring()\n"
00067     stringS  = stringS + "secFiles = cms.untracked.vstring()\n"
00068     stringS  = stringS + "\n"
00069     file1.write(stringS)
00070     
00071     filecount = 0
00072     extendOpen = 0
00073     for filename in file_list:
00074         
00075         if extendOpen == 0:
00076             stringS  = "readFiles.extend([\n"
00077             file1.write(stringS)
00078             extendOpen = 1
00079             
00080         stringS  =           "     '"
00081         stringS  = stringS + str(filename)
00082         stringS  = stringS + "',\n"
00083         file1.write(stringS)
00084         filecount = filecount + 1
00085         if filecount == 50:
00086             stringS  = "])\n\n"
00087             file1.write(stringS)
00088             filecount = 0
00089             extendOpen = 0
00090 
00091     if extendOpen == 1:
00092         stringS  =           "])\n\n"
00093         file1.write(stringS)
00094     
00095     stringS  =           "process.source = cms.Source(\"PoolSource\",\n"
00096     stringS  = stringS + "         fileNames = readFiles,\n"
00097     stringS  = stringS + "         secondaryFileNames = secFiles\n"
00098     stringS  = stringS + ")\n"
00099     file1.write(stringS)
00100     file1.close()
00101 
00102 
00103 def defineOptions():
00104     parser = OptionParser()
00105     parser.add_option("-w", "--workspace",
00106                       dest="workspaceName",
00107                       default="GLOBAL",
00108                       help="define workspace: GLOBAL TRACKER ")
00109 
00110     parser.add_option("-r", "--regexp",
00111                       dest="regexp",
00112                       type="string",
00113                       default='groupName : LIKE %Collisions10% , runNumber : = 136088',
00114                       help=" \"{runNumber} >= 148127 and {runNumber} < 148128 \" ")
00115 
00116     parser.add_option("-d", "--datasetPath",
00117                       dest="dataset", \
00118                       default="/MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO",
00119                       help="For example : --datasetPath /MinimumBias/Run2010A-TkAlMinBias-Dec22ReReco_v1/ALCARECO")
00120     
00121     parser.add_option("-s", "--site",
00122                       dest="site",
00123                       default="T2_CH_CAF",
00124                       help="For example : site T2_CH_CAF")                 
00125 
00126     parser.add_option("-i", "--info",
00127                       action="store_true",
00128                       dest="info",
00129                       default=False,
00130                       help="printout the column names on which it's possible to cut")
00131     
00132     (options, args) = parser.parse_args()
00133     if len(sys.argv) == 1:
00134         print("\nUsage: %s --help"%sys.argv[0])
00135         sys.exit(0)
00136     
00137     return options
00138 
00139     
00140 def serverQuery(workspaceName,regexp):
00141 
00142     # get handler to RR XML-RPC server
00143     server = xmlrpclib.ServerProxy('https://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
00144     if DEBUG:
00145         print regexp
00146     data = server.RunDatasetTable.export(workspaceName,'xml_all' ,regexp)
00147     return data
00148 
00149 #----------------------------------------------------
00150 
00151 def getText(nodelist):
00152     rc = ""
00153     for node in nodelist:
00154         if node.nodeType == node.TEXT_NODE:
00155             rc = rc + node.data
00156     return rc
00157 
00158 def getElement(obj,name):
00159     return obj.getElementsByTagName(name)
00160 
00161 def printObj(obj,name):
00162     return getText(getElement(obj,name)[0].childNodes)
00163 
00164 
00165 def getData(doc,options,dataset,site):
00166     server = xmlrpclib.ServerProxy('https://cms-service-runregistry-api.web.cern.ch/cms-service-runregistry-api/xmlrpc')
00167     runs = getElement(doc,'RUN')
00168     txtLongData=""
00169     txtkey=""
00170     lista=[]
00171     
00172     sep="\t"
00173 
00174     for run in runs:
00175         txtrun=printObj(run,'NUMBER') + sep + printObj(run,'HLTKEY')
00176         txtLongData+= txtrun + sep + "\n" 
00177 
00178     for run in runs:
00179         test=printObj(run,'HLTKEY')
00180         if not (test in lista):
00181             lista.append(test)
00182 
00183         file2=open("lista_key.txt",'w')
00184         for pkey in range(len(lista)):
00185             pwkey = lista[pkey] +"\n"
00186             file2.write(pwkey)
00187 
00188         file2.close()
00189 
00190     for i in range(len(lista)):
00191         if DEBUG:
00192             print lista[i]
00193         nameDBS=""
00194         nameDBS=str(i)+".data"
00195         name=""
00196         name=str(i)+".json"
00197         nameLumi=""
00198         nameLumi=str(i)+".lumi"
00199         file1 = open( name ,'w')
00200         listaDBS = []
00201         if DEBUG:
00202             print nameDBS
00203         for run in runs:
00204             key=printObj(run,'HLTKEY')
00205             if (key == lista[i]):
00206                 print "running......"
00207                 if DEBUG:
00208                     print printObj(run,'NUMBER')
00209                 txtruns = "{runNumber} >= " + printObj(run,'NUMBER') +  " and {runNumber} < " + str(int(printObj(run,'NUMBER'))+1)
00210                 txtriv = txtruns + " and {cmpPix} in ('GOOD') and {cmpStrip} in ('GOOD') and {cmpTrack} in ('GOOD')"
00211                 riv = server.RunDatasetTable.export('GLOBAL', 'csv_run_numbers',txtriv)
00212                 if riv:
00213                     lumirun = server.RunLumiSectionRangeTable.export('GLOBAL', 'json',txtruns)
00214                     ###dbs file
00215                     file = DBSquery(dataset,site,str(printObj(run,'NUMBER')))                    
00216                     for uno in file:
00217                         stringDBS = {}
00218                         stringDBS = uno['LogicalFileName']
00219                         listaDBS    += [stringDBS]
00220                     ###
00221                     if DEBUG:
00222                         print lumirun
00223                     comp="{}"
00224                     if (lumirun == comp):
00225                         print "LUMI ZERO"
00226                     else:
00227                         file1.write(lumirun)
00228                
00229         file1.close()
00230         string=""
00231         string="sed -i 's/}{/, /g'"
00232         string2=""
00233         string2= string + " " + name
00234         os.system(string2)
00235         printLumi(name,nameLumi)
00236         os.system("sed -i 's/\//_/g' lista_key.txt")
00237         listaDBS2 =[]
00238         for rootLSF in listaDBS:
00239             if not (rootLSF in listaDBS2):
00240                 listaDBS2.append(rootLSF)
00241         makecff(listaDBS2,nameDBS)
00242           
00243     return txtLongData
00244 
00245 #---------------------------------------------
00246 
00247 def extractData(mode,reg,dataset,site,options):
00248     doc = minidom.parseString(serverQuery(mode,reg))
00249     return getData(doc,options,dataset,site)
00250 
00251 def getRegExp(regexp):
00252     items = regexp.split(',') 
00253     dd = {}
00254     for item in items:
00255         key,value = item.split(':')
00256         dd[key.replace(' ','')] = value
00257     return dd
00258 
00259 
00260 #---------------------------------------------MAIN
00261 
00262 options = defineOptions()
00263 data=extractData(options.workspaceName,options.regexp,options.dataset,options.site,options)
00264