CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
checkPayloads.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 import sys,os,commands,re
3 import xmlrpclib
4 from CommonMethods import *
5 
6 try: # FUTURE: Python 2.6, prior to 2.6 requires simplejson
7  import json
8 except:
9  try:
10  import simplejson as json
11  except:
12  print "Please set a crab environment in order to get the proper JSON lib"
13  sys.exit(1)
14 
15 #####################################################################################
16 def getUploadedIOVs(tagName,destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT"):
17  listIOVCommand = "cmscond_list_iov -c " + destDB + " -P /afs/cern.ch/cms/DB/conddb -t " + tagName
18  dbError = commands.getstatusoutput( listIOVCommand )
19  if dbError[0] != 0 :
20  if dbError[1].find("metadata entry \"" + tagName + "\" does not exist") != -1:
21  exit(dbError[1])
22  else:
23  exit("ERROR: Can\'t connect to db because:\n" + dbError[1])
24 
25 
26  aCommand = listIOVCommand + " | grep DB= | awk \'{print $1}\'"
27  #print aCommand
28  output = commands.getstatusoutput( aCommand )
29 
30  #WARNING when we pass to lumi IOV this should be long long
31  if output[1] == '':
32  exit("ERROR: The tag " + tagName + " exists but I can't get the value of the last IOV")
33 
34  runs = []
35  for run in output[1].split('\n'):
36  runs.append(long(run))
37 
38  return runs
39 
40 #####################################################################################
41 def getListOfRunsAndLumiFromFile(firstRun=-1,fileName=""):
42  file = open(fileName);
43  jsonFile = file.read();
44  file.close()
45  jsonList=json.loads(jsonFile);
46 
47  selected_dcs = {};
48  for element in jsonList:
49  selected_dcs[long(element)]=jsonList[element]
50  return selected_dcs
51 
52 #####################################################################################
53 def getListOfRunsAndLumiFromRR(firstRun=-1,error=""):
54  RunReg ="http://pccmsdqm04.cern.ch/runregistry"
55  #RunReg = "http://localhost:40010/runregistry"
56  #Dataset=%Online%
57  Group = "Collisions10"
58 
59  # get handler to RR XML-RPC server
60  FULLADDRESS=RunReg + "/xmlrpc"
61  #print "RunRegistry from: ",FULLADDRESS
62  #firstRun = 153000
63  server = xmlrpclib.ServerProxy(FULLADDRESS)
64  #sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun) + " and {datasetName} LIKE '" + Dataset + "'"
65  sel_runtable="{groupName} ='" + Group + "' and {runNumber} > " + str(firstRun)
66 
67  tries = 0;
68  maxAttempts = 3
69  while tries<maxAttempts:
70  try:
71  run_data = server.DataExporter.export('RUN', 'GLOBAL', 'csv_runs', sel_runtable)
72  break
73  except:
74  tries += 1
75  print "Trying to get run data. This fails only 2-3 times so don't panic yet...", tries, "/", maxAttempts
76  time.sleep(1)
77  print "Exception type: ", sys.exc_info()[0]
78  if tries==maxAttempts:
79  error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
80  print error;
81  return {};
82 
83  listOfRuns=[]
84  runErrors = {}
85  for line in run_data.split("\n"):
86  run=line.split(',')[0]
87  if run.isdigit():
88  listOfRuns.append(run)
89 
90  tries = 0
91  maxAttempts = 3
92  firstRun = listOfRuns[len(listOfRuns)-1];
93  lastRun = listOfRuns[0];
94  sel_dcstable="{groupName} ='" + Group + "' and {runNumber} >= " + str(firstRun) + " and {runNumber} <= " + str(lastRun) + " and {parDcsBpix} = 1 and {parDcsFpix} = 1 and {parDcsTibtid} = 1 and {parDcsTecM} = 1 and {parDcsTecP} = 1 and {parDcsTob} = 1 and {parDcsEbminus} = 1 and {parDcsEbplus} = 1 and {parDcsEeMinus} = 1 and {parDcsEePlus} = 1 and {parDcsEsMinus} = 1 and {parDcsEsPlus} = 1 and {parDcsHbheA} = 1 and {parDcsHbheB} = 1 and {parDcsHbheC} = 1 and {parDcsH0} = 1 and {parDcsHf} = 1"
95  while tries<maxAttempts:
96  try:
97  dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json' , sel_dcstable)
98  break
99  except:
100  tries += 1
101  print "I was able to get the list of runs and now I am trying to access the detector status", tries, "/", maxAttempts
102  time.sleep(1)
103  print "Exception type: ", sys.exc_info()[0]
104 
105  if tries==maxAttempts:
106  error = "Ok, now panic...run registry unaccessible...I'll get the runs from a json file!"
107  print error;
108  return {};
109 
110  #This is the original and shold work in the furture as soon as the server will be moved to a more powerfull PC
111  #while tries<maxAttempts:
112  # try:
113  # run_data = server.DataExporter.export('RUN' , 'GLOBAL', 'csv_runs', sel_runtable)
114  # dcs_data = server.DataExporter.export('RUNLUMISECTION', 'GLOBAL', 'json' , sel_dcstable)
115  # #print run_data
116  # #print dcs_data
117  # break
118  # except:
119  # print "Something wrong in accessing runregistry, retrying in 5s...."
120  # tries += 1
121  # time.sleep(2)
122  # print "Exception type: ", sys.exc_info()[0]
123  #
124  # if tries==maxAttempts:
125  # error = "Run registry unaccessible.....exiting now"
126  # sys.exit(error)
127 
128 
129 
130  selected_dcs={}
131  jsonList=json.loads(dcs_data)
132 
133  #for element in jsonList:
134  for element in listOfRuns:
135  #if element in listOfRuns:
136  if element in jsonList:
137  selected_dcs[long(element)]=jsonList[element]
138  else:
139  #print "WARNING: Run " + element + " is a collision10 run with 0 lumis in Run Registry!"
140  selected_dcs[long(element)]= [[]]
141  return selected_dcs
142 
143 #####################################################################################
144 def main():
145  usage = "USAGE: ./checkPayloads.py (optional tagNumber) (optional \"lumi\") (optional \"z\" (optional destDB)"
146  printExtra = False
147  tagNumber = "14"
148  dbBase = ""
149  sigmaZ = ""
150 
151  if len(sys.argv) >= 2:
152  if not sys.argv[1].isdigit():
153  exit(usage)
154  else:
155  tagNumber = sys.argv[1]
156  if len(sys.argv) >= 3:
157  if not sys.argv[2] == "lumi":
158  exit(usage)
159  else:
160  dbBase = "_LumiBased"
161  if len(sys.argv) >= 4:
162  if not sys.argv[3] == "z":
163  exit(usage)
164  else:
165  sigmaZ = "_SigmaZ"
166  destDB = ""
167  if(len(sys.argv) > 4):
168  destDB = sys.argv[4]
169  #132573 Beam lost immediately
170  #132958 Bad strips
171  #133081 Bad pixels bad strips
172  #133242 Bad strips
173  #133472 Bad strips
174  #133473 Only 20 lumisection, run duration 00:00:03:00
175  #133509 Should be good!!!!!!!!!!
176  #136290 Bad Pixels bad strips
177  #138560 Bad pixels bad strips
178  #138562 Bad HLT bad L1T, need to rescale the Jet Triggers
179  #139363 NOT in the bad list but only 15 lumis and stopped for DAQ problems
180  #139455 Bad Pixels and Strips and stopped because of HCAL trigger rate too high
181  #140133 Beams dumped
182  #140182 No pixel and Strips with few entries
183  #141865 Pixel are bad but Strips work. Run is acceptable but need relaxed cuts since there are no pixels. BeamWidth measurement is bad 80um compared to 40um
184  #142461 Run crashed immediately due to PIX, stable beams since LS1
185  #142465 PostCollsions10, beams lost, HCAl DQM partly working
186  #142503 Bad pixels bad strips
187  #142653 Strips not in data taking
188  #143977 No Beam Strips and Pixels bad
189  #148859 Strips and Pixels HV off waiting for beam
190 
191  knownMissingRunList = [132573,132958,133081,133242,133472,133473,136290,138560,138562,139455,140133,140182,142461,142465,142503,142653,143977,148859]
192  tagName = "BeamSpotObjects_2009" + dbBase + sigmaZ + "_v" + tagNumber + "_offline"
193  print "Checking payloads for tag " + tagName
194  listOfRunsAndLumi = {};
195  #listOfRunsAndLumi = getListOfRunsAndLumiFromRR(-1);
196  if(not listOfRunsAndLumi):
197  listOfRunsAndLumi = getListOfRunsAndLumiFromFile(-1,"/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions10/7TeV/StreamExpress/Cert_132440-149442_7TeV_StreamExpress_Collisions10_JSON_v3.txt");
198  tmpListOfIOVs = []
199  if(destDB != ""):
200  tmpListOfIOVs = getUploadedIOVs(tagName,destDB)
201  else:
202  tmpListOfIOVs = getUploadedIOVs(tagName)
203 
204 
205  listOfIOVs = []
206  if(dbBase == ''):
207  listOfIOVs = tmpListOfIOVs
208  else:
209  for iov in tmpListOfIOVs:
210  if((iov >> 32) not in listOfIOVs):
211  listOfIOVs.append(iov >>32)
212  RRRuns = sorted(listOfRunsAndLumi.keys())
213  for run in RRRuns:
214  #print listOfRunsAndLumiFromRR[run]
215  if run not in listOfIOVs:
216  extraMsg = ""
217  if listOfRunsAndLumi[run] == [[]]:
218  extraMsg = " but it is empty in the RR"
219  if not printExtra: continue
220  if run in knownMissingRunList :
221  extraMsg = " but this run is know to be bad " #+ runErrors[run]
222  if not printExtra: continue
223  print "Run: " + str(run) + " is missing for DB tag " + tagName + extraMsg
224 
225 
226 if __name__ == "__main__":
227  main()
228 
def getListOfRunsAndLumiFromRR(firstRun=-1, error="")
void find(edm::Handle< EcalRecHitCollection > &hits, DetId thisDet, std::vector< EcalRecHitCollection::const_iterator > &hit, bool debug=false)
Definition: FindCaloHit.cc:20
def getUploadedIOVs(tagName, destDB="oracle://cms_orcoff_prod/CMS_COND_31X_BEAMSPOT")
def getListOfRunsAndLumiFromFile(firstRun=-1, fileName="")
Definition: main.py:1
#define str(s)
double split
Definition: MVATrainer.cc:139