CMS 3D CMS Logo
TkMap_script_phase1.py
Go to the documentation of this file.
1 #!/usr/bin/python
2 
3 import sys
4 from ROOT import *
5 import os
6 from subprocess import call
7 import os.path
8 import shutil
9 import subprocess
10 import codecs
11 import re
12 import errno
13 from getGTfromDQMFile_V2 import getGTfromDQMFile
14 
15 def setRunDirectory(runNumber):
16  # Don't forget to add an entry when there is a new era
17  dirDict = { 315252:['Data2018', 'Run2018'],\
18  308336:['Data2018', 'Commissioning2018'],\
19  294644:['Data2017', 'Run2017'],\
20  290123:['Data2017', 'Commissioning2017'],\
21  284500:['Data2016', 'PARun2016'],\
22  271024:['Data2016', 'Run2016'],\
23  264200:['Data2016', 'Commissioning2016'],\
24  246907:['Data2015', 'Run2015'],\
25  232881:['Data2015', 'Commissioning2015'],\
26  211658:['Data2013', 'Run2013'],\
27  209634:['Data2013', 'HIRun2013'],\
28  190450:['Data2012', 'Run2012']}
29  runKey=0
30  for key in sorted(dirDict):
31  if runNumber > key:
32  runKey=key
33  return dirDict[runKey]
34 
35 def downloadOfflineDQMhisto(run, Run_type):
36  runDirval=setRunDirectory(run)
37  DataLocalDir=runDirval[0]
38  DataOfflineDir=runDirval[1]
39  nnn=run/100
40  print 'Processing '+ Run_type + ' in '+DataOfflineDir+"..."
41  File_Name = ''
42  print 'Directory to fetch the DQM file from: https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
43  url = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'
44  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url+" > index.html")
45  f=codecs.open("index.html", 'r')
46  index = f.readlines()
47  if any(str(Run_Number[i]) in s for s in index):
48  for s in index:
49  if (str(Run_Number[i]) in s) and ("__DQMIO.root" in s):
50  File_Name = str(str(s).split("xx/")[1].split("'>DQM")[0])
51  else:
52  print 'No DQM file available. Please check the Offline server'
53  sys.exit(0)
54 
55  print 'Downloading DQM file:'+File_Name
56  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OfflineData/'+DataOfflineDir+'/'+Run_type+'/000'+str(nnn)+'xx/'+File_Name+' > /tmp/'+File_Name)
57 
58  return File_Name
59 
60 def downloadnlineDQMhisto(run, Run_type):
61  runDirval=setRunDirectory(run)
62  DataLocalDir=runDirval[0]
63  DataOfflineDir=runDirval[1]
64  nnn=run/100
65  nnnOnline = run/10000
66  File_Name_online=''
67  deadRocMap = False
68  ##################online file########
69  url1 = 'https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
70  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url1+" > index_online.html")
71 
72  url2 = 'https://cmsweb.cern.ch/dqm/offline/data/browse/ROOT/OnlineData/original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'
73  os.popen("curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET "+url2+" > index_online_backup.html")
74  f_online_backup=codecs.open("index_online_backup.html", 'r')
75  index_online_backup = f_online_backup.readlines()
76 
77  f_online=codecs.open("index_online.html", 'r')
78  index_online = f_online.readlines()
79  if any(str(run) in x for x in index_online):
80  for x in index_online:
81  if (str(run) in x) and ("_PixelPhase1_" in x):
82  File_Name_online=str(str(x).split(".root'>")[1].split("</a></td><td>")[0])
83  deadRocMap = True
84  else:
85  print "Can't find any file in offline server, trying the online server"
86  if any(str(run) in y for y in index_online_backup):
87  for y in index_online:
88  if (str(run) in y) and ("_PixelPhase1_" in y):
89  File_Name_online=str(str(y).split(".root'>")[1].split("</a></td><td>")[0])
90  deadRocMap = True
91  else:
92  print 'No Online DQM file available. Skip dead roc map'
93  deadRocMap = False
94 
95  print 'Downloading DQM file:'+File_Name_online
96 
97 
98  os.system('curl -k --cert /data/users/cctrkdata/current/auth/proxy/proxy.cert --key /data/users/cctrkdata/current/auth/proxy/proxy.cert -X GET https://cmsweb.cern.ch/dqm/online/data/browse/Original/000'+str(nnnOnline)+'xxxx/000'+str(nnn)+'xx/'+File_Name_online+' > /tmp/'+File_Name_online)
99 
100  os.remove('index_online.html')
101  os.remove('index_online_backup.html')
102  return deadRocMap, File_Name_online;
103 
104 
105 def getGT(DQMfile, RunNumber, globalTagVar):
106  globalTag_v0 = getGTfromDQMFile(DQMfile, RunNumber, globalTagVar)
107  print "Global Tag: " + globalTag_v0
108  globalTag = globalTag_v0
109 
110  for z in range(len(globalTag_v0)-2):#clean up the garbage string in the GT
111  if (globalTag_v0[z].isdigit()) and (globalTag_v0[z+1].isdigit()) and (globalTag_v0[z+2].isdigit()) and(globalTag_v0[z+3].isupper()):
112  globalTag = globalTag_v0[z:]
113  if globalTag == "":
114  print " No GlobalTag found: trying from DAS.... ";
115  globalTag = str(os.popen('getGTscript.sh '+filepath+ File_Name+' ' +str(Run_Number[i])));
116  if globalTag == "":
117  print " No GlobalTag found for run: "+str(Run_Number[i]);
118  return globalTag
119 
120 
121 Run_type = sys.argv[1]
122 Run_Number = [int(x) for x in sys.argv[2:]]
123 CMSSW_BASE = str(os.popen('echo ${CMSSW_BASE}').read().strip())
124 
125 ###########Check if user enter the right run type######################
126 if Run_type == 'Cosmics' or Run_type == 'StreamExpress' or Run_type == 'StreamExpressCosmics' or 'ZeroBias' or re.match('ZeroBias([0-9]+?)',Run_type):
127  print Run_type
128 else:
129  print "please enter a valid run type: Cosmics | ZeroBias | StreamExpress | StreamExpressCosmics ";
130  sys.exit(0)
131 
132 for i in range(len(Run_Number)):
133 #################Downloading DQM file############################
134  print "Downloading File!!"
135  nnnOut = Run_Number[i]/1000
136 
137  filepath = '/tmp/'
138  File_Name = downloadOfflineDQMhisto(Run_Number[i], Run_type)
139  deadRocMap, File_Name_online = downloadnlineDQMhisto(Run_Number[i], Run_type)
140 
141  runDirval=setRunDirectory(Run_Number[i])
142  DataLocalDir=runDirval[0]
143  #DataOfflineDir=runDirval[1]
144 
145 ################Check if run is complete##############
146 
147  print "get the run status from DQMFile"
148 
149 
150  check_command = 'check_runcomplete '+filepath+File_Name
151  Check_output = subprocess.call(check_command, shell=True)
152 
153 
154  if Check_output == 0:
155  print 'Using DQM file: '+File_Name
156  else:
157  print '*****************Warning: DQM file is not ready************************';
158  input_var = raw_input("DQM file is incompleted, do you want to continue? (y/n): ")
159  if (input_var == 'y') or (input_var == 'Y'):
160  print 'Using DQM file: '+File_Name
161  else:
162  sys.exit(0)
163 
164 
165 ###################Start making TkMaps################
166 
167  checkfolder = os.path.exists(str(Run_Number[i]))
168  if checkfolder == True:
169  shutil.rmtree(str(Run_Number[i]))
170  os.makedirs(str(Run_Number[i])+'/'+Run_type)
171  else:
172  os.makedirs(str(Run_Number[i])+'/'+Run_type)
173 
174 #######Getting GT##############
175  ####After switch production to 10_X_X release, the clean up section need to be reviewed and modified ##########
176  globalTag = getGT(filepath+File_Name, str(Run_Number[i]), 'globalTag_Step1')
177  ####################################################
178 
179 
180 
181  print " Creating the TrackerMap.... "
182 
183  detIdInfoFileName = 'TkDetIdInfo_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
184  workPath = os.popen('pwd').readline().strip()
185 
186  os.chdir(str(Run_Number[i])+'/'+Run_type)
187 
188 
189  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/SiStripDQM_OfflineTkMap_Template_cfg_DB.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name+' detIdInfoFile='+detIdInfoFileName)
190  os.system('rm -f *svg')
191 ####################### rename bad module list file######################
192  sefile = 'QualityTest_run'+str(Run_Number[i])+'.txt'
193  shutil.move('QTBadModules.log',sefile)
194 
195 ################### put color legend in the TrackerMap###################
196 
197 # PLEASE UNCOMMENT THE LINES BELOW TO GET THE LEGEND ON THE QT TkMAP (it will work only on vocms061)
198 # os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/LegendToQT.py QTestAlarm.png /data/users/cctrack/FinalLegendTrans.png')
199 # shutil.move('result.png', 'QTestAlarm.png')
200 
201 ####################Copying the template html file to index.html########################
202 
203  if Run_type == "Cosmics" or Run_type == "StreamExpressCosmics":
204  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap_cosmics.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
205  else:
206  os.system('cat ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/data/index_template_TKMap.html | sed -e "s@RunNumber@'+str(Run_Number[i])+'@g" > index.html')
207 
208  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/fedmap.html','fedmap.html')
209  shutil.copyfile(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/data/psumap.html','psumap.html')
210 
211  print " Check TrackerMap on "+str(Run_Number[i])+'/'+Run_type+" folder"
212 
213  output =[]
214  output.append(os.popen("/bin/ls ").readline().strip())
215  print output
216 
217 ## Producing the list of bad modules
218  print " Creating the list of bad modules "
219 
220  os.system('listbadmodule '+filepath+'/'+File_Name+' PCLBadComponents.log')
221 
222  ## if Run_type != "StreamExpress":
223  ## shutil.copyfile(sefile, checkdir+'/'+sefile)
224  ## os.system('/usr/bin/python ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/findBadModT9.py -p '+sefile+' -s /'+checkdir+'/'+sefile);
225 
226 
227 ## Producing the run certification by lumisection
228 
229 # print " Creating the lumisection certification:"
230 
231 # if (Run_type.startswith("ZeroBias")) or (Run_type == "StreamExpress"):
232 # os.system('ls_cert 0.95 0.95 '+filepath+'/'+File_Name)
233 
234 ## Producing the PrimaryVertex/BeamSpot quality test by LS..
235 # if (Run_type != "Cosmics") and ( Run_type != "StreamExpress") and (Run_type != "StreamExpressCosmics"):
236 # print " Creating the BeamSpot Calibration certification summary:"
237 # os.system('lsbs_cert '+filepath+'/'+File_Name)
238 
239 ## .. and harvest the bad beamspot LS with automatic emailing (if in period and if bad LS found)
240  os.system('bs_bad_ls_harvester . '+str(Run_Number[i]))
241 
242 
243 ## Producing the Module difference for ExpressStream
244 
245  dest='Beam'
246 
247  if (Run_type == "Cosmics") or (Run_type == "StreamExpressCosmics"):
248  dest="Cosmics"
249 
250 
251 ## create merged list of BadComponent from (PCL, RunInfo and FED Errors) ignore for now
252  os.system('cmsRun ${CMSSW_BASE}/src/DQM/SiStripMonitorClient/test/mergeBadChannel_Template_cfg.py globalTag='+globalTag+' runNumber='+str(Run_Number[i])+' dqmFile='+filepath+'/'+File_Name)
253  shutil.move('MergedBadComponents.log','MergedBadComponents_run'+str(Run_Number[i])+'.txt')
254 
255  os.system("mkdir -p /data/users/event_display/TkCommissioner_runs/"+DataLocalDir+"/"+dest+" 2> /dev/null")
256 
257 
258  shutil.copyfile(detIdInfoFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+detIdInfoFileName)
259 
260  os.remove(detIdInfoFileName)
261  os.remove('MergedBadComponentsTkMap_Canvas.root')
262  os.remove('MergedBadComponentsTkMap.root')
263 ##############counting dead pixel#######################
264  print "countig dead pixel ROCs"
265  if (Run_Number[i] < 290124) :
266 
267  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter.py '+filepath+'/'+File_Name)
268  else:
269  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROCCounter_Phase1.py '+filepath+'/'+File_Name)
270 
271  os.system('mkdir -p /data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type+' 2> /dev/null')
272 
273  shutil.move('PixZeroOccROCs_run'+str(Run_Number[i])+'.txt',workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
274 
275 
276 
277 
278 ######Counting Dead ROCs and Inefficient DC during the run#########################
279 
280  if deadRocMap == True:
281 
282  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/DeadROC_duringRun.py '+filepath+File_Name_online+' '+filepath+File_Name)
283  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/change_name.py')
284  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PixelMapPlotter.py MaskedROC_sum.txt -c')
285  os.remove('MaskedROC_sum.txt')
286  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/InefficientDoubleROC.py '+filepath+File_Name_online)
287  else:
288  print 'No Online DQM file available, Dead ROC maps will not be produced'
289  print 'No Online DQM file available, inefficient DC list will also not be produced'
290 
291 #######Merge Dead ROCs and Occupoancy Plot ###################
292 
293  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/MergeOccDeadROC.py '+filepath+File_Name)
294 
295 ###################copy ouput files###################
296  strip_files = os.listdir('.')
297  for file_name in strip_files:
298  full_stripfile_name = os.path.join('.', file_name)
299  if (os.path.isfile(full_stripfile_name)):
300  shutil.copy(full_stripfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
301 
302 
303 
304 
305 #########################Start making pixel maps#####################
306  os.chdir(workPath)
307  shutil.rmtree(str(Run_Number[i]))
308  os.remove('index.html')
309 
310  # produce pixel phase1 TH2Poly maps
311 # os.chdir(CMSSW_BASE+'/src/DQM/SiStripMonitorClient/scripts/PhaseIMaps/')
312  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/TH2PolyOfflineMaps.py ' + filepath+'/'+File_Name+' 3000 2000')
313  shutil.move(workPath+'/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt', 'OUT/PixZeroOccROCs_run'+str(Run_Number[i])+'.txt')
314 
315 
316 ###################copy ouput files##########
317  pixel_files = os.listdir('./OUT')
318  for file_name in pixel_files:
319  full_pixelfile_name = os.path.join('./OUT/', file_name)
320  if (os.path.isfile(full_pixelfile_name)):
321  shutil.copy(full_pixelfile_name, '/data/users/event_display/'+DataLocalDir+'/'+dest+'/'+str(nnnOut)+'/'+str(Run_Number[i])+'/'+Run_type)
322 
323 
324  shutil.rmtree('OUT')
325 
326 
327  # produce pixel phase1 tree for Offline TkCommissioner
328  pixelTreeFileName = 'PixelPhase1Tree_Run'+str(Run_Number[i])+'_'+Run_type+'.root'
329  os.system('${CMSSW_BASE}/src/DQM/SiStripMonitorClient/scripts/PhaseITreeProducer.py ' + filepath+'/'+File_Name + ' ' + pixelTreeFileName)
330 
331  shutil.copyfile(pixelTreeFileName,'/data/users/event_display/TkCommissioner_runs/'+DataLocalDir+'/'+dest+'/'+pixelTreeFileName)
332  os.remove(pixelTreeFileName)
333  if File_Name:
334  os.remove(filepath+File_Name)
335  if File_Name_online:
336  os.remove(filepath+File_Name_online)
337  os.chdir(workPath)
any
bool any(const std::vector< T > &v, const T &what)
Definition: ECalSD.cc:37
TkMap_script_phase1.downloadnlineDQMhisto
def downloadnlineDQMhisto(run, Run_type)
Definition: TkMap_script_phase1.py:60
TkMap_script_phase1.setRunDirectory
def setRunDirectory(runNumber)
Definition: TkMap_script_phase1.py:15
TkMap_script_phase1.getGT
def getGT(DQMfile, RunNumber, globalTagVar)
Definition: TkMap_script_phase1.py:105
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
split
double split
Definition: MVATrainer.cc:139
TkMap_script_phase1.downloadOfflineDQMhisto
def downloadOfflineDQMhisto(run, Run_type)
Definition: TkMap_script_phase1.py:35
Generated for CMSSW Reference Manual by doxygen 1.8.11