CMS 3D CMS Logo
conditionUploadTest.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 from __future__ import print_function
4 import cx_Oracle
5 import subprocess
6 import json
7 import os
8 import shutil
9 import datetime
10 
11 # Requirement 1: a conddb key for the authentication with valid permission on writing on prep CMS_CONDITIONS account
12 # this could be dropped introducing a specific entry in the .netrc
13 # Requirement 2: an entry "Dropbox" in the .netrc for the authentication
14 
15 class DB:
16  def __init__(self, serviceName, schemaName ):
17  self.serviceName = serviceName
18  self.schemaName = schemaName
19  self.connStr = None
20 
21  def connect( self ):
22  command = "cmscond_authentication_manager -s %s --list_conn | grep '%s@%s'" %(self.serviceName,self.schemaName,self.serviceName)
23  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
24  out = pipe.communicate()[0]
25  srvconn = '%s@%s' %(self.schemaName,self.serviceName)
26  rowpwd = out.split(srvconn)[1].split(self.schemaName)[1]
27  pwd = ''
28  for c in rowpwd:
29  if c != ' ' and c != '\n':
30  pwd += c
31  self.connStr = '%s/%s@%s' %(self.schemaName,pwd,self.serviceName)
32 
33  def setSynchronizationType( self, tag, synchType ):
34  db = cx_Oracle.connect(self.connStr)
35  cursor = db.cursor()
36  db.begin()
37  cursor.execute('UPDATE TAG SET SYNCHRONIZATION =:SYNCH WHERE NAME =:NAME',(synchType,tag,))
38  db.commit()
39 
40  def getLastInsertedSince( self, tag, snapshot ):
41  db = cx_Oracle.connect(self.connStr)
42  cursor = db.cursor()
43  cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =:TAG_NAME AND INSERTION_TIME >:TIME ORDER BY INSERTION_TIME DESC',(tag,snapshot))
44  row = cursor.fetchone()
45  return row
46 
47  def removeTag( self, tag ):
48  db = cx_Oracle.connect(self.connStr)
49  cursor = db.cursor()
50  db.begin()
51  cursor.execute('DELETE FROM IOV WHERE TAG_NAME =:TAG_NAME',(tag,))
52  cursor.execute('DELETE FROM TAG_LOG WHERE TAG_NAME=:TAG_NAME',(tag,))
53  cursor.execute('DELETE FROM TAG WHERE NAME=:NAME',(tag,))
54  db.commit()
55 
56 def makeBaseFile( inputTag, startingSince ):
57  cwd = os.getcwd()
58  baseFile = '%s_%s.db' %(inputTag,startingSince)
59  baseFilePath = os.path.join(cwd,baseFile)
60  if os.path.exists( baseFile ):
61  os.remove( baseFile )
62  command = "conddb_import -c sqlite_file:%s -f oracle://cms_orcon_adg/CMS_CONDITIONS -i %s -t %s -b %s" %(baseFile,inputTag,inputTag,startingSince)
63  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
64  out = pipe.communicate()[0]
65  if not os.path.exists( baseFile ):
66  msg = 'ERROR: base file has not been created: %s' %out
67  raise Exception( msg )
68  return baseFile
69 
70 
71 def makeMetadataFile( inputTag, destTag, since, description ):
72  cwd = os.getcwd()
73  metadataFile = os.path.join(cwd,'%s.txt') %destTag
74  if os.path.exists( metadataFile ):
75  os.remove( metadataFile )
76  metadata = {}
77  metadata[ "destinationDatabase" ] = "oracle://cms_orcoff_prep/CMS_CONDITIONS"
78  tagList = {}
79  tagList[ destTag ] = { "dependencies": {}, "synchronizeTo": "any" }
80  metadata[ "destinationTags" ] = tagList
81  metadata[ "inputTag" ] = inputTag
82  metadata[ "since" ] = since
83  metadata[ "userText" ] = description
84  fileName = destTag+".txt"
85  with open( fileName, "w" ) as file:
86  file.write(json.dumps(metadata,file,indent=4,sort_keys=True))
87 
88 def uploadFile( fileName, logFileName ):
89  command = "uploadConditions.py %s" %fileName
90  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
91  out = pipe.communicate()[0]
92  lines = out.split('\n')
93  ret = False
94  for line in lines:
95  if line.startswith('upload ended with code:'):
96  returnCode = line.split('upload ended with code:')[1].strip()
97  if returnCode == '0':
98  ret = True
99  break
100  with open(logFileName,'a') as logFile:
101  logFile.write(out)
102  return ret
103 
104 class UploadTest:
105  def __init__(self, db):
106  self.db = db
107  self.errors = 0
108  self.logFileName = 'conditionUploadTest.log'
109 
110  def log( self, msg ):
111  print(msg)
112  with open(self.logFileName,'a') as logFile:
113  logFile.write(msg)
114  logFile.write('\n')
115 
116  def upload( self, inputTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
117  insertedSince = None
118  destFile = '%s.db' %destTag
119  metaDestFile = '%s.txt' %destTag
120  shutil.copyfile( baseFile, destFile )
121  self.log( '# ---------------------------------------------------------------------------')
122  self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
123 
124  descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
125  makeMetadataFile( inputTag, destTag, destSince, descr )
126  beforeUpload = datetime.datetime.utcnow()
127  ret = uploadFile( destFile, self.logFileName )
128  if ret != success:
129  self.log( 'ERROR: the return value for the upload of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
130  self.errors += 1
131  else:
132  row = self.db.getLastInsertedSince( destTag, beforeUpload )
133  if ret == True:
134  if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
135  if destSince != row[0]:
136  self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
137  self.errors += 1
138  else:
139  self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
140  insertedSince = row[0]
141  elif expectedAction == 'SYNCHRONIZE':
142  if destSince == row[0]:
143  self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
144  self.errors += 1
145  else:
146  self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
147  insertedSince = row[0]
148  else:
149  self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
150  self.errors += 1
151  else:
152  if not row is None:
153  self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
154  self.errors += 1
155  if expectedAction != 'FAIL':
156  self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
157  self.errors += 1
158  else:
159  self.log( '# OK: Upload failed as expected.')
160  os.remove( destFile )
161  os.remove( metaDestFile )
162  return insertedSince
163 
164 
165 def main():
166  print('Testing...')
167  serviceName = 'cms_orcoff_prep'
168  schemaName = 'CMS_CONDITIONS'
169  db = DB(serviceName,schemaName)
170  db.connect()
171  inputTag = 'runinfo_31X_mc'
172  bfile0 = makeBaseFile( inputTag,1)
173  bfile1 = makeBaseFile( inputTag,100)
174  test = UploadTest( db )
175  # test with synch=any
176  tag = 'test_CondUpload_any'
177  test.upload( inputTag, bfile0, tag, 'any', 1, True, 'CREATE' )
178  test.upload( inputTag, bfile1, tag, 'any', 1, False, 'FAIL' )
179  test.upload( inputTag, bfile0, tag, 'any', 200, True, 'APPEND' )
180  test.upload( inputTag, bfile0, tag, 'any', 100, True, 'INSERT')
181  test.upload( inputTag, bfile0, tag, 'any', 200, True, 'INSERT')
182  db.removeTag( tag )
183  # test with synch=validation
184  tag = 'test_CondUpload_validation'
185  test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'CREATE')
186  db.setSynchronizationType( tag, 'validation' )
187  test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'INSERT')
188  test.upload( inputTag, bfile0, tag, 'validation', 200, True, 'APPEND')
189  test.upload( inputTag, bfile0, tag, 'validation', 100, True, 'INSERT')
190  db.removeTag( tag )
191  # test with synch=mc
192  tag = 'test_CondUpload_mc'
193  test.upload( inputTag, bfile1, tag, 'mc', 1, False, 'FAIL')
194  test.upload( inputTag, bfile0, tag, 'mc', 1, True, 'CREATE')
195  db.setSynchronizationType( tag, 'mc' )
196  test.upload( inputTag, bfile0, tag, 'mc', 1, False, 'FAIL')
197  test.upload( inputTag, bfile0, tag, 'mc', 200, False, 'FAIL')
198  db.removeTag( tag )
199  # test with synch=hlt
200  tag = 'test_CondUpload_hlt'
201  test.upload( inputTag, bfile0, tag, 'hlt', 1, True, 'CREATE')
202  db.setSynchronizationType( tag, 'hlt' )
203  test.upload( inputTag, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')
204  fcsr = test.upload( inputTag, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')
205  if not fcsr is None:
206  since = fcsr + 200
207  test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'APPEND')
208  since = fcsr + 100
209  test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'INSERT')
210  db.removeTag( tag )
211  # test with synch=express
212  tag = 'test_CondUpload_express'
213  test.upload( inputTag, bfile0, tag, 'express', 1, True, 'CREATE')
214  db.setSynchronizationType( tag, 'express' )
215  test.upload( inputTag, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')
216  fcsr = test.upload( inputTag, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')
217  if not fcsr is None:
218  since = fcsr + 200
219  test.upload( inputTag, bfile0, tag, 'express', since, True, 'APPEND')
220  since = fcsr + 100
221  test.upload( inputTag, bfile0, tag, 'express', since, True, 'INSERT')
222  db.removeTag( tag )
223  # test with synch=prompt
224  tag = 'test_CondUpload_prompt'
225  test.upload( inputTag, bfile0, tag, 'prompt', 1, True, 'CREATE')
226  db.setSynchronizationType( tag, 'prompt' )
227  test.upload( inputTag, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')
228  fcsr = test.upload( inputTag, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')
229  if not fcsr is None:
230  since = fcsr + 200
231  test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'APPEND')
232  since = fcsr + 100
233  test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'INSERT')
234  db.removeTag( tag )
235  # test with synch=pcl
236  tag = 'test_CondUpload_pcl'
237  test.upload( inputTag, bfile0, tag, 'pcl', 1, True, 'CREATE')
238  db.setSynchronizationType( tag, 'pcl' )
239  test.upload( inputTag, bfile0, tag, 'pcl', 200, False, 'FAIL')
240  if not fcsr is None:
241  since = fcsr + 200
242  test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'APPEND')
243  since = fcsr + 100
244  test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'INSERT')
245  db.removeTag( tag )
246  # test with synch=offline
247  tag = 'test_CondUpload_offline'
248  test.upload( inputTag, bfile0, tag, 'offline', 1, True, 'CREATE')
249  db.setSynchronizationType( tag, 'offline' )
250  test.upload( inputTag, bfile0, tag, 'offline', 1000, True, 'APPEND')
251  test.upload( inputTag, bfile0, tag, 'offline', 500, False, 'FAIL' )
252  test.upload( inputTag, bfile0, tag, 'offline', 1000, False, 'FAIL' )
253  test.upload( inputTag, bfile0, tag, 'offline', 2000, True, 'APPEND' )
254  db.removeTag( tag )
255  # test with synch=runmc
256  tag = 'test_CondUpload_runmc'
257  test.upload( inputTag, bfile0, tag, 'runmc', 1, True, 'CREATE')
258  db.setSynchronizationType( tag, 'runmc' )
259  test.upload( inputTag, bfile0, tag, 'runmc', 1000, True, 'APPEND')
260  test.upload( inputTag, bfile0, tag, 'runmc', 500, False, 'FAIL' )
261  test.upload( inputTag, bfile0, tag, 'runmc', 1000, False, 'FAIL' )
262  test.upload( inputTag, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
263  db.removeTag( tag )
264  os.remove( bfile0 )
265  os.remove( bfile1 )
266  print('Done. Errors: %s' %test.errors)
267 
268 if __name__ == '__main__':
269  main()
conditionUploadTest.UploadTest.db
db
Definition: conditionUploadTest.py:106
digitizers_cfi.strip
strip
Definition: digitizers_cfi.py:19
conditionUploadTest.UploadTest.logFileName
logFileName
Definition: conditionUploadTest.py:108
conditionUploadTest.DB.setSynchronizationType
def setSynchronizationType(self, tag, synchType)
Definition: conditionUploadTest.py:33
conditionUploadTest.DB.removeTag
def removeTag(self, tag)
Definition: conditionUploadTest.py:47
conditionUploadTest.makeMetadataFile
def makeMetadataFile(inputTag, destTag, since, description)
Definition: conditionUploadTest.py:71
conditionUploadTest.UploadTest.upload
def upload(self, inputTag, baseFile, destTag, synchro, destSince, success, expectedAction)
Definition: conditionUploadTest.py:116
conditionUploadTest.uploadFile
def uploadFile(fileName, logFileName)
Definition: conditionUploadTest.py:88
conditionUploadTest.DB
Definition: conditionUploadTest.py:15
conditionUploadTest.makeBaseFile
def makeBaseFile(inputTag, startingSince)
Definition: conditionUploadTest.py:56
conditionUploadTest.UploadTest.log
def log(self, msg)
Definition: conditionUploadTest.py:110
conditionUploadTest.main
def main()
Definition: conditionUploadTest.py:165
submitPVValidationJobs.split
def split(sequence, size)
Definition: submitPVValidationJobs.py:352
conditionUploadTest.DB.connStr
connStr
Definition: conditionUploadTest.py:19
conditionUploadTest.UploadTest
Definition: conditionUploadTest.py:104
print
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:46
Exception
conditionUploadTest.DB.schemaName
schemaName
Definition: conditionUploadTest.py:18
conditionUploadTest.DB.getLastInsertedSince
def getLastInsertedSince(self, tag, snapshot)
Definition: conditionUploadTest.py:40
conditionUploadTest.DB.connect
def connect(self)
Definition: conditionUploadTest.py:21
conditionUploadTest.DB.__init__
def __init__(self, serviceName, schemaName)
Definition: conditionUploadTest.py:16
main
Definition: main.py:1
conditionUploadTest.DB.serviceName
serviceName
Definition: conditionUploadTest.py:17
conditionUploadTest.UploadTest.errors
errors
Definition: conditionUploadTest.py:107
conditionUploadTest.UploadTest.__init__
def __init__(self, db)
Definition: conditionUploadTest.py:105
Generated for CMSSW Reference Manual by doxygen 1.8.16