CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
conditionUploadTest.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 import cx_Oracle
4 import subprocess
5 import json
6 import os
7 import shutil
8 import datetime
9 
10 # Requirement 1: a conddb key for the authentication with valid permission on writing on prep CMS_CONDITIONS account
11 # this could be dropped introducing a specific entry in the .netrc
12 # Requirement 2: an entry "Dropbox" in the .netrc for the authentication
13 
14 class DB:
15  def __init__(self, serviceName, schemaName ):
16  self.serviceName = serviceName
17  self.schemaName = schemaName
18  self.connStr = None
19 
20  def connect( self ):
21  command = "cmscond_authentication_manager -s %s --list_conn | grep '%s@%s'" %(self.serviceName,self.schemaName,self.serviceName)
22  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
23  out = pipe.communicate()[0]
24  srvconn = '%s@%s' %(self.schemaName,self.serviceName)
25  rowpwd = out.split(srvconn)[1].split(self.schemaName)[1]
26  pwd = ''
27  for c in rowpwd:
28  if c != ' ' and c != '\n':
29  pwd += c
30  self.connStr = '%s/%s@%s' %(self.schemaName,pwd,self.serviceName)
31 
32  def setSynchronizationType( self, tag, synchType ):
33  db = cx_Oracle.connect(self.connStr)
34  cursor = db.cursor()
35  db.begin()
36  cursor.execute('UPDATE TAG SET SYNCHRONIZATION =:SYNCH WHERE NAME =:NAME',(synchType,tag,))
37  db.commit()
38 
39  def getLastInsertedSince( self, tag, snapshot ):
40  db = cx_Oracle.connect(self.connStr)
41  cursor = db.cursor()
42  cursor.execute('SELECT SINCE, INSERTION_TIME FROM IOV WHERE TAG_NAME =:TAG_NAME AND INSERTION_TIME >:TIME ORDER BY INSERTION_TIME DESC',(tag,snapshot))
43  row = cursor.fetchone()
44  return row
45 
46  def removeTag( self, tag ):
47  db = cx_Oracle.connect(self.connStr)
48  cursor = db.cursor()
49  db.begin()
50  cursor.execute('DELETE FROM IOV WHERE TAG_NAME =:TAG_NAME',(tag,))
51  cursor.execute('DELETE FROM TAG WHERE NAME=:NAME',(tag,))
52  db.commit()
53 
54 def makeBaseFile( inputTag, startingSince ):
55  cwd = os.getcwd()
56  baseFile = '%s_%s.db' %(inputTag,startingSince)
57  baseFilePath = os.path.join(cwd,baseFile)
58  if os.path.exists( baseFile ):
59  os.remove( baseFile )
60  command = "conddb_import -c sqlite_file:%s -f oracle://cms_orcon_adg/CMS_CONDITIONS -i %s -t %s -b %s" %(baseFile,inputTag,inputTag,startingSince)
61  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
62  out = pipe.communicate()[0]
63  if not os.path.exists( baseFile ):
64  msg = 'ERROR: base file has not been created: %s' %out
65  raise Exception( msg )
66  return baseFile
67 
68 
69 def makeMetadataFile( inputTag, destTag, since, description ):
70  cwd = os.getcwd()
71  metadataFile = os.path.join(cwd,'%s.txt') %destTag
72  if os.path.exists( metadataFile ):
73  os.remove( metadataFile )
74  metadata = {}
75  metadata[ "destinationDatabase" ] = "oracle://cms_orcoff_prep/CMS_CONDITIONS"
76  tagList = {}
77  tagList[ destTag ] = { "dependencies": {}, "synchronizeTo": "any" }
78  metadata[ "destinationTags" ] = tagList
79  metadata[ "inputTag" ] = inputTag
80  metadata[ "since" ] = since
81  metadata[ "userText" ] = description
82  fileName = destTag+".txt"
83  with open( fileName, "w" ) as file:
84  file.write(json.dumps(metadata,file,indent=4,sort_keys=True))
85 
86 def uploadFile( fileName, logFileName ):
87  command = "uploadConditions.py %s" %fileName
88  pipe = subprocess.Popen( command, shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
89  out = pipe.communicate()[0]
90  lines = out.split('\n')
91  ret = False
92  for line in lines:
93  if line.startswith('\t '):
94  if line.startswith('\t status : -2'):
95  print 'ERROR: upload of file %s failed.' %fileName
96  if line.startswith('\t %s' %fileName):
97  returnCode = line.split('\t %s :' %fileName)[1].strip()
98  if returnCode == 'True':
99  ret = True
100  with open(logFileName,'a') as logFile:
101  logFile.write(out)
102  return ret
103 
104 class UploadTest:
105  def __init__(self, db):
106  self.db = db
107  self.errors = 0
108  self.logFileName = 'conditionUloadTest.log'
109 
110  def log( self, msg ):
111  print msg
112  with open(self.logFileName,'a') as logFile:
113  logFile.write(msg)
114  logFile.write('\n')
115 
116  def upload( self, inputTag, baseFile, destTag, synchro, destSince, success, expectedAction ):
117  insertedSince = None
118  destFile = '%s.db' %destTag
119  metaDestFile = '%s.txt' %destTag
120  shutil.copyfile( baseFile, destFile )
121  self.log( '# ---------------------------------------------------------------------------')
122  self.log( '# Testing tag %s with synch=%s, destSince=%s - expecting ret=%s action=%s' %(destTag,synchro,destSince,success,expectedAction))
123 
124  descr = 'Testing conditionsUpload with synch:%s - expected action: %s' %(synchro,expectedAction)
125  makeMetadataFile( inputTag, destTag, destSince, descr )
126  beforeUpload = datetime.datetime.utcnow()
127  ret = uploadFile( destFile, self.logFileName )
128  if ret != success:
129  self.log( 'ERROR: the return value for the upload of tag %s with sychro %s was %s, while the expected result is %s' %(destTag,synchro,ret,success))
130  self.errors += 1
131  else:
132  row = self.db.getLastInsertedSince( destTag, beforeUpload )
133  if ret == True:
134  if expectedAction == 'CREATE' or expectedAction == 'INSERT' or expectedAction == 'APPEND':
135  if destSince != row[0]:
136  self.log( 'ERROR: the since inserted is %s, expected value is %s - expected action: %s' %(row[0],destSince,expectedAction))
137  self.errors += 1
138  else:
139  self.log( '# OK: Found expected value for last since inserted: %s timestamp: %s' %(row[0],row[1]))
140  insertedSince = row[0]
141  elif expectedAction == 'SYNCHRONIZE':
142  if destSince == row[0]:
143  self.log( 'ERROR: the since inserted %s has not been synchronized with the FCSR - expected action: %s' %(row[0],expectedAction))
144  self.errors += 1
145  else:
146  self.log( '# OK: Found synchronized value for the last since inserted: %s timestamp: %s' %(row[0],row[1]))
147  insertedSince = row[0]
148  else:
149  self.log( 'ERROR: found an appended since %s - expected action: %s' %(row[0],expectedAction))
150  self.errors += 1
151  else:
152  if not row is None:
153  self.log( 'ERROR: found new insered since: %s timestamp: %s' %(row[0],row[1]))
154  self.errors += 1
155  if expectedAction != 'FAIL':
156  self.log( 'ERROR: Upload failed. Expected value: %s' %(destSince))
157  self.errors += 1
158  else:
159  self.log( '# OK: Upload failed as expected.')
160  os.remove( destFile )
161  os.remove( metaDestFile )
162  return insertedSince
163 
164 
165 def main():
166  print 'Testing...'
167  serviceName = 'cms_orcoff_prep'
168  schemaName = 'CMS_CONDITIONS'
169  db = DB(serviceName,schemaName)
170  db.connect()
171  inputTag = 'runinfo_31X_mc'
172  bfile0 = makeBaseFile( inputTag,1)
173  bfile1 = makeBaseFile( inputTag,100)
174  test = UploadTest( db )
175  # test with synch=any
176  tag = 'test_CondUpload_any'
177  test.upload( inputTag, bfile0, tag, 'any', 1, True, 'CREATE' )
178  test.upload( inputTag, bfile1, tag, 'any', 1, False, 'FAIL' )
179  test.upload( inputTag, bfile0, tag, 'any', 200, True, 'APPEND' )
180  test.upload( inputTag, bfile0, tag, 'any', 100, True, 'INSERT')
181  test.upload( inputTag, bfile0, tag, 'any', 200, True, 'INSERT')
182  db.removeTag( tag )
183  # test with synch=validation
184  tag = 'test_CondUpload_validation'
185  test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'CREATE')
186  db.setSynchronizationType( tag, 'validation' )
187  test.upload( inputTag, bfile0, tag, 'validation', 1, True, 'INSERT')
188  test.upload( inputTag, bfile0, tag, 'validation', 200, True, 'APPEND')
189  test.upload( inputTag, bfile0, tag, 'validation', 100, True, 'INSERT')
190  db.removeTag( tag )
191  # test with synch=mc
192  tag = 'test_CondUpload_mc'
193  test.upload( inputTag, bfile1, tag, 'mc', 1, False, 'FAIL')
194  test.upload( inputTag, bfile0, tag, 'mc', 1, True, 'CREATE')
195  db.setSynchronizationType( tag, 'mc' )
196  test.upload( inputTag, bfile0, tag, 'mc', 1, False, 'FAIL')
197  test.upload( inputTag, bfile0, tag, 'mc', 200, False, 'FAIL')
198  db.removeTag( tag )
199  # test with synch=hlt
200  tag = 'test_CondUpload_hlt'
201  test.upload( inputTag, bfile0, tag, 'hlt', 1, True, 'CREATE')
202  db.setSynchronizationType( tag, 'hlt' )
203  test.upload( inputTag, bfile0, tag, 'hlt', 200, True, 'SYNCHRONIZE')
204  fcsr = test.upload( inputTag, bfile0, tag, 'hlt', 100, True, 'SYNCHRONIZE')
205  if not fcsr is None:
206  since = fcsr + 200
207  test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'APPEND')
208  since = fcsr + 100
209  test.upload( inputTag, bfile0, tag, 'hlt', since, True, 'INSERT')
210  db.removeTag( tag )
211  # test with synch=express
212  tag = 'test_CondUpload_express'
213  test.upload( inputTag, bfile0, tag, 'express', 1, True, 'CREATE')
214  db.setSynchronizationType( tag, 'express' )
215  test.upload( inputTag, bfile0, tag, 'express', 200, True, 'SYNCHRONIZE')
216  fcsr = test.upload( inputTag, bfile0, tag, 'express', 100, True, 'SYNCHRONIZE')
217  if not fcsr is None:
218  since = fcsr + 200
219  test.upload( inputTag, bfile0, tag, 'express', since, True, 'APPEND')
220  since = fcsr + 100
221  test.upload( inputTag, bfile0, tag, 'express', since, True, 'INSERT')
222  db.removeTag( tag )
223  # test with synch=prompt
224  tag = 'test_CondUpload_prompt'
225  test.upload( inputTag, bfile0, tag, 'prompt', 1, True, 'CREATE')
226  db.setSynchronizationType( tag, 'prompt' )
227  test.upload( inputTag, bfile0, tag, 'prompt', 200, True, 'SYNCHRONIZE')
228  fcsr = test.upload( inputTag, bfile0, tag, 'prompt', 100, True, 'SYNCHRONIZE')
229  if not fcsr is None:
230  since = fcsr + 200
231  test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'APPEND')
232  since = fcsr + 100
233  test.upload( inputTag, bfile0, tag, 'prompt', since, True, 'INSERT')
234  db.removeTag( tag )
235  # test with synch=pcl
236  tag = 'test_CondUpload_pcl'
237  test.upload( inputTag, bfile0, tag, 'pcl', 1, True, 'CREATE')
238  db.setSynchronizationType( tag, 'pcl' )
239  test.upload( inputTag, bfile0, tag, 'pcl', 200, False, 'FAIL')
240  if not fcsr is None:
241  since = fcsr + 200
242  test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'APPEND')
243  since = fcsr + 100
244  test.upload( inputTag, bfile0, tag, 'pcl', since, True, 'INSERT')
245  db.removeTag( tag )
246  # test with synch=offline
247  tag = 'test_CondUpload_offline'
248  test.upload( inputTag, bfile0, tag, 'offline', 1, True, 'CREATE')
249  db.setSynchronizationType( tag, 'offline' )
250  test.upload( inputTag, bfile0, tag, 'offline', 1000, True, 'APPEND')
251  test.upload( inputTag, bfile0, tag, 'offline', 500, False, 'FAIL' )
252  test.upload( inputTag, bfile0, tag, 'offline', 1000, False, 'FAIL' )
253  test.upload( inputTag, bfile0, tag, 'offline', 2000, True, 'APPEND' )
254  db.removeTag( tag )
255  # test with synch=runmc
256  tag = 'test_CondUpload_runmc'
257  test.upload( inputTag, bfile0, tag, 'runmc', 1, True, 'CREATE')
258  db.setSynchronizationType( tag, 'runmc' )
259  test.upload( inputTag, bfile0, tag, 'runmc', 1000, True, 'APPEND')
260  test.upload( inputTag, bfile0, tag, 'runmc', 500, False, 'FAIL' )
261  test.upload( inputTag, bfile0, tag, 'runmc', 1000, False, 'FAIL' )
262  test.upload( inputTag, bfile0, tag, 'runmc', 2000, True, 'APPEND' )
263  db.removeTag( tag )
264  os.remove( bfile0 )
265  os.remove( bfile1 )
266  print 'Done. Errors: %s' %test.errors
267 
268 if __name__ == '__main__':
269  main()
main
Definition: main.py:1
split
double split
Definition: MVATrainer.cc:139
Generated for CMSSW Reference Manual by doxygen 1.8.5