CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
popcon2dropbox.py
Go to the documentation of this file.
1 import subprocess
2 import json
3 import netrc
4 import sqlite3
5 import os
6 import shutil
7 from datetime import datetime
8 
9 confFileName ='popcon2dropbox.json'
10 fileNameForDropBox = 'input_for_dropbox'
11 dbFileForDropBox = '%s.db' %fileNameForDropBox
12 dbLogFile = '%s_log.db' %fileNameForDropBox
13 errorInUploadFileFolder = 'upload_errors'
14 dateformatForFolder = "%y-%m-%d-%H-%M-%S"
15 dateformatForLabel = "%y-%m-%d %H:%M:%S"
16 
17 import upload_popcon
18 
19 class CondMetaData(object):
20  def __init__( self, fileName ):
21  self.md = {}
22  self.datef = datetime.now()
23  with open(fileName) as jf:
24  try:
25  self.md = json.load(jf)
26  except ValueError as e:
27  errorMessage = 'CondMetaData.__init__: Problem in decoding JSON file. Original error: ' + e.message
28  raise ValueError(errorMessage)
29 
30  def authPath( self ):
31  apath = ''
32  if self.md.has_key('authenticationPath'):
33  apath = self.md.get('authenticationPath')
34  return apath
35 
36  def authSys( self ):
37  asys = 1
38  if self.md.has_key('authenticationSys'):
39  asys = self.md.get('authenticationSystem')
40  return asys
41 
42  def destinationDatabase( self ):
43  return self.md.get('destinationDatabase')
44 
45  def logDbFileName( self ):
46  return self.md.get('logDbFileName')
47 
48  def records( self ):
49  return self.md.get('records')
50 
51  def dumpMetadataForUpload( self, inputtag, desttag, comment ):
52  uploadMd = {}
53  uploadMd['destinationDatabase'] = self.destinationDatabase()
54  tags = {}
55  tagInfo = {}
56  tags[ desttag ] = tagInfo
57  uploadMd['destinationTags'] = tags
58  uploadMd['inputTag'] = inputtag
59  uploadMd['since'] = None
60  datelabel = self.datef.strftime(dateformatForLabel)
61  uploadMd['userText'] = '%s : %s' %(datelabel,comment)
62  with open( '%s.txt' %fileNameForDropBox, 'wb') as jf:
63  jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
64  jf.write('\n')
65 
66 def runO2O( cmsswdir, releasepath, release, arch, jobfilename, logfilename, *p ):
67  # first remove any existing metadata file...
68  if os.path.exists( '%s.db' %fileNameForDropBox ):
69  print "Removing files with name %s" %fileNameForDropBox
70  os.remove( '%s.db' %fileNameForDropBox )
71  if os.path.exists( '%s.txt' %fileNameForDropBox ):
72  os.remove( '%s.txt' %fileNameForDropBox )
73  command = 'export SCRAM_ARCH=%s;' %arch
74  command += 'CMSSWDIR=%s;' %cmsswdir
75  command += 'source ${CMSSWDIR}/cmsset_default.sh;'
76  command += 'cd %s/%s/src;' %(releasepath,release)
77  command += 'eval `scramv1 runtime -sh`;'
78  command += 'cd -;'
79  command += 'pwd;'
80  command += 'cmsRun %s ' %jobfilename
81  command += ' '.join(p)
82  command += ' 2>&1'
83  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
84  stdout_val = pipe.communicate()[0]
85  return stdout_val
86 
87 def upload_to_dropbox( backend ):
88 
89  md = CondMetaData(confFileName)
90  # check if the expected input file is there...
91  if not os.path.exists( dbFileForDropBox ):
92  print 'The input sqlite file has not been produced.'
93  return False
94  # first remove any existing metadata file...
95  if os.path.exists( '%s.txt' %fileNameForDropBox ):
96  os.remove( '%s.txt' %fileNameForDropBox )
97  try:
98  dropBox = upload_popcon.ConditionsUploader(upload_popcon.defaultHostname, upload_popcon.defaultUrlTemplate)
99  # Try to find the netrc entry
100  try:
101  (username, account, password) = netrc.netrc().authenticators(upload_popcon.defaultNetrcHost)
102  except Exception:
103  print 'Netrc entry "%s" not found.' %upload_popcon.defaultNetrcHost
104  return False
105  dropBox.signIn(username, password)
106  ret = True
107  for k,v in md.records().items():
108  destTag = v.get("destinationTag")
109  inputTag = v.get("sqliteTag")
110  if inputTag == None:
111  inputTag = destTag
112  comment = v.get("comment")
113  metadata = md.dumpMetadataForUpload( inputTag, destTag, comment )
114  ret &= dropBox.uploadFile(dbFileForDropBox, backend, upload_popcon.defaultTemporaryFile)
115  dropBox.signOut()
116  if ret:
117  print 'File %s successfully uploaded.' %dbFileForDropBox
118  return ret
119  except upload_popcon.HTTPError as e:
120  print e
121  return False
122 
123 def upload( cfileName, authPath ):
124  md = CondMetaData(cfileName)
125 
126  # check if the expected input file is there...
127  if not os.path.exists( dbFileForDropBox ):
128  print 'The input sqlite file has not been produced.'
129  return -1
130 
131  empty = True
132  try:
133  dbcon = sqlite3.connect( dbFileForDropBox )
134  dbcur = dbcon.cursor()
135  dbcur.execute('SELECT * FROM IOV')
136  rows = dbcur.fetchall()
137  for r in rows:
138  empty = False
139  dbcon.close()
140  if empty:
141  print 'The input sqlite file produced contains no data. The upload will be skipped.'
142  return 0
143  except Exception as e:
144  print 'Check on input data failed: %s' %str(e)
145  return -2
146 
147  # first remove any existing metadata file...
148  if os.path.exists( '%s.txt' %fileNameForDropBox ):
149  os.remove( '%s.txt' %fileNameForDropBox )
150 
151  # loop over the records to process...
152  ret = 0
153  for k,v in md.records().items():
154  destTag = v.get("destinationTag")
155  inputTag = v.get("sqliteTag")
156  if inputTag == None:
157  inputTag = destTag
158  comment = v.get("comment")
159  metadata = md.dumpMetadataForUpload( inputTag, destTag, comment )
160  uploadCommand = 'uploadConditions.py %s' %fileNameForDropBox
161  if not authPath is None:
162  uploadCommand += ' -a %s' %authPath
163  try:
164  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
165  stdout = pipe.communicate()[0]
166  print stdout
167  retCode = pipe.returncode
168  if retCode != 0:
169  # save a copy of the files in case of upload failure...
170  leafFolderName = md.datef.strftime(dateformatForFolder)
171  fileFolder = os.path.join( errorInUploadFileFolder, leafFolderName)
172  if not os.path.exists(fileFolder):
173  os.makedirs(fileFolder)
174  df= '%s.db' %fileNameForDropBox
175  mf= '%s.txt' %fileNameForDropBox
176  dataDestFile = os.path.join( fileFolder, df)
177  if not os.path.exists(dataDestFile):
178  shutil.copy2(df, dataDestFile)
179  shutil.copy2(mf,os.path.join(fileFolder,mf))
180  print "Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder)
181  ret |= retCode
182  except Exception as e:
183  ret |= 1
184  print e
185  return ret
186 
187 def run( jobfilename, authPath ):
188  fns = os.path.splitext( jobfilename )
189  confFile = '%s.json' %fns[0]
190  if os.path.exists( '%s.db' %fileNameForDropBox ):
191  print "Removing files with name %s" %fileNameForDropBox
192  os.remove( '%s.db' %fileNameForDropBox )
193  if os.path.exists( '%s.txt' %fileNameForDropBox ):
194  os.remove( '%s.txt' %fileNameForDropBox )
195  command = 'cmsRun %s ' %jobfilename
196  command += ' popconConfigFileName=%s' %confFile
197  command += ' 2>&1'
198  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
199  stdout = pipe.communicate()[0]
200  retCode = pipe.returncode
201  print stdout
202  print 'Return code is: %s' %retCode
203  if retCode!=0:
204  print 'O2O job failed. Skipping upload.'
205  return retCode
206  return upload( confFile, authPath )
static std::string join(char **cmd)
Definition: RemoteFile.cc:18