CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
popcon2dropbox.py
Go to the documentation of this file.
1 from __future__ import print_function
2 import subprocess
3 import json
4 import netrc
5 import sqlite3
6 import os
7 import sys
8 import shutil
9 import logging
10 from datetime import datetime
11 
12 errorInImportFileFolder = 'import_errors'
13 dateformatForFolder = "%Y-%m-%d-%H-%M-%S"
14 dateformatForLabel = "%Y-%m-%d %H:%M:%S"
15 
16 auth_path_key = 'COND_AUTH_PATH'
17 
18 messageLevelEnvVar = 'POPCON_LOG_LEVEL'
19 fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
20 logLevel = logging.INFO
21 if messageLevelEnvVar in os.environ:
22  levStr = os.environ[messageLevelEnvVar]
23  if levStr == 'DEBUG':
24  logLevel = logging.DEBUG
25 logFormatter = logging.Formatter(fmt_str)
26 logger = logging.getLogger()
27 logger.setLevel(logLevel)
28 consoleHandler = logging.StreamHandler(sys.stdout)
29 consoleHandler.setFormatter(logFormatter)
30 logger.addHandler(consoleHandler)
31 
32 def checkFile( dbName ):
33  dbFileName = '%s.db' %dbName
34  # check if the expected input file is there...
35  # exit code < 0 => error
36  # exit code = 0 => skip
37  # exit code = 1 => import
38  if not os.path.exists( dbFileName ):
39  logger.error('The file generated by PopCon with the data to be imported %s has not been found.'%dbFileName )
40  return -1
41 
42  empty = True
43  try:
44  dbcon = sqlite3.connect( dbFileName )
45  dbcur = dbcon.cursor()
46  dbcur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='IOV'");
47  if dbcur.fetchone() is None:
48  logger.error('The condition database with the data to be imported has not been found in the file generated by PopCon: %s'%dbFileName )
49  return -1
50  dbcur.execute('SELECT * FROM IOV')
51  rows = dbcur.fetchall()
52  for r in rows:
53  empty = False
54  dbcon.close()
55  if empty:
56  logger.warning('The data set generated by PopCon contains no data to be imported. The import will be skipped.')
57  return 0
58  return 1
59  except Exception as e:
60  logger.error('Check on input data failed: %s' %str(e))
61  return -2
62 
63 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
64  # save a copy of the files in case of upload failure...
65  leafFolderName = datef.strftime(dateformatForFolder)
66  fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
67  if not os.path.exists(fileFolder):
68  os.makedirs(fileFolder)
69  df= '%s.db' %dbName
70  dataDestFile = os.path.join( fileFolder, df)
71  if not os.path.exists(dataDestFile):
72  shutil.copy2(df, dataDestFile)
73  if withMetadata:
74  mf= '%s.txt' %dbName
75  metadataDestFile = os.path.join( fileFolder, mf )
76  if not os.path.exists(metadataDestFile):
77  shutil.copy2(df, metadataDestFile)
78  logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
79 
80 def upload( args, dbName ):
81  destDb = args.destDb
82  destTag = args.destTag
83  comment = args.comment
84 
85  datef = datetime.now()
86 
87  # first remove any existing metadata file...
88  if os.path.exists( '%s.txt' %dbName ):
89  logger.debug('Removing already existing file %s' %dbName)
90  os.remove( '%s.txt' %dbName )
91 
92  # dump Metadata for the Upload
93  uploadMd = {}
94  uploadMd['destinationDatabase'] = destDb
95  tags = {}
96  tagInfo = {}
97  tags[ destTag ] = tagInfo
98  uploadMd['destinationTags'] = tags
99  uploadMd['inputTag'] = destTag
100  uploadMd['since'] = None
101  datelabel = datef.strftime(dateformatForLabel)
102  commentStr = ''
103  if not comment is None:
104  commentStr = comment
105  uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
106  with open( '%s.txt' %dbName, 'wb') as jf:
107  jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
108  jf.write('\n')
109 
110  # run the upload
111  uploadCommand = 'uploadConditions.py %s' %dbName
112  ret = 0
113  try:
114  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
115  stdout = pipe.communicate()[0]
116  print(stdout)
117  retCode = pipe.returncode
118  if retCode != 0:
119  saveFileForImportErrors( datef, dbName, True )
120  ret |= retCode
121  except Exception as e:
122  ret |= 1
123  logger.error(str(e))
124  return ret
125 
126 def copy( args, dbName ):
127  dbFileName = '%s.db' %dbName
128  destDb = args.destDb
129  destTag = args.destTag
130  comment = args.comment
131 
132  datef = datetime.now()
133  destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro" }
134  if destDb.lower() in destMap.keys():
135  destDb = destMap[destDb.lower()]
136  else:
137  if destDb.startswith('sqlite'):
138  destDb = destDb.split(':')[1]
139  else:
140  logger.error( 'Destination connection %s is not supported.' %destDb )
141  return
142  # run the copy
143  note = '"Importing data with O2O execution"'
144  commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
145  copyCommand = 'conddb %s' %commandOptions
146  logger.info( 'Executing command: %s' %copyCommand )
147  try:
148  pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
149  stdout = pipe.communicate()[0]
150  print(stdout)
151  retCode = pipe.returncode
152  if retCode != 0:
153  saveFileForImportErrors( datef, dbName )
154  ret = retCode
155  except Exception as e:
156  ret = 1
157  logger.error( str(e) )
158  return ret
159 
160 def run( args ):
161 
162  dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
163  dbFileName = '%s.db' %dbName
164 
165  if args.auth is not None and not args.auth=='':
166  if auth_path_key in os.environ:
167  logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
168  else:
169  logger.info("Setting the authentication path to %s in the environment." %args.auth)
170  os.environ[auth_path_key]=args.auth
171  if os.path.exists( '%s.db' %dbName ):
172  logger.info("Removing files with name %s" %dbName )
173  os.remove( '%s.db' %dbName )
174  if os.path.exists( '%s.txt' %dbName ):
175  os.remove( '%s.txt' %dbName )
176  command = 'cmsRun %s ' %args.job_file
177  command += ' targetFile=%s' %dbFileName
178  command += ' destinationDatabase=%s' %args.destDb
179  command += ' destinationTag=%s' %args.destTag
180  command += ' 2>&1'
181  logger.info( 'Executing command: %s' %command )
182  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
183  stdout = pipe.communicate()[0]
184  retCode = pipe.returncode
185  print(stdout)
186  logger.info('PopCon Analyzer return code is: %s' %retCode )
187  if retCode!=0:
188  logger.error( 'O2O job failed. Skipping upload.' )
189  return retCode
190 
191  ret = checkFile( dbName )
192  if ret > 0:
193  if args.copy:
194  ret = copy( args, dbName )
195  else:
196  ret = upload( args, dbName )
197  if ret >=0:
198  logger.info('Deleting local file %s.db' %dbName )
199  os.remove( '%s.db' %dbName )
200  return ret
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def copy(args, dbName)
def upload(args, dbName)
def checkFile(dbName)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)