CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
Functions | Variables
popcon2dropbox Namespace Reference

Functions

def checkFile
 
def copy
 
def run
 
def saveFileForImportErrors
 
def upload
 

Variables

string auth_path_key = 'COND_AUTH_PATH'
 
tuple consoleHandler = logging.StreamHandler(sys.stdout)
 
string dateformatForFolder = "%Y-%m-%d-%H-%M-%S"
 
string dateformatForLabel = "%Y-%m-%d %H:%M:%S"
 
string errorInImportFileFolder = 'import_errors'
 
string fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"
 
list levStr = os.environ[messageLevelEnvVar]
 
tuple logFormatter = logging.Formatter(fmt_str)
 
tuple logger = logging.getLogger()
 
 logLevel = logging.INFO
 
string messageLevelEnvVar = 'POPCON_LOG_LEVEL'
 

Function Documentation

def popcon2dropbox.checkFile (   dbName)

Definition at line 32 of file popcon2dropbox.py.

References str.

Referenced by run().

32 
33 def checkFile( dbName ):
34  dbFileName = '%s.db' %dbName
35  # check if the expected input file is there...
36  # exit code < 0 => error
37  # exit code = 0 => skip
38  # exit code = 1 => import
39  if not os.path.exists( dbFileName ):
40  logger.error('The file generated by PopCon with the data to be imported %s has not been found.'%dbFileName )
41  return -1
42 
43  empty = True
44  try:
45  dbcon = sqlite3.connect( dbFileName )
46  dbcur = dbcon.cursor()
47  dbcur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='IOV'");
48  if dbcur.fetchone() is None:
49  logger.error('The condition database with the data to be imported has not been found in the file generated by PopCon: %s'%dbFileName )
50  return -1
51  dbcur.execute('SELECT * FROM IOV')
52  rows = dbcur.fetchall()
53  for r in rows:
54  empty = False
55  dbcon.close()
56  if empty:
57  logger.warning('The data set generated by PopCon contains no data to be imported. The import will be skipped.')
58  return 0
59  return 1
60  except Exception as e:
61  logger.error('Check on input data failed: %s' %str(e))
62  return -2
#define str(s)
def popcon2dropbox.copy (   args,
  dbName 
)

Definition at line 126 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

127 def copy( args, dbName ):
128  dbFileName = '%s.db' %dbName
129  destDb = args.destDb
130  destTag = args.destTag
131  comment = args.comment
132 
133  datef = datetime.now()
134  destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro" }
135  if destDb.lower() in destMap.keys():
136  destDb = destMap[destDb.lower()]
137  else:
138  if destDb.startswith('sqlite'):
139  destDb = destDb.split(':')[1]
140  else:
141  logger.error( 'Destination connection %s is not supported.' %destDb )
142  return
143  # run the copy
144  note = '"Importing data with O2O execution"'
145  commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
146  copyCommand = 'conddb %s' %commandOptions
147  logger.info( 'Executing command: %s' %copyCommand )
148  try:
149  pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
150  stdout = pipe.communicate()[0]
151  print(stdout)
152  retCode = pipe.returncode
153  if retCode != 0:
154  saveFileForImportErrors( datef, dbName )
155  ret = retCode
156  except Exception as e:
157  ret = 1
158  logger.error( str(e) )
159  return ret
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def saveFileForImportErrors
#define str(s)
def popcon2dropbox.run (   args)

Definition at line 160 of file popcon2dropbox.py.

References checkFile(), copy(), print(), and upload().

161 def run( args ):
162 
163  dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
164  dbFileName = '%s.db' %dbName
165 
166  if args.auth is not None and not args.auth=='':
167  if auth_path_key in os.environ:
168  logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
169  else:
170  logger.info("Setting the authentication path to %s in the environment." %args.auth)
171  os.environ[auth_path_key]=args.auth
172  if os.path.exists( '%s.db' %dbName ):
173  logger.info("Removing files with name %s" %dbName )
174  os.remove( '%s.db' %dbName )
175  if os.path.exists( '%s.txt' %dbName ):
176  os.remove( '%s.txt' %dbName )
177  command = 'cmsRun %s ' %args.job_file
178  command += ' targetFile=%s' %dbFileName
179  command += ' destinationDatabase=%s' %args.destDb
180  command += ' destinationTag=%s' %args.destTag
181  command += ' 2>&1'
182  logger.info( 'Executing command: %s' %command )
183  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
184  stdout = pipe.communicate()[0]
185  retCode = pipe.returncode
186  print(stdout)
187  logger.info('PopCon Analyzer return code is: %s' %retCode )
188  if retCode!=0:
189  logger.error( 'O2O job failed. Skipping upload.' )
190  return retCode
191 
192  ret = checkFile( dbName )
193  if ret > 0:
194  if args.copy:
195  ret = copy( args, dbName )
196  else:
197  ret = upload( args, dbName )
198  if ret >=0:
199  logger.info('Deleting local file %s.db' %dbName )
200  os.remove( '%s.db' %dbName )
201  return ret
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def popcon2dropbox.saveFileForImportErrors (   datef,
  dbName,
  withMetadata = False 
)

Definition at line 63 of file popcon2dropbox.py.

Referenced by copy(), and upload().

63 
64 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
65  # save a copy of the files in case of upload failure...
66  leafFolderName = datef.strftime(dateformatForFolder)
67  fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
68  if not os.path.exists(fileFolder):
69  os.makedirs(fileFolder)
70  df= '%s.db' %dbName
71  dataDestFile = os.path.join( fileFolder, df)
72  if not os.path.exists(dataDestFile):
73  shutil.copy2(df, dataDestFile)
74  if withMetadata:
75  mf= '%s.txt' %dbName
76  metadataDestFile = os.path.join( fileFolder, mf )
77  if not os.path.exists(metadataDestFile):
78  shutil.copy2(df, metadataDestFile)
79  logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
def saveFileForImportErrors
def popcon2dropbox.upload (   args,
  dbName 
)

Definition at line 80 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

80 
81 def upload( args, dbName ):
82  destDb = args.destDb
83  destTag = args.destTag
84  comment = args.comment
85 
86  datef = datetime.now()
87 
88  # first remove any existing metadata file...
89  if os.path.exists( '%s.txt' %dbName ):
90  logger.debug('Removing already existing file %s' %dbName)
91  os.remove( '%s.txt' %dbName )
92 
93  # dump Metadata for the Upload
94  uploadMd = {}
95  uploadMd['destinationDatabase'] = destDb
96  tags = {}
97  tagInfo = {}
98  tags[ destTag ] = tagInfo
99  uploadMd['destinationTags'] = tags
100  uploadMd['inputTag'] = destTag
101  uploadMd['since'] = None
102  datelabel = datef.strftime(dateformatForLabel)
103  commentStr = ''
104  if not comment is None:
105  commentStr = comment
106  uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
107  with open( '%s.txt' %dbName, 'wb') as jf:
108  jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
109  jf.write('\n')
110 
111  # run the upload
112  uploadCommand = 'uploadConditions.py %s' %dbName
113  ret = 0
114  try:
115  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
116  stdout = pipe.communicate()[0]
117  print(stdout)
118  retCode = pipe.returncode
119  if retCode != 0:
120  saveFileForImportErrors( datef, dbName, True )
121  ret |= retCode
122  except Exception as e:
123  ret |= 1
124  logger.error(str(e))
125  return ret
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
def saveFileForImportErrors
#define str(s)

Variable Documentation

string popcon2dropbox.auth_path_key = 'COND_AUTH_PATH'

Definition at line 16 of file popcon2dropbox.py.

tuple popcon2dropbox.consoleHandler = logging.StreamHandler(sys.stdout)

Definition at line 28 of file popcon2dropbox.py.

string popcon2dropbox.dateformatForFolder = "%Y-%m-%d-%H-%M-%S"

Definition at line 13 of file popcon2dropbox.py.

string popcon2dropbox.dateformatForLabel = "%Y-%m-%d %H:%M:%S"

Definition at line 14 of file popcon2dropbox.py.

string popcon2dropbox.errorInImportFileFolder = 'import_errors'

Definition at line 12 of file popcon2dropbox.py.

string popcon2dropbox.fmt_str = "[%(asctime)s] %(levelname)s: %(message)s"

Definition at line 19 of file popcon2dropbox.py.

list popcon2dropbox.levStr = os.environ[messageLevelEnvVar]

Definition at line 22 of file popcon2dropbox.py.

tuple popcon2dropbox.logFormatter = logging.Formatter(fmt_str)

Definition at line 25 of file popcon2dropbox.py.

tuple popcon2dropbox.logger = logging.getLogger()

Definition at line 26 of file popcon2dropbox.py.

popcon2dropbox.logLevel = logging.INFO

Definition at line 20 of file popcon2dropbox.py.

Referenced by edm::storage::DavixFile.configureDavixLogLevel().

string popcon2dropbox.messageLevelEnvVar = 'POPCON_LOG_LEVEL'

Definition at line 18 of file popcon2dropbox.py.