CMS 3D CMS Logo

All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
Functions
popcon2dropbox Namespace Reference

Functions

def checkFile (dbName)
 
def copy (args, dbName)
 
def run (args)
 
def saveFileForImportErrors (datef, dbName, withMetadata=False)
 
def upload (args, dbName)
 

Function Documentation

◆ checkFile()

def popcon2dropbox.checkFile (   dbName)

Definition at line 32 of file popcon2dropbox.py.

References str.

Referenced by run().

32 def checkFile( dbName ):
33  dbFileName = '%s.db' %dbName
34  # check if the expected input file is there...
35  # exit code < 0 => error
36  # exit code = 0 => skip
37  # exit code = 1 => import
38  if not os.path.exists( dbFileName ):
39  logger.error('The file generated by PopCon with the data to be imported %s has not been found.'%dbFileName )
40  return -1
41 
42  empty = True
43  try:
44  dbcon = sqlite3.connect( dbFileName )
45  dbcur = dbcon.cursor()
46  dbcur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='IOV'");
47  if dbcur.fetchone() is None:
48  logger.error('The condition database with the data to be imported has not been found in the file generated by PopCon: %s'%dbFileName )
49  return -1
50  dbcur.execute('SELECT * FROM IOV')
51  rows = dbcur.fetchall()
52  for r in rows:
53  empty = False
54  dbcon.close()
55  if empty:
56  logger.warning('The data set generated by PopCon contains no data to be imported. The import will be skipped.')
57  return 0
58  return 1
59  except Exception as e:
60  logger.error('Check on input data failed: %s' %str(e))
61  return -2
62 
def checkFile(dbName)
#define str(s)

◆ copy()

def popcon2dropbox.copy (   args,
  dbName 
)

Definition at line 126 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

126 def copy( args, dbName ):
127  dbFileName = '%s.db' %dbName
128  destDb = args.destDb
129  destTag = args.destTag
130  comment = args.comment
131 
132  datef = datetime.now()
133  destMap = { "oracle://cms_orcoff_prep/cms_conditions": "oradev", "oracle://cms_orcon_prod/cms_conditions": "onlineorapro" }
134  if destDb.lower() in destMap.keys():
135  destDb = destMap[destDb.lower()]
136  else:
137  if destDb.startswith('sqlite'):
138  destDb = destDb.split(':')[1]
139  else:
140  logger.error( 'Destination connection %s is not supported.' %destDb )
141  return
142  # run the copy
143  note = '"Importing data with O2O execution"'
144  commandOptions = '--force --yes --db %s copy %s %s --destdb %s --synchronize --note %s' %(dbFileName,destTag,destTag,destDb,note)
145  copyCommand = 'conddb %s' %commandOptions
146  logger.info( 'Executing command: %s' %copyCommand )
147  try:
148  pipe = subprocess.Popen( copyCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
149  stdout = pipe.communicate()[0]
150  print(stdout)
151  retCode = pipe.returncode
152  if retCode != 0:
153  saveFileForImportErrors( datef, dbName )
154  ret = retCode
155  except Exception as e:
156  ret = 1
157  logger.error( str(e) )
158  return ret
159 
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def copy(args, dbName)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)

◆ run()

def popcon2dropbox.run (   args)

Definition at line 160 of file popcon2dropbox.py.

References checkFile(), copy(), print(), and upload().

160 def run( args ):
161 
162  dbName = datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S-%f')
163  dbFileName = '%s.db' %dbName
164 
165  if args.auth is not None and not args.auth=='':
166  if auth_path_key in os.environ:
167  logger.warning("Cannot set authentication path to %s in the environment, since it is already set." %args.auth)
168  else:
169  logger.info("Setting the authentication path to %s in the environment." %args.auth)
170  os.environ[auth_path_key]=args.auth
171  if os.path.exists( '%s.db' %dbName ):
172  logger.info("Removing files with name %s" %dbName )
173  os.remove( '%s.db' %dbName )
174  if os.path.exists( '%s.txt' %dbName ):
175  os.remove( '%s.txt' %dbName )
176  command = 'cmsRun %s ' %args.job_file
177  command += ' targetFile=%s' %dbFileName
178  command += ' destinationDatabase=%s' %args.destDb
179  command += ' destinationTag=%s' %args.destTag
180  command += ' 2>&1'
181  logger.info( 'Executing command: %s' %command )
182  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
183  stdout = pipe.communicate()[0]
184  retCode = pipe.returncode
185  print(stdout)
186  logger.info('PopCon Analyzer return code is: %s' %retCode )
187  if retCode!=0:
188  logger.error( 'O2O job failed. Skipping upload.' )
189  return retCode
190 
191  ret = checkFile( dbName )
192  if ret > 0:
193  if args.copy:
194  ret = copy( args, dbName )
195  else:
196  ret = upload( args, dbName )
197  if ret >=0:
198  logger.info('Deleting local file %s.db' %dbName )
199  os.remove( '%s.db' %dbName )
200  return ret
201 
def copy(args, dbName)
def upload(args, dbName)
def checkFile(dbName)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47

◆ saveFileForImportErrors()

def popcon2dropbox.saveFileForImportErrors (   datef,
  dbName,
  withMetadata = False 
)

Definition at line 63 of file popcon2dropbox.py.

Referenced by copy(), and upload().

63 def saveFileForImportErrors( datef, dbName, withMetadata=False ):
64  # save a copy of the files in case of upload failure...
65  leafFolderName = datef.strftime(dateformatForFolder)
66  fileFolder = os.path.join( errorInImportFileFolder, leafFolderName)
67  if not os.path.exists(fileFolder):
68  os.makedirs(fileFolder)
69  df= '%s.db' %dbName
70  dataDestFile = os.path.join( fileFolder, df)
71  if not os.path.exists(dataDestFile):
72  shutil.copy2(df, dataDestFile)
73  if withMetadata:
74  mf= '%s.txt' %dbName
75  metadataDestFile = os.path.join( fileFolder, mf )
76  if not os.path.exists(metadataDestFile):
77  shutil.copy2(df, metadataDestFile)
78  logger.error("Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder))
79 
def saveFileForImportErrors(datef, dbName, withMetadata=False)

◆ upload()

def popcon2dropbox.upload (   args,
  dbName 
)

Definition at line 80 of file popcon2dropbox.py.

References print(), saveFileForImportErrors(), and str.

Referenced by run().

80 def upload( args, dbName ):
81  destDb = args.destDb
82  destTag = args.destTag
83  comment = args.comment
84 
85  datef = datetime.now()
86 
87  # first remove any existing metadata file...
88  if os.path.exists( '%s.txt' %dbName ):
89  logger.debug('Removing already existing file %s' %dbName)
90  os.remove( '%s.txt' %dbName )
91 
92  # dump Metadata for the Upload
93  uploadMd = {}
94  uploadMd['destinationDatabase'] = destDb
95  tags = {}
96  tagInfo = {}
97  tags[ destTag ] = tagInfo
98  uploadMd['destinationTags'] = tags
99  uploadMd['inputTag'] = destTag
100  uploadMd['since'] = None
101  datelabel = datef.strftime(dateformatForLabel)
102  commentStr = ''
103  if not comment is None:
104  commentStr = comment
105  uploadMd['userText'] = '%s : %s' %(datelabel,commentStr)
106  with open( '%s.txt' %dbName, 'wb') as jf:
107  jf.write( json.dumps( uploadMd, sort_keys=True, indent = 2 ) )
108  jf.write('\n')
109 
110  # run the upload
111  uploadCommand = 'uploadConditions.py %s' %dbName
112  ret = 0
113  try:
114  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
115  stdout = pipe.communicate()[0]
116  print(stdout)
117  retCode = pipe.returncode
118  if retCode != 0:
119  saveFileForImportErrors( datef, dbName, True )
120  ret |= retCode
121  except Exception as e:
122  ret |= 1
123  logger.error(str(e))
124  return ret
125 
def saveFileForImportErrors(datef, dbName, withMetadata=False)
def upload(args, dbName)
void print(TMatrixD &m, const char *label=nullptr, bool mathematicaFormat=false)
Definition: Utilities.cc:47
#define str(s)