test
CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
popcon2dropbox Namespace Reference

Classes

class  CondMetaData
 

Functions

def run
 
def runO2O
 
def upload
 
def upload_to_dropbox
 

Variables

string confFileName = 'popcon2dropbox.json'
 
string dateformatForFolder = "%y-%m-%d-%H-%M-%S"
 
string dateformatForLabel = "%y-%m-%d %H:%M:%S"
 
string dbFileForDropBox = '%s.db'
 
string dbLogFile = '%s_log.db'
 
string errorInUploadFileFolder = 'upload_errors'
 
string fileNameForDropBox = 'input_for_dropbox'
 

Function Documentation

def popcon2dropbox.run (   jobfilename,
  authPath 
)

Definition at line 187 of file popcon2dropbox.py.

References upload().

188 def run( jobfilename, authPath ):
189  fns = os.path.splitext( jobfilename )
190  confFile = '%s.json' %fns[0]
191  if os.path.exists( '%s.db' %fileNameForDropBox ):
192  print "Removing files with name %s" %fileNameForDropBox
193  os.remove( '%s.db' %fileNameForDropBox )
194  if os.path.exists( '%s.txt' %fileNameForDropBox ):
195  os.remove( '%s.txt' %fileNameForDropBox )
196  command = 'cmsRun %s ' %jobfilename
197  command += ' popconConfigFileName=%s' %confFile
198  command += ' 2>&1'
199  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
200  stdout = pipe.communicate()[0]
201  retCode = pipe.returncode
202  print stdout
203  print 'Return code is: %s' %retCode
204  if retCode!=0:
205  print 'O2O job failed. Skipping upload.'
206  return retCode
207  return upload( confFile, authPath )
def popcon2dropbox.runO2O (   cmsswdir,
  releasepath,
  release,
  arch,
  jobfilename,
  logfilename,
  p 
)

Definition at line 66 of file popcon2dropbox.py.

References join().

66 
67 def runO2O( cmsswdir, releasepath, release, arch, jobfilename, logfilename, *p ):
68  # first remove any existing metadata file...
69  if os.path.exists( '%s.db' %fileNameForDropBox ):
70  print "Removing files with name %s" %fileNameForDropBox
71  os.remove( '%s.db' %fileNameForDropBox )
72  if os.path.exists( '%s.txt' %fileNameForDropBox ):
73  os.remove( '%s.txt' %fileNameForDropBox )
74  command = 'export SCRAM_ARCH=%s;' %arch
75  command += 'CMSSWDIR=%s;' %cmsswdir
76  command += 'source ${CMSSWDIR}/cmsset_default.sh;'
77  command += 'cd %s/%s/src;' %(releasepath,release)
78  command += 'eval `scramv1 runtime -sh`;'
79  command += 'cd -;'
80  command += 'pwd;'
81  command += 'cmsRun %s ' %jobfilename
82  command += ' '.join(p)
83  command += ' 2>&1'
84  pipe = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
85  stdout_val = pipe.communicate()[0]
86  return stdout_val
static std::string join(char **cmd)
Definition: RemoteFile.cc:18
def popcon2dropbox.upload (   cfileName,
  authPath 
)

Definition at line 123 of file popcon2dropbox.py.

References mps_monitormerge.items.

Referenced by run().

124 def upload( cfileName, authPath ):
125  md = CondMetaData(cfileName)
126 
127  # check if the expected input file is there...
128  if not os.path.exists( dbFileForDropBox ):
129  print 'The input sqlite file has not been produced.'
130  return -1
131 
132  empty = True
133  try:
134  dbcon = sqlite3.connect( dbFileForDropBox )
135  dbcur = dbcon.cursor()
136  dbcur.execute('SELECT * FROM IOV')
137  rows = dbcur.fetchall()
138  for r in rows:
139  empty = False
140  dbcon.close()
141  if empty:
142  print 'The input sqlite file produced contains no data. The upload will be skipped.'
143  return 0
144  except Exception as e:
145  print 'Check on input data failed: %s' %str(e)
146  return -2
147 
148  # first remove any existing metadata file...
149  if os.path.exists( '%s.txt' %fileNameForDropBox ):
150  os.remove( '%s.txt' %fileNameForDropBox )
151 
152  # loop over the records to process...
153  ret = 0
154  for k,v in md.records().items():
155  destTag = v.get("destinationTag")
156  inputTag = v.get("sqliteTag")
157  if inputTag == None:
158  inputTag = destTag
159  comment = v.get("comment")
160  metadata = md.dumpMetadataForUpload( inputTag, destTag, comment )
161  uploadCommand = 'uploadConditions.py %s' %fileNameForDropBox
162  if not authPath is None:
163  uploadCommand += ' -a %s' %authPath
164  try:
165  pipe = subprocess.Popen( uploadCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
166  stdout = pipe.communicate()[0]
167  print stdout
168  retCode = pipe.returncode
169  if retCode != 0:
170  # save a copy of the files in case of upload failure...
171  leafFolderName = md.datef.strftime(dateformatForFolder)
172  fileFolder = os.path.join( errorInUploadFileFolder, leafFolderName)
173  if not os.path.exists(fileFolder):
174  os.makedirs(fileFolder)
175  df= '%s.db' %fileNameForDropBox
176  mf= '%s.txt' %fileNameForDropBox
177  dataDestFile = os.path.join( fileFolder, df)
178  if not os.path.exists(dataDestFile):
179  shutil.copy2(df, dataDestFile)
180  shutil.copy2(mf,os.path.join(fileFolder,mf))
181  print "Upload failed. Data file and metadata saved in folder '%s'" %os.path.abspath(fileFolder)
182  ret |= retCode
183  except Exception as e:
184  ret |= 1
185  print e
186  return ret
def popcon2dropbox.upload_to_dropbox (   backend)

Definition at line 87 of file popcon2dropbox.py.

References mps_monitormerge.items.

87 
88 def upload_to_dropbox( backend ):
89 
90  md = CondMetaData(confFileName)
91  # check if the expected input file is there...
92  if not os.path.exists( dbFileForDropBox ):
93  print 'The input sqlite file has not been produced.'
94  return False
95  # first remove any existing metadata file...
96  if os.path.exists( '%s.txt' %fileNameForDropBox ):
97  os.remove( '%s.txt' %fileNameForDropBox )
98  try:
99  dropBox = upload_popcon.ConditionsUploader(upload_popcon.defaultHostname, upload_popcon.defaultUrlTemplate)
100  # Try to find the netrc entry
101  try:
102  (username, account, password) = netrc.netrc().authenticators(upload_popcon.defaultNetrcHost)
103  except Exception:
104  print 'Netrc entry "%s" not found.' %upload_popcon.defaultNetrcHost
105  return False
106  dropBox.signIn(username, password)
107  ret = True
108  for k,v in md.records().items():
109  destTag = v.get("destinationTag")
110  inputTag = v.get("sqliteTag")
111  if inputTag == None:
112  inputTag = destTag
113  comment = v.get("comment")
114  metadata = md.dumpMetadataForUpload( inputTag, destTag, comment )
115  ret &= dropBox.uploadFile(dbFileForDropBox, backend, upload_popcon.defaultTemporaryFile)
116  dropBox.signOut()
117  if ret:
118  print 'File %s successfully uploaded.' %dbFileForDropBox
119  return ret
120  except upload_popcon.HTTPError as e:
121  print e
122  return False

Variable Documentation

string popcon2dropbox.confFileName = 'popcon2dropbox.json'

Definition at line 9 of file popcon2dropbox.py.

string popcon2dropbox.dateformatForFolder = "%y-%m-%d-%H-%M-%S"

Definition at line 14 of file popcon2dropbox.py.

string popcon2dropbox.dateformatForLabel = "%y-%m-%d %H:%M:%S"

Definition at line 15 of file popcon2dropbox.py.

string popcon2dropbox.dbFileForDropBox = '%s.db'

Definition at line 11 of file popcon2dropbox.py.

string popcon2dropbox.dbLogFile = '%s_log.db'

Definition at line 12 of file popcon2dropbox.py.

string popcon2dropbox.errorInUploadFileFolder = 'upload_errors'

Definition at line 13 of file popcon2dropbox.py.

string popcon2dropbox.fileNameForDropBox = 'input_for_dropbox'

Definition at line 10 of file popcon2dropbox.py.