CMS 3D CMS Logo

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
uploadConditions.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 '''Script that uploads to the new CMS conditions uploader.
3 Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
4 '''
5 
6 __author__ = 'Andreas Pfeiffer'
7 __copyright__ = 'Copyright 2015, CERN CMS'
8 __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
9 __license__ = 'Unknown'
10 __maintainer__ = 'Andreas Pfeiffer'
11 __email__ = 'andreas.pfeiffer@cern.ch'
12 __version__ = 1
13 
14 
15 import os
16 import sys
17 import optparse
18 import hashlib
19 import tarfile
20 import netrc
21 import getpass
22 import errno
23 import sqlite3
24 import json
25 import tempfile
26 
27 defaultBackend = 'online'
28 defaultHostname = 'cms-conddb-prod.cern.ch'
29 defaultDevHostname = 'cms-conddb-dev.cern.ch'
30 defaultUrlTemplate = 'https://%s/cmsDbUpload/'
31 defaultTemporaryFile = 'upload.tar.bz2'
32 defaultNetrcHost = 'ConditionUploader'
33 defaultWorkflow = 'offline'
34 
35 # common/http.py start (plus the "# Try to extract..." section bit)
36 import time
37 import logging
38 import cStringIO
39 
40 import pycurl
41 import socket
42 import copy
43 
44 def getInput(default, prompt = ''):
45  '''Like raw_input() but with a default and automatic strip().
46  '''
47 
48  answer = raw_input(prompt)
49  if answer:
50  return answer.strip()
51 
52  return default.strip()
53 
54 
55 def getInputWorkflow(prompt = ''):
56  '''Like getInput() but tailored to get target workflows (synchronization options).
57  '''
58 
59  while True:
60  workflow = getInput(defaultWorkflow, prompt)
61 
62  if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
63  return workflow
64 
65  logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
66 
67 
68 def getInputChoose(optionsList, default, prompt = ''):
69  '''Makes the user choose from a list of options.
70  '''
71 
72  while True:
73  index = getInput(default, prompt)
74 
75  try:
76  return optionsList[int(index)]
77  except ValueError:
78  logging.error('Please specify an index of the list (i.e. integer).')
79  except IndexError:
80  logging.error('The index you provided is not in the given list.')
81 
82 
83 def getInputRepeat(prompt = ''):
84  '''Like raw_input() but repeats if nothing is provided and automatic strip().
85  '''
86 
87  while True:
88  answer = raw_input(prompt)
89  if answer:
90  return answer.strip()
91 
92  logging.error('You need to provide a value.')
93 
94 
95 def runWizard(basename, dataFilename, metadataFilename):
96  while True:
97  print '''\nWizard for metadata for %s
98 
99 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
100 
101  # Try to get the available inputTags
102  try:
103  dataConnection = sqlite3.connect(dataFilename)
104  dataCursor = dataConnection.cursor()
105  dataCursor.execute('select name from sqlite_master where type == "table"')
106  tables = set(zip(*dataCursor.fetchall())[0])
107 
108  # only conddb V2 supported...
109  if 'TAG' in tables:
110  dataCursor.execute('select NAME from TAG')
111  # In any other case, do not try to get the inputTags
112  else:
113  raise Exception()
114 
115  inputTags = dataCursor.fetchall()
116  if len(inputTags) == 0:
117  raise Exception()
118  inputTags = zip(*inputTags)[0]
119 
120  except Exception:
121  inputTags = []
122 
123  if len(inputTags) == 0:
124  print '\nI could not find any input tag in your data file, but you can still specify one manually.'
125 
126  inputTag = getInputRepeat(
127  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
128 
129  else:
130  print '\nI found the following input tags in your SQLite data file:'
131  for (index, inputTag) in enumerate(inputTags):
132  print ' %s) %s' % (index, inputTag)
133 
134  inputTag = getInputChoose(inputTags, '0',
135  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
136 
137  destinationDatabase = ''
138  ntry = 0
139  while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
140  if ntry==0:
141  inputMessage = \
142  '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
143  elif ntry==1:
144  inputMessage = \
145  '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
146 \ndestinationDatabase: '
147  else:
148  raise Exception('No valid destination chosen. Bailing out...')
149  destinationDatabase = getInputRepeat(inputMessage)
150  ntry += 1
151 
152  while True:
153  since = getInput('',
154  '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
155  if not since:
156  since = None
157  break
158  else:
159  try:
160  since = int(since)
161  break
162  except ValueError:
163  logging.error('The since value has to be an integer or empty (null).')
164 
165  userText = getInput('',
166  '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
167 
168  destinationTags = {}
169  while True:
170  destinationTag = getInput('',
171  '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
172  if not destinationTag:
173  if len(destinationTags) == 0:
174  logging.error('There must be at least one destination tag.')
175  continue
176  break
177 
178  if destinationTag in destinationTags:
179  logging.warning(
180  'You already added this destination tag. Overwriting the previous one with this new one.')
181 
182  destinationTags[destinationTag] = {
183  }
184 
185  metadata = {
186  'destinationDatabase': destinationDatabase,
187  'destinationTags': destinationTags,
188  'inputTag': inputTag,
189  'since': since,
190  'userText': userText,
191  }
192 
193  metadata = json.dumps(metadata, sort_keys=True, indent=4)
194  print '\nThis is the generated metadata:\n%s' % metadata
195 
196  if getInput('n',
197  '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
198  break
199  logging.info('Saving generated metadata in %s...', metadataFilename)
200  with open(metadataFilename, 'wb') as metadataFile:
201  metadataFile.write(metadata)
202 
204  '''A common HTTP exception.
205 
206  self.code is the response HTTP code as an integer.
207  self.response is the response body (i.e. page).
208  '''
209 
210  def __init__(self, code, response):
211  self.code = code
212  self.response = response
213 
214  # Try to extract the error message if possible (i.e. known error page format)
215  try:
216  self.args = (response.split('<p>')[1].split('</p>')[0], )
217  except Exception:
218  self.args = (self.response, )
219 
220 
221 CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
222 
223 class HTTP(object):
224  '''Class used for querying URLs using the HTTP protocol.
225  '''
226 
227  retryCodes = frozenset([502, 503])
228 
229  def __init__(self):
230  self.setBaseUrl()
231  self.setRetries()
232 
233  self.curl = pycurl.Curl()
234  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
235 
236  #-toDo: make sure we have the right options set here to use ssl
237  #-review(2015-09-25): check and see - action: AP
238  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
239  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
240  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
241 
242  self.baseUrl = None
243 
244  self.token = None
245 
246  def getCookies(self):
247  '''Returns the list of cookies.
248  '''
249  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
250 
251  def discardCookies(self):
252  '''Discards cookies.
253  '''
254  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
255 
256 
257  def setBaseUrl(self, baseUrl = ''):
258  '''Allows to set a base URL which will be prefixed to all the URLs
259  that will be queried later.
260  '''
261  self.baseUrl = baseUrl
262 
263 
264  def setProxy(self, proxy = ''):
265  '''Allows to set a proxy.
266  '''
267  self.curl.setopt(self.curl.PROXY, proxy)
268 
269 
270  def setTimeout(self, timeout = 0):
271  '''Allows to set a timeout.
272  '''
273  self.curl.setopt(self.curl.TIMEOUT, timeout)
274 
275 
276  def setRetries(self, retries = ()):
277  '''Allows to set retries.
278 
279  The retries are a sequence of the seconds to wait per retry.
280 
281  The retries are done on:
282  * PyCurl errors (includes network problems, e.g. not being able
283  to connect to the host).
284  * 502 Bad Gateway (for the moment, to avoid temporary
285  Apache-CherryPy issues).
286  * 503 Service Temporarily Unavailable (for when we update
287  the frontends).
288  '''
289  self.retries = retries
290 
291  def getToken(self, username, password):
292 
293  url = self.baseUrl + 'token'
294 
295  self.curl.setopt(pycurl.URL, url)
296  self.curl.setopt(pycurl.VERBOSE, 0)
297 
298  #-toDo: check if/why these are needed ...
299  #-ap: hmm ...
300  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
301  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
302  #-end hmmm ...
303  #-review(2015-09-25): check and see - action: AP
304 
305 
306  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
307  # self.curl.setopt( self.curl.POST, {})
308  self.curl.setopt(self.curl.HTTPGET, 0)
309 
310  response = cStringIO.StringIO()
311  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
312  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
313 
314  logging.debug('going to connect to server at: %s' % url )
315 
316  self.curl.perform()
317  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
318  logging.debug('got: %s ', str(code))
319 
320  try:
321  self.token = json.loads( response.getvalue() )['token']
322  except Exception as e:
323  logging.error('http::getToken> got error from server: %s ', str(e) )
324  if 'No JSON object could be decoded' in str(e):
325  return None
326  logging.error("error getting token: %s", str(e))
327  return None
328 
329  logging.debug('token: %s', self.token)
330  logging.debug('returning: %s', response.getvalue())
331 
332  return response.getvalue()
333 
334  def query(self, url, data = None, files = None, keepCookies = True):
335  '''Queries a URL, optionally with some data (dictionary).
336 
337  If no data is specified, a GET request will be used.
338  If some data is specified, a POST request will be used.
339 
340  If files is specified, it must be a dictionary like data but
341  the values are filenames.
342 
343  By default, cookies are kept in-between requests.
344 
345  A HTTPError exception is raised if the response's HTTP code is not 200.
346  '''
347 
348  if not keepCookies:
349  self.discardCookies()
350 
351  url = self.baseUrl + url
352 
353  # make sure the logs are safe ... at least somewhat :)
354  data4log = copy.copy(data)
355  if data4log:
356  if 'password' in data4log.keys():
357  data4log['password'] = '*'
358 
359  retries = [0] + list(self.retries)
360 
361  while True:
362  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
363 
364  time.sleep(retries.pop(0))
365 
366  try:
367  self.curl.setopt(self.curl.URL, url)
368  self.curl.setopt(self.curl.HTTPGET, 1)
369 
370  # from now on we use the token we got from the login
371  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
372  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
373 
374  if data is not None or files is not None:
375  # If there is data or files to send, use a POST request
376 
377  finalData = {}
378 
379  if data is not None:
380  finalData.update(data)
381 
382  if files is not None:
383  for (key, fileName) in files.items():
384  finalData[key] = (self.curl.FORM_FILE, fileName)
385  self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
386 
387  self.curl.setopt(pycurl.VERBOSE, 0)
388 
389  response = cStringIO.StringIO()
390  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
391  self.curl.perform()
392 
393  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
394 
395  if code in self.retryCodes and len(retries) > 0:
396  logging.debug('Retrying since we got the %s error code...', code)
397  continue
398 
399  if code != 200:
400  raise HTTPError(code, response.getvalue())
401 
402  return response.getvalue()
403 
404  except pycurl.error as e:
405  if len(retries) == 0:
406  raise e
407  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
408 
409 # common/http.py end
410 
411 def addToTarFile(tarFile, fileobj, arcname):
412  tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
413  tarInfo.mode = 0o400
414  tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
415  tarInfo.uname = tarInfo.gname = 'root'
416  tarFile.addfile(tarInfo, fileobj)
417 
418 class ConditionsUploader(object):
419  '''Upload conditions to the CMS conditions uploader service.
420  '''
421 
422  def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
423  self.hostname = hostname
424  self.urlTemplate = urlTemplate
425  self.userName = None
426  self.http = None
427  self.password = None
428 
429  def setHost( self, hostname ):
430  self.hostname = hostname
431 
432  def signIn(self, username, password):
433  ''' init the server.
434  '''
435  self.http = HTTP()
436  if socket.getfqdn().strip().endswith('.cms'):
437  self.http.setProxy('https://cmsproxy.cms:3128/')
438  self.http.setBaseUrl(self.urlTemplate % self.hostname)
439  '''Signs in the server.
440  '''
441 
442  logging.info('%s: Signing in user %s ...', self.hostname, username)
443  try:
444  self.token = self.http.getToken(username, password)
445  except Exception as e:
446  logging.error("Caught exception when trying to get token for user %s from %s: %s" % (username, self.hostname, str(e)) )
447  return False
448 
449  if not self.token:
450  logging.error("could not get token for user %s from %s" % (username, self.hostname) )
451  return False
452 
453  logging.debug( "got: '%s'", str(self.token) )
454  self.userName = username
455  self.password = password
456  return True
457 
458  def signInAgain(self):
459  return self.signIn( self.userName, self.password )
460 
461  def signOut(self):
462  '''Signs out the server.
463  '''
464 
465  logging.info('%s: Signing out...', self.hostname)
466  # self.http.query('logout')
467  self.token = None
468 
469 
470  def _checkForUpdates(self):
471  '''Updates this script, if a new version is found.
472  '''
473 
474  logging.debug('%s: Checking if a newer version of this script is available ...', self.hostname)
475  version = int(self.http.query('getUploadScriptVersion'))
476 
477  if version <= __version__:
478  logging.debug('%s: Script is up-to-date.', self.hostname)
479  return
480 
481  logging.info('%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.hostname, version, __version__)
482 
483  uploadScript = self.http.query('getUploadScript')
484 
485  self.signOut()
486 
487  logging.info('%s: ... saving the new version ...', self.hostname)
488  with open(sys.argv[0], 'wb') as f:
489  f.write(uploadScript)
490 
491  logging.info('%s: ... executing the new version...', self.hostname)
492  os.execl(sys.executable, *([sys.executable] + sys.argv))
493 
494 
495  def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
496  '''Uploads a file to the dropBox.
497 
498  The filename can be without extension, with .db or with .txt extension.
499  It will be stripped and then both .db and .txt files are used.
500  '''
501 
502  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
503  basename = os.path.basename(basepath)
504 
505  logging.debug('%s: %s: Creating tar file for upload ...', self.hostname, basename)
506 
507  try:
508  tarFile = tarfile.open(temporaryFile, 'w:bz2')
509 
510  with open('%s.db' % basepath, 'rb') as data:
511  addToTarFile(tarFile, data, 'data.db')
512  except Exception as e:
513  msg = 'Error when creating tar file. \n'
514  msg += 'Please check that you have write access to the directory you are running,\n'
515  msg += 'and that you have enough space on this disk (df -h .)\n'
516  logging.error(msg)
517  raise Exception(msg)
518 
519  with tempfile.NamedTemporaryFile() as metadata:
520  with open('%s.txt' % basepath, 'rb') as originalMetadata:
521  json.dump(json.load(originalMetadata), metadata, sort_keys = True, indent = 4)
522 
523  metadata.seek(0)
524  addToTarFile(tarFile, metadata, 'metadata.txt')
525 
526  tarFile.close()
527 
528  logging.debug('%s: %s: Calculating hash...', self.hostname, basename)
529 
530  fileHash = hashlib.sha1()
531  with open(temporaryFile, 'rb') as f:
532  while True:
533  data = f.read(4 * 1024 * 1024)
534  if not data:
535  break
536  fileHash.update(data)
537 
538  fileHash = fileHash.hexdigest()
539  fileInfo = os.stat(temporaryFile)
540  fileSize = fileInfo.st_size
541 
542  logging.debug('%s: %s: Hash: %s', self.hostname, basename, fileHash)
543 
544  logging.info('%s: %s: Uploading file (%s, size %s) to the %s backend...', self.hostname, basename, fileHash, fileSize, backend)
545  os.rename(temporaryFile, fileHash)
546  try:
547  ret = self.http.query('uploadFile',
548  {
549  'backend': backend,
550  'fileName': basename,
551  'userName': self.userName,
552  },
553  files = {
554  'uploadedFile': fileHash,
555  }
556  )
557  except Exception as e:
558  logging.error('Error from uploading: %s' % str(e))
559  ret = json.dumps( { "status": -1, "upload" : { 'itemStatus' : { basename : {'status':'failed', 'info':str(e)}}}, "error" : str(e)} )
560 
561  os.unlink(fileHash)
562 
563  statusInfo = json.loads(ret)['upload']
564  logging.debug( 'upload returned: %s', statusInfo )
565 
566  okTags = []
567  skippedTags = []
568  failedTags = []
569  for tag, info in statusInfo['itemStatus'].items():
570  logging.debug('checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=True)) )
571  if 'ok' in info['status'].lower() :
572  okTags.append( tag )
573  logging.info('tag %s successfully uploaded', tag)
574  if 'skip' in info['status'].lower() :
575  skippedTags.append( tag )
576  logging.warning('found tag %s to be skipped. reason: \n ... \t%s ', tag, info['info'])
577  if 'fail' in info['status'].lower() :
578  failedTags.append( tag )
579  logging.error('found tag %s failed to upload. reason: \n ... \t%s ', tag, info['info'])
580 
581  if len(okTags) > 0: logging.info ("tags sucessfully uploaded: %s ", str(okTags) )
582  if len(skippedTags) > 0: logging.warning("tags SKIPped to upload : %s ", str(skippedTags) )
583  if len(failedTags) > 0: logging.error ("tags FAILed to upload : %s ", str(failedTags) )
584 
585  fileLogURL = 'https://%s/logs/dropBox/getFileLog?fileHash=%s'
586  logging.info('file log at: %s', fileLogURL % (self.hostname,fileHash))
587 
588  return len(okTags)>0
589 
590 def authenticateUser(dropBox, options):
591 
592  try:
593  # Try to find the netrc entry
594  (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
595  except Exception:
596  # netrc entry not found, ask for the username and password
597  logging.info(
598  'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
599  options.netrcHost)
600 
601  # Try to get a default username
602  defaultUsername = getpass.getuser()
603  if defaultUsername is None:
604  defaultUsername = '(not found)'
605 
606  username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
607  password = getpass.getpass('Password: ')
608 
609  # Now we have a username and password, authenticate with them
610  return dropBox.signIn(username, password)
611 
612 
613 def uploadAllFiles(options, arguments):
614 
615  ret = {}
616  ret['status'] = 0
617 
618  # Check that we can read the data and metadata files
619  # If the metadata file does not exist, start the wizard
620  for filename in arguments:
621  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
622  basename = os.path.basename(basepath)
623  dataFilename = '%s.db' % basepath
624  metadataFilename = '%s.txt' % basepath
625 
626  logging.info('Checking %s...', basename)
627 
628  # Data file
629  try:
630  with open(dataFilename, 'rb') as dataFile:
631  pass
632  except IOError as e:
633  errMsg = 'Impossible to open SQLite data file %s' %dataFilename
634  logging.error( errMsg )
635  ret['status'] = -3
636  ret['error'] = errMsg
637  return ret
638 
639  # Check the data file
640  empty = True
641  try:
642  dbcon = sqlite3.connect( dataFilename )
643  dbcur = dbcon.cursor()
644  dbcur.execute('SELECT * FROM IOV')
645  rows = dbcur.fetchall()
646  for r in rows:
647  empty = False
648  dbcon.close()
649  if empty:
650  errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
651  logging.error( errMsg )
652  ret['status'] = -4
653  ret['error'] = errMsg
654  return ret
655  except Exception as e:
656  errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
657  logging.error( errMsg )
658  ret['status'] = -5
659  ret['error'] = errMsg
660  return ret
661 
662  # Metadata file
663  try:
664  with open(metadataFilename, 'rb') as metadataFile:
665  pass
666  except IOError as e:
667  if e.errno != errno.ENOENT:
668  errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
669  logging.error( errMsg )
670  ret['status'] = -4
671  ret['error'] = errMsg
672  return ret
673 
674  if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
675  errMsg = 'Metadata file %s does not exist' %metadataFilename
676  logging.error( errMsg )
677  ret['status'] = -5
678  ret['error'] = errMsg
679  return ret
680  # Wizard
681  runWizard(basename, dataFilename, metadataFilename)
682 
683  # Upload files
684  try:
685  dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
686 
687  # Authentication
688  if not authenticateUser(dropBox, options):
689  logging.error("Error authenticating user. Aborting.")
690  return { 'status' : -2, 'error' : "Error authenticating user. Aborting." }
691 
692  # At this point we must be authenticated
693  dropBox._checkForUpdates()
694 
695  results = {}
696  for filename in arguments:
697  backend = options.backend
698  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
699  metadataFilename = '%s.txt' % basepath
700  with open(metadataFilename, 'rb') as metadataFile:
701  metadata = json.load( metadataFile )
702  # When dest db = prep the hostname has to be set to dev.
703  forceHost = False
704  destDb = metadata['destinationDatabase']
705  if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
706  if destDb.startswith('oracle://cms_orcoff_prep'):
707  dropBox.setHost( defaultDevHostname )
708  dropBox.signInAgain()
709  forceHost = True
710  results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
711  if forceHost:
712  # set back the hostname to the original global setting
713  dropBox.setHost( options.hostname )
714  dropBox.signInAgain()
715  else:
716  results[filename] = False
717  logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
718  if not results[filename]:
719  if ret['status']<0:
720  ret['status'] = 0
721  ret['status'] += 1
722  ret['files'] = results
723  logging.debug("all files processed, logging out now.")
724 
725  dropBox.signOut()
726 
727  except HTTPError as e:
728  logging.error('got HTTP error: %s', str(e))
729  return { 'status' : -1, 'error' : str(e) }
730 
731  return ret
732 
733 def uploadTier0Files(filenames, username, password, cookieFileName = None):
734  '''Uploads a bunch of files coming from Tier0.
735  This has the following requirements:
736  * Username/Password based authentication.
737  * Uses the online backend.
738  * Ignores errors related to the upload/content (e.g. duplicated file).
739  '''
740 
741  dropBox = ConditionsUploader()
742 
743  dropBox.signIn(username, password)
744 
745  for filename in filenames:
746  try:
747  result = dropBox.uploadFile(filename, backend = 'test')
748  except HTTPError as e:
749  if e.code == 400:
750  # 400 Bad Request: This is an exception related to the upload
751  # being wrong for some reason (e.g. duplicated file).
752  # Since for Tier0 this is not an issue, continue
753  logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
754  continue
755 
756  # In any other case, re-raise.
757  raise
758 
759  #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
760  # if we flag as failed any retry would not help and would result in the same error (e.g.
761  # when a file with an identical hash is uploaded again)
762  #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
763 
764  if not result: # dropbox reported an error when uploading, do not retry.
765  logging.error('Error from dropbox, upload-related, skipping.')
766  continue
767 
768  dropBox.signOut()
769 
770 
771 def main():
772  '''Entry point.
773  '''
774 
775  parser = optparse.OptionParser(usage =
776  'Usage: %prog [options] <file> [<file> ...]\n'
777  )
778 
779  parser.add_option('-d', '--debug',
780  dest = 'debug',
781  action="store_true",
782  default = False,
783  help = 'Switch on printing debug information. Default: %default',
784  )
785 
786  parser.add_option('-b', '--backend',
787  dest = 'backend',
788  default = defaultBackend,
789  help = 'dropBox\'s backend to upload to. Default: %default',
790  )
791 
792  parser.add_option('-H', '--hostname',
793  dest = 'hostname',
794  default = defaultHostname,
795  help = 'dropBox\'s hostname. Default: %default',
796  )
797 
798  parser.add_option('-u', '--urlTemplate',
799  dest = 'urlTemplate',
800  default = defaultUrlTemplate,
801  help = 'dropBox\'s URL template. Default: %default',
802  )
803 
804  parser.add_option('-f', '--temporaryFile',
805  dest = 'temporaryFile',
806  default = defaultTemporaryFile,
807  help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
808  )
809 
810  parser.add_option('-n', '--netrcHost',
811  dest = 'netrcHost',
812  default = defaultNetrcHost,
813  help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
814  )
815 
816  (options, arguments) = parser.parse_args()
817 
818  if len(arguments) < 1:
819  parser.print_help()
820  return -2
821 
822  logLevel = logging.INFO
823  if options.debug:
824  logLevel = logging.DEBUG
825  logging.basicConfig(
826  format = '[%(asctime)s] %(levelname)s: %(message)s',
827  level = logLevel,
828  )
829 
830  results = uploadAllFiles(options, arguments)
831 
832  if not results.has_key('status'):
833  print 'Unexpected error.'
834  return -1
835  ret = results['status']
836  print results
837  print "upload ended with code: %s" %ret
838  #for hash, res in results.items():
839  # print "\t %s : %s " % (hash, str(res))
840  return ret
841 
843 
844  global defaultNetrcHost
845 
846  (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
847 
848  filenames = ['testFiles/localSqlite-top2']
849 
850  uploadTier0Files(filenames, username, password, cookieFileName = None)
851 
852 
853 if __name__ == '__main__':
854 
855  sys.exit(main())
856  # testTier0Upload()
Definition: main.py:1
double split
Definition: MVATrainer.cc:139
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run