CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
uploadConditions Namespace Reference

Classes

class  ConditionsUploader
 
class  HTTP
 
class  HTTPError
 

Functions

def addToTarFile
 
def authenticateUser
 
def getInput
 
def getInputChoose
 
def getInputRepeat
 
def getInputWorkflow
 
def main
 
def runWizard
 
def testTier0Upload
 
def uploadAllFiles
 
def uploadTier0Files
 

Variables

string __author__ = 'Andreas Pfeiffer'
 
string __copyright__ = 'Copyright 2015, CERN CMS'
 
list __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'andreas.pfeiffer@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Andreas Pfeiffer'
 
int __version__ = 1
 
string CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
 
string defaultBackend = 'online'
 
string defaultDevHostname = 'cms-conddb-dev.cern.ch'
 
string defaultHostname = 'cms-conddb-prod.cern.ch'
 
string defaultNetrcHost = 'ConditionUploader'
 
string defaultTemporaryFile = 'upload.tar.bz2'
 
string defaultUrlTemplate = 'https://%s/cmsDbUpload/'
 
string defaultWorkflow = 'offline'
 

Function Documentation

def uploadConditions.addToTarFile (   tarFile,
  fileobj,
  arcname 
)

Definition at line 411 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.uploadFile().

411 
412 def addToTarFile(tarFile, fileobj, arcname):
413  tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
414  tarInfo.mode = 0o400
415  tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
416  tarInfo.uname = tarInfo.gname = 'root'
417  tarFile.addfile(tarInfo, fileobj)
def uploadConditions.authenticateUser (   dropBox,
  options 
)

Definition at line 590 of file uploadConditions.py.

References getInput().

Referenced by uploadAllFiles().

590 
591 def authenticateUser(dropBox, options):
592 
593  netrcPath = None
594  if options.authPath is not None:
595  netrcPath = os.path.join( options.authPath,'.netrc' )
596  try:
597  # Try to find the netrc entry
598  (username, account, password) = netrc.netrc( netrcPath ).authenticators(options.netrcHost)
599  except Exception:
600  # netrc entry not found, ask for the username and password
601  logging.info(
602  'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
603  options.netrcHost)
604 
605  # Try to get a default username
606  defaultUsername = getpass.getuser()
607  if defaultUsername is None:
608  defaultUsername = '(not found)'
609 
610  username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
611  password = getpass.getpass('Password: ')
612 
613  # Now we have a username and password, authenticate with them
614  return dropBox.signIn(username, password)
615 
def uploadConditions.getInput (   default,
  prompt = '' 
) 
Like raw_input() but with a default and automatic strip().

Definition at line 44 of file uploadConditions.py.

Referenced by authenticateUser(), getInputChoose(), getInputWorkflow(), runWizard(), and uploadAllFiles().

44 
45 def getInput(default, prompt = ''):
46  '''Like raw_input() but with a default and automatic strip().
47  '''
48 
49  answer = raw_input(prompt)
50  if answer:
51  return answer.strip()
52 
53  return default.strip()
54 
def uploadConditions.getInputChoose (   optionsList,
  default,
  prompt = '' 
) 
Makes the user choose from a list of options.

Definition at line 68 of file uploadConditions.py.

References getInput().

Referenced by runWizard().

68 
69 def getInputChoose(optionsList, default, prompt = ''):
70  '''Makes the user choose from a list of options.
71  '''
72 
73  while True:
74  index = getInput(default, prompt)
75 
76  try:
77  return optionsList[int(index)]
78  except ValueError:
79  logging.error('Please specify an index of the list (i.e. integer).')
80  except IndexError:
81  logging.error('The index you provided is not in the given list.')
82 
def uploadConditions.getInputRepeat (   prompt = '') 
Like raw_input() but repeats if nothing is provided and automatic strip().

Definition at line 83 of file uploadConditions.py.

Referenced by runWizard().

83 
84 def getInputRepeat(prompt = ''):
85  '''Like raw_input() but repeats if nothing is provided and automatic strip().
86  '''
87 
88  while True:
89  answer = raw_input(prompt)
90  if answer:
91  return answer.strip()
92 
93  logging.error('You need to provide a value.')
94 
def uploadConditions.getInputWorkflow (   prompt = '') 
Like getInput() but tailored to get target workflows (synchronization options).

Definition at line 55 of file uploadConditions.py.

References getInput().

55 
56 def getInputWorkflow(prompt = ''):
57  '''Like getInput() but tailored to get target workflows (synchronization options).
58  '''
59 
60  while True:
61  workflow = getInput(defaultWorkflow, prompt)
62 
63  if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
64  return workflow
65 
66  logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
67 
def uploadConditions.main ( ) 
Entry point.

Definition at line 774 of file uploadConditions.py.

References uploadAllFiles().

774 
775 def main():
776  '''Entry point.
777  '''
778 
779  parser = optparse.OptionParser(usage =
780  'Usage: %prog [options] <file> [<file> ...]\n'
781  )
782 
783  parser.add_option('-d', '--debug',
784  dest = 'debug',
785  action="store_true",
786  default = False,
787  help = 'Switch on printing debug information. Default: %default',
788  )
789 
790  parser.add_option('-b', '--backend',
791  dest = 'backend',
792  default = defaultBackend,
793  help = 'dropBox\'s backend to upload to. Default: %default',
794  )
795 
796  parser.add_option('-H', '--hostname',
797  dest = 'hostname',
798  default = defaultHostname,
799  help = 'dropBox\'s hostname. Default: %default',
800  )
801 
802  parser.add_option('-u', '--urlTemplate',
803  dest = 'urlTemplate',
804  default = defaultUrlTemplate,
805  help = 'dropBox\'s URL template. Default: %default',
806  )
807 
808  parser.add_option('-f', '--temporaryFile',
809  dest = 'temporaryFile',
810  default = defaultTemporaryFile,
811  help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
812  )
813 
814  parser.add_option('-n', '--netrcHost',
815  dest = 'netrcHost',
816  default = defaultNetrcHost,
817  help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
818  )
819 
820  parser.add_option('-a', '--authPath',
821  dest = 'authPath',
822  default = None,
823  help = 'The path of the .netrc file for the authentication. Default: $HOME',
824  )
825 
826  (options, arguments) = parser.parse_args()
827 
828  if len(arguments) < 1:
829  parser.print_help()
830  return -2
831 
832  logLevel = logging.INFO
833  if options.debug:
834  logLevel = logging.DEBUG
835  logging.basicConfig(
836  format = '[%(asctime)s] %(levelname)s: %(message)s',
837  level = logLevel,
838  )
839 
840  results = uploadAllFiles(options, arguments)
841 
842  if not results.has_key('status'):
843  print 'Unexpected error.'
844  return -1
845  ret = results['status']
846  print results
847  print "upload ended with code: %s" %ret
848  #for hash, res in results.items():
849  # print "\t %s : %s " % (hash, str(res))
850  return ret
def uploadConditions.runWizard (   basename,
  dataFilename,
  metadataFilename 
)

Definition at line 95 of file uploadConditions.py.

References getInput(), getInputChoose(), and getInputRepeat().

Referenced by uploadAllFiles().

95 
96 def runWizard(basename, dataFilename, metadataFilename):
97  while True:
98  print '''\nWizard for metadata for %s
99 
100 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
101 
102  # Try to get the available inputTags
103  try:
104  dataConnection = sqlite3.connect(dataFilename)
105  dataCursor = dataConnection.cursor()
106  dataCursor.execute('select name from sqlite_master where type == "table"')
107  tables = set(zip(*dataCursor.fetchall())[0])
108 
109  # only conddb V2 supported...
110  if 'TAG' in tables:
111  dataCursor.execute('select NAME from TAG')
112  # In any other case, do not try to get the inputTags
113  else:
114  raise Exception()
115 
116  inputTags = dataCursor.fetchall()
117  if len(inputTags) == 0:
118  raise Exception()
119  inputTags = zip(*inputTags)[0]
120 
121  except Exception:
122  inputTags = []
123 
124  if len(inputTags) == 0:
125  print '\nI could not find any input tag in your data file, but you can still specify one manually.'
126 
127  inputTag = getInputRepeat(
128  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
129 
130  else:
131  print '\nI found the following input tags in your SQLite data file:'
132  for (index, inputTag) in enumerate(inputTags):
133  print ' %s) %s' % (index, inputTag)
134 
135  inputTag = getInputChoose(inputTags, '0',
136  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
137 
138  destinationDatabase = ''
139  ntry = 0
140  while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
141  if ntry==0:
142  inputMessage = \
143  '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
144  elif ntry==1:
145  inputMessage = \
146  '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
147 \ndestinationDatabase: '
148  else:
149  raise Exception('No valid destination chosen. Bailing out...')
150  destinationDatabase = getInputRepeat(inputMessage)
151  ntry += 1
152 
153  while True:
154  since = getInput('',
155  '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
156  if not since:
157  since = None
158  break
159  else:
160  try:
161  since = int(since)
162  break
163  except ValueError:
164  logging.error('The since value has to be an integer or empty (null).')
165 
166  userText = getInput('',
167  '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
168 
169  destinationTags = {}
170  while True:
171  destinationTag = getInput('',
172  '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
173  if not destinationTag:
174  if len(destinationTags) == 0:
175  logging.error('There must be at least one destination tag.')
176  continue
177  break
178 
179  if destinationTag in destinationTags:
180  logging.warning(
181  'You already added this destination tag. Overwriting the previous one with this new one.')
182 
183  destinationTags[destinationTag] = {
184  }
185 
186  metadata = {
187  'destinationDatabase': destinationDatabase,
188  'destinationTags': destinationTags,
189  'inputTag': inputTag,
190  'since': since,
191  'userText': userText,
192  }
193 
194  metadata = json.dumps(metadata, sort_keys=True, indent=4)
195  print '\nThis is the generated metadata:\n%s' % metadata
196 
197  if getInput('n',
198  '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
199  break
200  logging.info('Saving generated metadata in %s...', metadataFilename)
201  with open(metadataFilename, 'wb') as metadataFile:
202  metadataFile.write(metadata)
def uploadConditions.testTier0Upload ( )

Definition at line 851 of file uploadConditions.py.

References uploadTier0Files().

851 
852 def testTier0Upload():
853 
854  global defaultNetrcHost
855 
856  (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
857 
858  filenames = ['testFiles/localSqlite-top2']
859 
860  uploadTier0Files(filenames, username, password, cookieFileName = None)
861 
def uploadConditions.uploadAllFiles (   options,
  arguments 
)

Definition at line 616 of file uploadConditions.py.

References authenticateUser(), getInput(), and runWizard().

Referenced by main().

616 
617 def uploadAllFiles(options, arguments):
618 
619  ret = {}
620  ret['status'] = 0
621 
622  # Check that we can read the data and metadata files
623  # If the metadata file does not exist, start the wizard
624  for filename in arguments:
625  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
626  basename = os.path.basename(basepath)
627  dataFilename = '%s.db' % basepath
628  metadataFilename = '%s.txt' % basepath
629 
630  logging.info('Checking %s...', basename)
631 
632  # Data file
633  try:
634  with open(dataFilename, 'rb') as dataFile:
635  pass
636  except IOError as e:
637  errMsg = 'Impossible to open SQLite data file %s' %dataFilename
638  logging.error( errMsg )
639  ret['status'] = -3
640  ret['error'] = errMsg
641  return ret
642 
643  # Check the data file
644  empty = True
645  try:
646  dbcon = sqlite3.connect( dataFilename )
647  dbcur = dbcon.cursor()
648  dbcur.execute('SELECT * FROM IOV')
649  rows = dbcur.fetchall()
650  for r in rows:
651  empty = False
652  dbcon.close()
653  if empty:
654  errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
655  logging.error( errMsg )
656  ret['status'] = -4
657  ret['error'] = errMsg
658  return ret
659  except Exception as e:
660  errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
661  logging.error( errMsg )
662  ret['status'] = -5
663  ret['error'] = errMsg
664  return ret
665 
666  # Metadata file
667  try:
668  with open(metadataFilename, 'rb') as metadataFile:
669  pass
670  except IOError as e:
671  if e.errno != errno.ENOENT:
672  errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
673  logging.error( errMsg )
674  ret['status'] = -4
675  ret['error'] = errMsg
676  return ret
677 
678  if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
679  errMsg = 'Metadata file %s does not exist' %metadataFilename
680  logging.error( errMsg )
681  ret['status'] = -5
682  ret['error'] = errMsg
683  return ret
684  # Wizard
685  runWizard(basename, dataFilename, metadataFilename)
686 
687  # Upload files
688  try:
689  dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
690 
691  # Authentication
692  if not authenticateUser(dropBox, options):
693  logging.error("Error authenticating user. Aborting.")
694  return { 'status' : -2, 'error' : "Error authenticating user. Aborting." }
695 
696  # At this point we must be authenticated
697  dropBox._checkForUpdates()
698 
699  results = {}
700  for filename in arguments:
701  backend = options.backend
702  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
703  metadataFilename = '%s.txt' % basepath
704  with open(metadataFilename, 'rb') as metadataFile:
705  metadata = json.load( metadataFile )
706  # When dest db = prep the hostname has to be set to dev.
707  forceHost = False
708  destDb = metadata['destinationDatabase']
709  if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
710  if destDb.startswith('oracle://cms_orcoff_prep'):
711  dropBox.setHost( defaultDevHostname )
712  dropBox.signInAgain()
713  forceHost = True
714  results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
715  if forceHost:
716  # set back the hostname to the original global setting
717  dropBox.setHost( options.hostname )
718  dropBox.signInAgain()
719  else:
720  results[filename] = False
721  logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
722  if not results[filename]:
723  if ret['status']<0:
724  ret['status'] = 0
725  ret['status'] += 1
726  ret['files'] = results
727  logging.debug("all files processed, logging out now.")
728 
729  dropBox.signOut()
730 
731  except HTTPError as e:
732  logging.error('got HTTP error: %s', str(e))
733  return { 'status' : -1, 'error' : str(e) }
734 
735  return ret
def uploadConditions.uploadTier0Files (   filenames,
  username,
  password,
  cookieFileName = None 
) 
Uploads a bunch of files coming from Tier0.
This has the following requirements:
    * Username/Password based authentication.
    * Uses the online backend.
    * Ignores errors related to the upload/content (e.g. duplicated file).

Definition at line 736 of file uploadConditions.py.

Referenced by testTier0Upload().

736 
737 def uploadTier0Files(filenames, username, password, cookieFileName = None):
738  '''Uploads a bunch of files coming from Tier0.
739  This has the following requirements:
740  * Username/Password based authentication.
741  * Uses the online backend.
742  * Ignores errors related to the upload/content (e.g. duplicated file).
743  '''
744 
745  dropBox = ConditionsUploader()
746 
747  dropBox.signIn(username, password)
748 
749  for filename in filenames:
750  try:
751  result = dropBox.uploadFile(filename, backend = 'test')
752  except HTTPError as e:
753  if e.code == 400:
754  # 400 Bad Request: This is an exception related to the upload
755  # being wrong for some reason (e.g. duplicated file).
756  # Since for Tier0 this is not an issue, continue
757  logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
758  continue
759 
760  # In any other case, re-raise.
761  raise
762 
763  #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
764  # if we flag as failed any retry would not help and would result in the same error (e.g.
765  # when a file with an identical hash is uploaded again)
766  #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
767 
768  if not result: # dropbox reported an error when uploading, do not retry.
769  logging.error('Error from dropbox, upload-related, skipping.')
770  continue
771 
772  dropBox.signOut()
773 

Variable Documentation

string uploadConditions.__author__ = 'Andreas Pfeiffer'

Definition at line 6 of file uploadConditions.py.

string uploadConditions.__copyright__ = 'Copyright 2015, CERN CMS'

Definition at line 7 of file uploadConditions.py.

list uploadConditions.__credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 8 of file uploadConditions.py.

string uploadConditions.__email__ = 'andreas.pfeiffer@cern.ch'

Definition at line 11 of file uploadConditions.py.

string uploadConditions.__license__ = 'Unknown'

Definition at line 9 of file uploadConditions.py.

string uploadConditions.__maintainer__ = 'Andreas Pfeiffer'

Definition at line 10 of file uploadConditions.py.

int uploadConditions.__version__ = 1

Definition at line 12 of file uploadConditions.py.

string uploadConditions.CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'

Definition at line 221 of file uploadConditions.py.

string uploadConditions.defaultBackend = 'online'

Definition at line 27 of file uploadConditions.py.

string uploadConditions.defaultDevHostname = 'cms-conddb-dev.cern.ch'

Definition at line 29 of file uploadConditions.py.

string uploadConditions.defaultHostname = 'cms-conddb-prod.cern.ch'

Definition at line 28 of file uploadConditions.py.

string uploadConditions.defaultNetrcHost = 'ConditionUploader'

Definition at line 32 of file uploadConditions.py.

string uploadConditions.defaultTemporaryFile = 'upload.tar.bz2'

Definition at line 31 of file uploadConditions.py.

string uploadConditions.defaultUrlTemplate = 'https://%s/cmsDbUpload/'

Definition at line 30 of file uploadConditions.py.

string uploadConditions.defaultWorkflow = 'offline'

Definition at line 33 of file uploadConditions.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5