CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
uploadConditions Namespace Reference

Classes

class  ConditionsUploader
 
class  HTTP
 
class  HTTPError
 

Functions

def addToTarFile
 
def authenticateUser
 
def getInput
 
def getInputChoose
 
def getInputRepeat
 
def getInputWorkflow
 
def main
 
def runWizard
 
def testTier0Upload
 
def uploadAllFiles
 
def uploadTier0Files
 

Variables

string __author__ = 'Andreas Pfeiffer'
 
string __copyright__ = 'Copyright 2015, CERN CMS'
 
list __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'andreas.pfeiffer@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Andreas Pfeiffer'
 
int __version__ = 1
 
string CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
 
string defaultBackend = 'online'
 
string defaultDevHostname = 'cms-conddb-dev.cern.ch'
 
string defaultHostname = 'cms-conddb-prod.cern.ch'
 
string defaultNetrcHost = 'Dropbox'
 
string defaultTemporaryFile = 'upload.tar.bz2'
 
string defaultUrlTemplate = 'https://%s/cmsDbUpload/'
 
string defaultWorkflow = 'offline'
 

Function Documentation

def uploadConditions.addToTarFile (   tarFile,
  fileobj,
  arcname 
)

Definition at line 410 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.uploadFile().

410 
411 def addToTarFile(tarFile, fileobj, arcname):
412  tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
413  tarInfo.mode = 0o400
414  tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
415  tarInfo.uname = tarInfo.gname = 'root'
416  tarFile.addfile(tarInfo, fileobj)
def uploadConditions.authenticateUser (   dropBox,
  options 
)

Definition at line 587 of file uploadConditions.py.

References getInput().

Referenced by uploadAllFiles().

587 
588 def authenticateUser(dropBox, options):
589 
590  try:
591  # Try to find the netrc entry
592  (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
593  except Exception:
594  # netrc entry not found, ask for the username and password
595  logging.info(
596  'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
597  options.netrcHost)
598 
599  # Try to get a default username
600  defaultUsername = getpass.getuser()
601  if defaultUsername is None:
602  defaultUsername = '(not found)'
603 
604  username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
605  password = getpass.getpass('Password: ')
606 
607  # Now we have a username and password, authenticate with them
608  return dropBox.signIn(username, password)
609 
def uploadConditions.getInput (   default,
  prompt = '' 
) 
Like raw_input() but with a default and automatic strip().

Definition at line 43 of file uploadConditions.py.

Referenced by authenticateUser(), getInputChoose(), getInputWorkflow(), runWizard(), and uploadAllFiles().

43 
44 def getInput(default, prompt = ''):
45  '''Like raw_input() but with a default and automatic strip().
46  '''
47 
48  answer = raw_input(prompt)
49  if answer:
50  return answer.strip()
51 
52  return default.strip()
53 
def uploadConditions.getInputChoose (   optionsList,
  default,
  prompt = '' 
) 
Makes the user choose from a list of options.

Definition at line 67 of file uploadConditions.py.

References getInput().

Referenced by runWizard().

67 
68 def getInputChoose(optionsList, default, prompt = ''):
69  '''Makes the user choose from a list of options.
70  '''
71 
72  while True:
73  index = getInput(default, prompt)
74 
75  try:
76  return optionsList[int(index)]
77  except ValueError:
78  logging.error('Please specify an index of the list (i.e. integer).')
79  except IndexError:
80  logging.error('The index you provided is not in the given list.')
81 
def uploadConditions.getInputRepeat (   prompt = '') 
Like raw_input() but repeats if nothing is provided and automatic strip().

Definition at line 82 of file uploadConditions.py.

Referenced by runWizard().

82 
83 def getInputRepeat(prompt = ''):
84  '''Like raw_input() but repeats if nothing is provided and automatic strip().
85  '''
86 
87  while True:
88  answer = raw_input(prompt)
89  if answer:
90  return answer.strip()
91 
92  logging.error('You need to provide a value.')
93 
def uploadConditions.getInputWorkflow (   prompt = '') 
Like getInput() but tailored to get target workflows (synchronization options).

Definition at line 54 of file uploadConditions.py.

References getInput().

54 
55 def getInputWorkflow(prompt = ''):
56  '''Like getInput() but tailored to get target workflows (synchronization options).
57  '''
58 
59  while True:
60  workflow = getInput(defaultWorkflow, prompt)
61 
62  if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
63  return workflow
64 
65  logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
66 
def uploadConditions.main ( ) 
Entry point.

Definition at line 731 of file uploadConditions.py.

References uploadAllFiles().

731 
732 def main():
733  '''Entry point.
734  '''
735 
736  parser = optparse.OptionParser(usage =
737  'Usage: %prog [options] <file> [<file> ...]\n'
738  )
739 
740  parser.add_option('-d', '--debug',
741  dest = 'debug',
742  action="store_true",
743  default = False,
744  help = 'Switch on printing debug information. Default: %default',
745  )
746 
747  parser.add_option('-b', '--backend',
748  dest = 'backend',
749  default = defaultBackend,
750  help = 'dropBox\'s backend to upload to. Default: %default',
751  )
752 
753  parser.add_option('-H', '--hostname',
754  dest = 'hostname',
755  default = defaultHostname,
756  help = 'dropBox\'s hostname. Default: %default',
757  )
758 
759  parser.add_option('-u', '--urlTemplate',
760  dest = 'urlTemplate',
761  default = defaultUrlTemplate,
762  help = 'dropBox\'s URL template. Default: %default',
763  )
764 
765  parser.add_option('-f', '--temporaryFile',
766  dest = 'temporaryFile',
767  default = defaultTemporaryFile,
768  help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
769  )
770 
771  parser.add_option('-n', '--netrcHost',
772  dest = 'netrcHost',
773  default = defaultNetrcHost,
774  help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
775  )
776 
777  (options, arguments) = parser.parse_args()
778 
779  if len(arguments) < 1:
780  parser.print_help()
781  return -2
782 
783  logLevel = logging.INFO
784  if options.debug:
785  logLevel = logging.DEBUG
786  logging.basicConfig(
787  format = '[%(asctime)s] %(levelname)s: %(message)s',
788  level = logLevel,
789  )
790 
791  results = uploadAllFiles(options, arguments)
792 
793  print "uploadAllFiles returned:"
794  for hash, res in results.items():
795  print "\t %s : %s " % (hash, str(res))
def uploadConditions.runWizard (   basename,
  dataFilename,
  metadataFilename 
)

Definition at line 94 of file uploadConditions.py.

References getInput(), getInputChoose(), and getInputRepeat().

Referenced by uploadAllFiles().

94 
95 def runWizard(basename, dataFilename, metadataFilename):
96  while True:
97  print '''\nWizard for metadata for %s
98 
99 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
100 
101  # Try to get the available inputTags
102  try:
103  dataConnection = sqlite3.connect(dataFilename)
104  dataCursor = dataConnection.cursor()
105  dataCursor.execute('select name from sqlite_master where type == "table"')
106  tables = set(zip(*dataCursor.fetchall())[0])
107 
108  # only conddb V2 supported...
109  if 'TAG' in tables:
110  dataCursor.execute('select NAME from TAG')
111  # In any other case, do not try to get the inputTags
112  else:
113  raise Exception()
114 
115  inputTags = dataCursor.fetchall()
116  if len(inputTags) == 0:
117  raise Exception()
118  inputTags = zip(*inputTags)[0]
119 
120  except Exception:
121  inputTags = []
122 
123  if len(inputTags) == 0:
124  print '\nI could not find any input tag in your data file, but you can still specify one manually.'
125 
126  inputTag = getInputRepeat(
127  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
128 
129  else:
130  print '\nI found the following input tags in your SQLite data file:'
131  for (index, inputTag) in enumerate(inputTags):
132  print ' %s) %s' % (index, inputTag)
133 
134  inputTag = getInputChoose(inputTags, '0',
135  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
136 
137  destinationDatabase = ''
138  ntry = 0
139  while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
140  if ntry==0:
141  inputMessage = \
142  '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
143  elif ntry==1:
144  inputMessage = \
145  '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
146 \ndestinationDatabase: '
147  else:
148  raise Exception('No valid destination chosen. Bailing out...')
149  destinationDatabase = getInputRepeat(inputMessage)
150  ntry += 1
151 
152  while True:
153  since = getInput('',
154  '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
155  if not since:
156  since = None
157  break
158  else:
159  try:
160  since = int(since)
161  break
162  except ValueError:
163  logging.error('The since value has to be an integer or empty (null).')
164 
165  userText = getInput('',
166  '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
167 
168  destinationTags = {}
169  while True:
170  destinationTag = getInput('',
171  '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
172  if not destinationTag:
173  if len(destinationTags) == 0:
174  logging.error('There must be at least one destination tag.')
175  continue
176  break
177 
178  if destinationTag in destinationTags:
179  logging.warning(
180  'You already added this destination tag. Overwriting the previous one with this new one.')
181 
182  destinationTags[destinationTag] = {
183  }
184 
185  metadata = {
186  'destinationDatabase': destinationDatabase,
187  'destinationTags': destinationTags,
188  'inputTag': inputTag,
189  'since': since,
190  'userText': userText,
191  }
192 
193  metadata = json.dumps(metadata, sort_keys=True, indent=4)
194  print '\nThis is the generated metadata:\n%s' % metadata
195 
196  if getInput('n',
197  '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
198  break
199  logging.info('Saving generated metadata in %s...', metadataFilename)
200  with open(metadataFilename, 'wb') as metadataFile:
201  metadataFile.write(metadata)
def uploadConditions.testTier0Upload ( )

Definition at line 796 of file uploadConditions.py.

References uploadTier0Files().

796 
797 def testTier0Upload():
798 
799  global defaultNetrcHost
800 
801  (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
802 
803  filenames = ['testFiles/localSqlite-top2']
804 
805  uploadTier0Files(filenames, username, password, cookieFileName = None)
806 
def uploadConditions.uploadAllFiles (   options,
  arguments 
)

Definition at line 610 of file uploadConditions.py.

References authenticateUser(), getInput(), and runWizard().

Referenced by main().

610 
611 def uploadAllFiles(options, arguments):
612 
613  results = {}
614 
615  # Check that we can read the data and metadata files
616  # If the metadata file does not exist, start the wizard
617  for filename in arguments:
618  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
619  basename = os.path.basename(basepath)
620  dataFilename = '%s.db' % basepath
621  metadataFilename = '%s.txt' % basepath
622 
623  logging.info('Checking %s...', basename)
624 
625  # Data file
626  try:
627  with open(dataFilename, 'rb') as dataFile:
628  pass
629  except IOError as e:
630  logging.error('Impossible to open SQLite data file %s', dataFilename)
631  return -3
632 
633  # Metadata file
634  try:
635  with open(metadataFilename, 'rb') as metadataFile:
636  pass
637  except IOError as e:
638  if e.errno != errno.ENOENT:
639  logging.error('Impossible to open file %s (for other reason than not existing)', metadataFilename)
640  return -4
641 
642  if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
643  logging.error('Metadata file %s does not exist', metadataFilename)
644  return -5
645 
646  # Wizard
647  runWizard(basename, dataFilename, metadataFilename)
648 
649  # Upload files
650  try:
651  dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
652 
653  # Authentication
654  if not authenticateUser(dropBox, options):
655  logging.error("Error authenticating user. Aborting.")
656  return { 'status' : -2, 'error' : "Error authenticating user. Aborting." }
657 
658  # At this point we must be authenticated
659  dropBox._checkForUpdates()
660 
661  for filename in arguments:
662  backend = options.backend
663  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
664  metadataFilename = '%s.txt' % basepath
665  with open(metadataFilename, 'rb') as metadataFile:
666  metadata = json.load( metadataFile )
667  # When dest db = prep the hostname has to be set to dev.
668  forceHost = False
669  destDb = metadata['destinationDatabase']
670  if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
671  if destDb.startswith('oracle://cms_orcoff_prep'):
672  dropBox.setHost( defaultDevHostname )
673  dropBox.signInAgain()
674  forceHost = True
675  results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
676  if forceHost:
677  # set back the hostname to the original global setting
678  dropBox.setHost( options.hostname )
679  dropBox.signInAgain()
680  else:
681  results[filename] = False
682  logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
683 
684  logging.debug("all files processed, logging out now.")
685 
686  dropBox.signOut()
687 
688  except HTTPError as e:
689  logging.error('got HTTP error: %s', str(e))
690  return { 'status' : -1, 'error' : str(e) }
691 
692  return results
def uploadConditions.uploadTier0Files (   filenames,
  username,
  password,
  cookieFileName = None 
) 
Uploads a bunch of files coming from Tier0.
This has the following requirements:
    * Username/Password based authentication.
    * Uses the online backend.
    * Ignores errors related to the upload/content (e.g. duplicated file).

Definition at line 693 of file uploadConditions.py.

Referenced by testTier0Upload().

693 
694 def uploadTier0Files(filenames, username, password, cookieFileName = None):
695  '''Uploads a bunch of files coming from Tier0.
696  This has the following requirements:
697  * Username/Password based authentication.
698  * Uses the online backend.
699  * Ignores errors related to the upload/content (e.g. duplicated file).
700  '''
701 
702  dropBox = ConditionsUploader()
703 
704  dropBox.signIn(username, password)
705 
706  for filename in filenames:
707  try:
708  result = dropBox.uploadFile(filename, backend = 'test')
709  except HTTPError as e:
710  if e.code == 400:
711  # 400 Bad Request: This is an exception related to the upload
712  # being wrong for some reason (e.g. duplicated file).
713  # Since for Tier0 this is not an issue, continue
714  logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
715  continue
716 
717  # In any other case, re-raise.
718  raise
719 
720  #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
721  # if we flag as failed any retry would not help and would result in the same error (e.g.
722  # when a file with an identical hash is uploaded again)
723  #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
724 
725  if not result: # dropbox reported an error when uploading, do not retry.
726  logging.error('Error from dropbox, upload-related, skipping.')
727  continue
728 
729  dropBox.signOut()
730 

Variable Documentation

string uploadConditions.__author__ = 'Andreas Pfeiffer'

Definition at line 6 of file uploadConditions.py.

string uploadConditions.__copyright__ = 'Copyright 2015, CERN CMS'

Definition at line 7 of file uploadConditions.py.

list uploadConditions.__credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 8 of file uploadConditions.py.

string uploadConditions.__email__ = 'andreas.pfeiffer@cern.ch'

Definition at line 11 of file uploadConditions.py.

string uploadConditions.__license__ = 'Unknown'

Definition at line 9 of file uploadConditions.py.

string uploadConditions.__maintainer__ = 'Andreas Pfeiffer'

Definition at line 10 of file uploadConditions.py.

int uploadConditions.__version__ = 1

Definition at line 12 of file uploadConditions.py.

string uploadConditions.CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'

Definition at line 220 of file uploadConditions.py.

string uploadConditions.defaultBackend = 'online'

Definition at line 27 of file uploadConditions.py.

string uploadConditions.defaultDevHostname = 'cms-conddb-dev.cern.ch'

Definition at line 29 of file uploadConditions.py.

string uploadConditions.defaultHostname = 'cms-conddb-prod.cern.ch'

Definition at line 28 of file uploadConditions.py.

string uploadConditions.defaultNetrcHost = 'Dropbox'

Definition at line 32 of file uploadConditions.py.

string uploadConditions.defaultTemporaryFile = 'upload.tar.bz2'

Definition at line 31 of file uploadConditions.py.

string uploadConditions.defaultUrlTemplate = 'https://%s/cmsDbUpload/'

Definition at line 30 of file uploadConditions.py.

string uploadConditions.defaultWorkflow = 'offline'

Definition at line 33 of file uploadConditions.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5