CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
uploadConditions Namespace Reference

Classes

class  ConditionsUploader
 
class  HTTP
 
class  HTTPError
 

Functions

def addToTarFile
 
def authenticateUser
 
def getInput
 
def getInputChoose
 
def getInputRepeat
 
def getInputWorkflow
 
def main
 
def runWizard
 
def testTier0Upload
 
def uploadAllFiles
 
def uploadTier0Files
 

Variables

string __author__ = 'Andreas Pfeiffer'
 
string __copyright__ = 'Copyright 2015, CERN CMS'
 
list __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'andreas.pfeiffer@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Andreas Pfeiffer'
 
int __version__ = 1
 
string CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
 
string defaultBackend = 'online'
 
string defaultDevHostname = 'cms-conddb-dev.cern.ch'
 
string defaultHostname = 'cms-conddb-prod.cern.ch'
 
string defaultNetrcHost = 'ConditionUploader'
 
string defaultTemporaryFile = 'upload.tar.bz2'
 
string defaultUrlTemplate = 'https://%s/cmsDbUpload/'
 
string defaultWorkflow = 'offline'
 

Function Documentation

def uploadConditions.addToTarFile (   tarFile,
  fileobj,
  arcname 
)

Definition at line 410 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.uploadFile().

410 
411 def addToTarFile(tarFile, fileobj, arcname):
412  tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
413  tarInfo.mode = 0o400
414  tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
415  tarInfo.uname = tarInfo.gname = 'root'
416  tarFile.addfile(tarInfo, fileobj)
def uploadConditions.authenticateUser (   dropBox,
  options 
)

Definition at line 587 of file uploadConditions.py.

References getInput().

Referenced by uploadAllFiles().

587 
588 def authenticateUser(dropBox, options):
589 
590  try:
591  # Try to find the netrc entry
592  (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
593  except Exception:
594  # netrc entry not found, ask for the username and password
595  logging.info(
596  'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
597  options.netrcHost)
598 
599  # Try to get a default username
600  defaultUsername = getpass.getuser()
601  if defaultUsername is None:
602  defaultUsername = '(not found)'
603 
604  username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
605  password = getpass.getpass('Password: ')
606 
607  # Now we have a username and password, authenticate with them
608  return dropBox.signIn(username, password)
609 
def uploadConditions.getInput (   default,
  prompt = '' 
) 
Like raw_input() but with a default and automatic strip().

Definition at line 43 of file uploadConditions.py.

Referenced by authenticateUser(), getInputChoose(), getInputWorkflow(), runWizard(), and uploadAllFiles().

43 
44 def getInput(default, prompt = ''):
45  '''Like raw_input() but with a default and automatic strip().
46  '''
47 
48  answer = raw_input(prompt)
49  if answer:
50  return answer.strip()
51 
52  return default.strip()
53 
def uploadConditions.getInputChoose (   optionsList,
  default,
  prompt = '' 
) 
Makes the user choose from a list of options.

Definition at line 67 of file uploadConditions.py.

References getInput().

Referenced by runWizard().

67 
68 def getInputChoose(optionsList, default, prompt = ''):
69  '''Makes the user choose from a list of options.
70  '''
71 
72  while True:
73  index = getInput(default, prompt)
74 
75  try:
76  return optionsList[int(index)]
77  except ValueError:
78  logging.error('Please specify an index of the list (i.e. integer).')
79  except IndexError:
80  logging.error('The index you provided is not in the given list.')
81 
def uploadConditions.getInputRepeat (   prompt = '') 
Like raw_input() but repeats if nothing is provided and automatic strip().

Definition at line 82 of file uploadConditions.py.

Referenced by runWizard().

82 
83 def getInputRepeat(prompt = ''):
84  '''Like raw_input() but repeats if nothing is provided and automatic strip().
85  '''
86 
87  while True:
88  answer = raw_input(prompt)
89  if answer:
90  return answer.strip()
91 
92  logging.error('You need to provide a value.')
93 
def uploadConditions.getInputWorkflow (   prompt = '') 
Like getInput() but tailored to get target workflows (synchronization options).

Definition at line 54 of file uploadConditions.py.

References getInput().

54 
55 def getInputWorkflow(prompt = ''):
56  '''Like getInput() but tailored to get target workflows (synchronization options).
57  '''
58 
59  while True:
60  workflow = getInput(defaultWorkflow, prompt)
61 
62  if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
63  return workflow
64 
65  logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
66 
def uploadConditions.main ( ) 
Entry point.

Definition at line 745 of file uploadConditions.py.

References uploadAllFiles().

745 
746 def main():
747  '''Entry point.
748  '''
749 
750  parser = optparse.OptionParser(usage =
751  'Usage: %prog [options] <file> [<file> ...]\n'
752  )
753 
754  parser.add_option('-d', '--debug',
755  dest = 'debug',
756  action="store_true",
757  default = False,
758  help = 'Switch on printing debug information. Default: %default',
759  )
760 
761  parser.add_option('-b', '--backend',
762  dest = 'backend',
763  default = defaultBackend,
764  help = 'dropBox\'s backend to upload to. Default: %default',
765  )
766 
767  parser.add_option('-H', '--hostname',
768  dest = 'hostname',
769  default = defaultHostname,
770  help = 'dropBox\'s hostname. Default: %default',
771  )
772 
773  parser.add_option('-u', '--urlTemplate',
774  dest = 'urlTemplate',
775  default = defaultUrlTemplate,
776  help = 'dropBox\'s URL template. Default: %default',
777  )
778 
779  parser.add_option('-f', '--temporaryFile',
780  dest = 'temporaryFile',
781  default = defaultTemporaryFile,
782  help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
783  )
784 
785  parser.add_option('-n', '--netrcHost',
786  dest = 'netrcHost',
787  default = defaultNetrcHost,
788  help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
789  )
790 
791  (options, arguments) = parser.parse_args()
792 
793  if len(arguments) < 1:
794  parser.print_help()
795  return -2
796 
797  logLevel = logging.INFO
798  if options.debug:
799  logLevel = logging.DEBUG
800  logging.basicConfig(
801  format = '[%(asctime)s] %(levelname)s: %(message)s',
802  level = logLevel,
803  )
804 
805  results = uploadAllFiles(options, arguments)
806 
807  if not results.has_key('status'):
808  print 'Unexpected error.'
809  return -1
810  ret = results['status']
811  print results
812  print "upload ended with code: %s" %ret
813  #for hash, res in results.items():
814  # print "\t %s : %s " % (hash, str(res))
815  return ret
def uploadConditions.runWizard (   basename,
  dataFilename,
  metadataFilename 
)

Definition at line 94 of file uploadConditions.py.

References getInput(), getInputChoose(), and getInputRepeat().

Referenced by uploadAllFiles().

94 
95 def runWizard(basename, dataFilename, metadataFilename):
96  while True:
97  print '''\nWizard for metadata for %s
98 
99 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
100 
101  # Try to get the available inputTags
102  try:
103  dataConnection = sqlite3.connect(dataFilename)
104  dataCursor = dataConnection.cursor()
105  dataCursor.execute('select name from sqlite_master where type == "table"')
106  tables = set(zip(*dataCursor.fetchall())[0])
107 
108  # only conddb V2 supported...
109  if 'TAG' in tables:
110  dataCursor.execute('select NAME from TAG')
111  # In any other case, do not try to get the inputTags
112  else:
113  raise Exception()
114 
115  inputTags = dataCursor.fetchall()
116  if len(inputTags) == 0:
117  raise Exception()
118  inputTags = zip(*inputTags)[0]
119 
120  except Exception:
121  inputTags = []
122 
123  if len(inputTags) == 0:
124  print '\nI could not find any input tag in your data file, but you can still specify one manually.'
125 
126  inputTag = getInputRepeat(
127  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
128 
129  else:
130  print '\nI found the following input tags in your SQLite data file:'
131  for (index, inputTag) in enumerate(inputTags):
132  print ' %s) %s' % (index, inputTag)
133 
134  inputTag = getInputChoose(inputTags, '0',
135  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
136 
137  destinationDatabase = ''
138  ntry = 0
139  while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
140  if ntry==0:
141  inputMessage = \
142  '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
143  elif ntry==1:
144  inputMessage = \
145  '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
146 \ndestinationDatabase: '
147  else:
148  raise Exception('No valid destination chosen. Bailing out...')
149  destinationDatabase = getInputRepeat(inputMessage)
150  ntry += 1
151 
152  while True:
153  since = getInput('',
154  '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
155  if not since:
156  since = None
157  break
158  else:
159  try:
160  since = int(since)
161  break
162  except ValueError:
163  logging.error('The since value has to be an integer or empty (null).')
164 
165  userText = getInput('',
166  '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
167 
168  destinationTags = {}
169  while True:
170  destinationTag = getInput('',
171  '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
172  if not destinationTag:
173  if len(destinationTags) == 0:
174  logging.error('There must be at least one destination tag.')
175  continue
176  break
177 
178  if destinationTag in destinationTags:
179  logging.warning(
180  'You already added this destination tag. Overwriting the previous one with this new one.')
181 
182  destinationTags[destinationTag] = {
183  }
184 
185  metadata = {
186  'destinationDatabase': destinationDatabase,
187  'destinationTags': destinationTags,
188  'inputTag': inputTag,
189  'since': since,
190  'userText': userText,
191  }
192 
193  metadata = json.dumps(metadata, sort_keys=True, indent=4)
194  print '\nThis is the generated metadata:\n%s' % metadata
195 
196  if getInput('n',
197  '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
198  break
199  logging.info('Saving generated metadata in %s...', metadataFilename)
200  with open(metadataFilename, 'wb') as metadataFile:
201  metadataFile.write(metadata)
def uploadConditions.testTier0Upload ( )

Definition at line 816 of file uploadConditions.py.

References uploadTier0Files().

816 
817 def testTier0Upload():
818 
819  global defaultNetrcHost
820 
821  (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
822 
823  filenames = ['testFiles/localSqlite-top2']
824 
825  uploadTier0Files(filenames, username, password, cookieFileName = None)
826 
def uploadConditions.uploadAllFiles (   options,
  arguments 
)

Definition at line 610 of file uploadConditions.py.

References authenticateUser(), getInput(), and runWizard().

Referenced by main().

610 
611 def uploadAllFiles(options, arguments):
612 
613  ret = {}
614  ret['status'] = 0
615 
616  # Check that we can read the data and metadata files
617  # If the metadata file does not exist, start the wizard
618  for filename in arguments:
619  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
620  basename = os.path.basename(basepath)
621  dataFilename = '%s.db' % basepath
622  metadataFilename = '%s.txt' % basepath
623 
624  logging.info('Checking %s...', basename)
625 
626  # Data file
627  try:
628  with open(dataFilename, 'rb') as dataFile:
629  pass
630  except IOError as e:
631  errMsg = 'Impossible to open SQLite data file %s' %dataFilename
632  logging.error( errMsg )
633  ret['status'] = -3
634  ret['error'] = errMsg
635  return ret
636 
637  # Metadata file
638  try:
639  with open(metadataFilename, 'rb') as metadataFile:
640  pass
641  except IOError as e:
642  if e.errno != errno.ENOENT:
643  errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
644  logging.error( errMsg )
645  ret['status'] = -4
646  ret['error'] = errMsg
647  return ret
648 
649  if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
650  errMsg = 'Metadata file %s does not exist' %metadataFilename
651  logging.error( errMsg )
652  ret['status'] = -5
653  ret['error'] = errMsg
654  return ret
655  # Wizard
656  runWizard(basename, dataFilename, metadataFilename)
657 
658  # Upload files
659  try:
660  dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
661 
662  # Authentication
663  if not authenticateUser(dropBox, options):
664  logging.error("Error authenticating user. Aborting.")
665  return { 'status' : -2, 'error' : "Error authenticating user. Aborting." }
666 
667  # At this point we must be authenticated
668  dropBox._checkForUpdates()
669 
670  results = {}
671  for filename in arguments:
672  backend = options.backend
673  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
674  metadataFilename = '%s.txt' % basepath
675  with open(metadataFilename, 'rb') as metadataFile:
676  metadata = json.load( metadataFile )
677  # When dest db = prep the hostname has to be set to dev.
678  forceHost = False
679  destDb = metadata['destinationDatabase']
680  if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
681  if destDb.startswith('oracle://cms_orcoff_prep'):
682  dropBox.setHost( defaultDevHostname )
683  dropBox.signInAgain()
684  forceHost = True
685  results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
686  if forceHost:
687  # set back the hostname to the original global setting
688  dropBox.setHost( options.hostname )
689  dropBox.signInAgain()
690  else:
691  results[filename] = False
692  logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
693  if not results[filename]:
694  if ret['status']<0:
695  ret['status'] = 0
696  ret['status'] += 1
697  ret['files'] = results
698  logging.debug("all files processed, logging out now.")
699 
700  dropBox.signOut()
701 
702  except HTTPError as e:
703  logging.error('got HTTP error: %s', str(e))
704  return { 'status' : -1, 'error' : str(e) }
705 
706  return ret
def uploadConditions.uploadTier0Files (   filenames,
  username,
  password,
  cookieFileName = None 
) 
Uploads a bunch of files coming from Tier0.
This has the following requirements:
    * Username/Password based authentication.
    * Uses the online backend.
    * Ignores errors related to the upload/content (e.g. duplicated file).

Definition at line 707 of file uploadConditions.py.

Referenced by testTier0Upload().

707 
708 def uploadTier0Files(filenames, username, password, cookieFileName = None):
709  '''Uploads a bunch of files coming from Tier0.
710  This has the following requirements:
711  * Username/Password based authentication.
712  * Uses the online backend.
713  * Ignores errors related to the upload/content (e.g. duplicated file).
714  '''
715 
716  dropBox = ConditionsUploader()
717 
718  dropBox.signIn(username, password)
719 
720  for filename in filenames:
721  try:
722  result = dropBox.uploadFile(filename, backend = 'test')
723  except HTTPError as e:
724  if e.code == 400:
725  # 400 Bad Request: This is an exception related to the upload
726  # being wrong for some reason (e.g. duplicated file).
727  # Since for Tier0 this is not an issue, continue
728  logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
729  continue
730 
731  # In any other case, re-raise.
732  raise
733 
734  #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
735  # if we flag as failed any retry would not help and would result in the same error (e.g.
736  # when a file with an identical hash is uploaded again)
737  #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
738 
739  if not result: # dropbox reported an error when uploading, do not retry.
740  logging.error('Error from dropbox, upload-related, skipping.')
741  continue
742 
743  dropBox.signOut()
744 

Variable Documentation

string uploadConditions.__author__ = 'Andreas Pfeiffer'

Definition at line 6 of file uploadConditions.py.

string uploadConditions.__copyright__ = 'Copyright 2015, CERN CMS'

Definition at line 7 of file uploadConditions.py.

list uploadConditions.__credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 8 of file uploadConditions.py.

string uploadConditions.__email__ = 'andreas.pfeiffer@cern.ch'

Definition at line 11 of file uploadConditions.py.

string uploadConditions.__license__ = 'Unknown'

Definition at line 9 of file uploadConditions.py.

string uploadConditions.__maintainer__ = 'Andreas Pfeiffer'

Definition at line 10 of file uploadConditions.py.

int uploadConditions.__version__ = 1

Definition at line 12 of file uploadConditions.py.

string uploadConditions.CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'

Definition at line 220 of file uploadConditions.py.

string uploadConditions.defaultBackend = 'online'

Definition at line 27 of file uploadConditions.py.

string uploadConditions.defaultDevHostname = 'cms-conddb-dev.cern.ch'

Definition at line 29 of file uploadConditions.py.

string uploadConditions.defaultHostname = 'cms-conddb-prod.cern.ch'

Definition at line 28 of file uploadConditions.py.

string uploadConditions.defaultNetrcHost = 'ConditionUploader'

Definition at line 32 of file uploadConditions.py.

string uploadConditions.defaultTemporaryFile = 'upload.tar.bz2'

Definition at line 31 of file uploadConditions.py.

string uploadConditions.defaultUrlTemplate = 'https://%s/cmsDbUpload/'

Definition at line 30 of file uploadConditions.py.

string uploadConditions.defaultWorkflow = 'offline'

Definition at line 33 of file uploadConditions.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5