CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
Classes | Functions | Variables
uploadConditions Namespace Reference

Classes

class  ConditionsUploader
 
class  HTTP
 
class  HTTPError
 

Functions

def addToTarFile
 
def authenticateUser
 
def getInput
 
def getInputChoose
 
def getInputRepeat
 
def getInputWorkflow
 
def main
 
def runWizard
 
def testTier0Upload
 
def uploadAllFiles
 
def uploadTier0Files
 

Variables

string __author__ = 'Andreas Pfeiffer'
 
string __copyright__ = 'Copyright 2015, CERN CMS'
 
list __credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
 
string __email__ = 'andreas.pfeiffer@cern.ch'
 
string __license__ = 'Unknown'
 
string __maintainer__ = 'Andreas Pfeiffer'
 
int __version__ = 1
 
string CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'
 
string defaultBackend = 'online'
 
string defaultDevHostname = 'cms-conddb-dev.cern.ch'
 
string defaultHostname = 'cms-conddb-prod.cern.ch'
 
string defaultNetrcHost = 'ConditionUploader'
 
string defaultTemporaryFile = 'upload.tar.bz2'
 
string defaultUrlTemplate = 'https://%s/cmsDbUpload/'
 
string defaultWorkflow = 'offline'
 

Function Documentation

def uploadConditions.addToTarFile (   tarFile,
  fileobj,
  arcname 
)

Definition at line 411 of file uploadConditions.py.

Referenced by uploadConditions.ConditionsUploader.uploadFile().

411 
412 def addToTarFile(tarFile, fileobj, arcname):
413  tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
414  tarInfo.mode = 0o400
415  tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
416  tarInfo.uname = tarInfo.gname = 'root'
417  tarFile.addfile(tarInfo, fileobj)
def uploadConditions.authenticateUser (   dropBox,
  options 
)

Definition at line 590 of file uploadConditions.py.

References getInput().

Referenced by uploadAllFiles().

590 
591 def authenticateUser(dropBox, options):
592 
593  try:
594  # Try to find the netrc entry
595  (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
596  except Exception:
597  # netrc entry not found, ask for the username and password
598  logging.info(
599  'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
600  options.netrcHost)
601 
602  # Try to get a default username
603  defaultUsername = getpass.getuser()
604  if defaultUsername is None:
605  defaultUsername = '(not found)'
606 
607  username = getInput(defaultUsername, '\nUsername [%s]: ' % defaultUsername)
608  password = getpass.getpass('Password: ')
609 
610  # Now we have a username and password, authenticate with them
611  return dropBox.signIn(username, password)
612 
def uploadConditions.getInput (   default,
  prompt = '' 
) 
Like raw_input() but with a default and automatic strip().

Definition at line 44 of file uploadConditions.py.

Referenced by authenticateUser(), getInputChoose(), getInputWorkflow(), runWizard(), and uploadAllFiles().

44 
45 def getInput(default, prompt = ''):
46  '''Like raw_input() but with a default and automatic strip().
47  '''
48 
49  answer = raw_input(prompt)
50  if answer:
51  return answer.strip()
52 
53  return default.strip()
54 
def uploadConditions.getInputChoose (   optionsList,
  default,
  prompt = '' 
) 
Makes the user choose from a list of options.

Definition at line 68 of file uploadConditions.py.

References getInput().

Referenced by runWizard().

68 
69 def getInputChoose(optionsList, default, prompt = ''):
70  '''Makes the user choose from a list of options.
71  '''
72 
73  while True:
74  index = getInput(default, prompt)
75 
76  try:
77  return optionsList[int(index)]
78  except ValueError:
79  logging.error('Please specify an index of the list (i.e. integer).')
80  except IndexError:
81  logging.error('The index you provided is not in the given list.')
82 
def uploadConditions.getInputRepeat (   prompt = '') 
Like raw_input() but repeats if nothing is provided and automatic strip().

Definition at line 83 of file uploadConditions.py.

Referenced by runWizard().

83 
84 def getInputRepeat(prompt = ''):
85  '''Like raw_input() but repeats if nothing is provided and automatic strip().
86  '''
87 
88  while True:
89  answer = raw_input(prompt)
90  if answer:
91  return answer.strip()
92 
93  logging.error('You need to provide a value.')
94 
def uploadConditions.getInputWorkflow (   prompt = '') 
Like getInput() but tailored to get target workflows (synchronization options).

Definition at line 55 of file uploadConditions.py.

References getInput().

55 
56 def getInputWorkflow(prompt = ''):
57  '''Like getInput() but tailored to get target workflows (synchronization options).
58  '''
59 
60  while True:
61  workflow = getInput(defaultWorkflow, prompt)
62 
63  if workflow in frozenset(['offline', 'hlt', 'express', 'prompt', 'pcl']):
64  return workflow
65 
66  logging.error('Please specify one of the allowed workflows. See above for the explanation on each of them.')
67 
def uploadConditions.main ( ) 
Entry point.

Definition at line 771 of file uploadConditions.py.

References uploadAllFiles().

771 
772 def main():
773  '''Entry point.
774  '''
775 
776  parser = optparse.OptionParser(usage =
777  'Usage: %prog [options] <file> [<file> ...]\n'
778  )
779 
780  parser.add_option('-d', '--debug',
781  dest = 'debug',
782  action="store_true",
783  default = False,
784  help = 'Switch on printing debug information. Default: %default',
785  )
786 
787  parser.add_option('-b', '--backend',
788  dest = 'backend',
789  default = defaultBackend,
790  help = 'dropBox\'s backend to upload to. Default: %default',
791  )
792 
793  parser.add_option('-H', '--hostname',
794  dest = 'hostname',
795  default = defaultHostname,
796  help = 'dropBox\'s hostname. Default: %default',
797  )
798 
799  parser.add_option('-u', '--urlTemplate',
800  dest = 'urlTemplate',
801  default = defaultUrlTemplate,
802  help = 'dropBox\'s URL template. Default: %default',
803  )
804 
805  parser.add_option('-f', '--temporaryFile',
806  dest = 'temporaryFile',
807  default = defaultTemporaryFile,
808  help = 'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
809  )
810 
811  parser.add_option('-n', '--netrcHost',
812  dest = 'netrcHost',
813  default = defaultNetrcHost,
814  help = 'The netrc host (machine) from where the username and password will be read. Default: %default',
815  )
816 
817  (options, arguments) = parser.parse_args()
818 
819  if len(arguments) < 1:
820  parser.print_help()
821  return -2
822 
823  logLevel = logging.INFO
824  if options.debug:
825  logLevel = logging.DEBUG
826  logging.basicConfig(
827  format = '[%(asctime)s] %(levelname)s: %(message)s',
828  level = logLevel,
829  )
830 
831  results = uploadAllFiles(options, arguments)
832 
833  if not results.has_key('status'):
834  print 'Unexpected error.'
835  return -1
836  ret = results['status']
837  print results
838  print "upload ended with code: %s" %ret
839  #for hash, res in results.items():
840  # print "\t %s : %s " % (hash, str(res))
841  return ret
def uploadConditions.runWizard (   basename,
  dataFilename,
  metadataFilename 
)

Definition at line 95 of file uploadConditions.py.

References getInput(), getInputChoose(), and getInputRepeat().

Referenced by uploadAllFiles().

95 
96 def runWizard(basename, dataFilename, metadataFilename):
97  while True:
98  print '''\nWizard for metadata for %s
99 
100 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
101 
102  # Try to get the available inputTags
103  try:
104  dataConnection = sqlite3.connect(dataFilename)
105  dataCursor = dataConnection.cursor()
106  dataCursor.execute('select name from sqlite_master where type == "table"')
107  tables = set(zip(*dataCursor.fetchall())[0])
108 
109  # only conddb V2 supported...
110  if 'TAG' in tables:
111  dataCursor.execute('select NAME from TAG')
112  # In any other case, do not try to get the inputTags
113  else:
114  raise Exception()
115 
116  inputTags = dataCursor.fetchall()
117  if len(inputTags) == 0:
118  raise Exception()
119  inputTags = zip(*inputTags)[0]
120 
121  except Exception:
122  inputTags = []
123 
124  if len(inputTags) == 0:
125  print '\nI could not find any input tag in your data file, but you can still specify one manually.'
126 
127  inputTag = getInputRepeat(
128  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
129 
130  else:
131  print '\nI found the following input tags in your SQLite data file:'
132  for (index, inputTag) in enumerate(inputTags):
133  print ' %s) %s' % (index, inputTag)
134 
135  inputTag = getInputChoose(inputTags, '0',
136  '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
137 
138  destinationDatabase = ''
139  ntry = 0
140  while ( destinationDatabase != 'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase != 'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
141  if ntry==0:
142  inputMessage = \
143  '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
144  elif ntry==1:
145  inputMessage = \
146  '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
147 \ndestinationDatabase: '
148  else:
149  raise Exception('No valid destination chosen. Bailing out...')
150  destinationDatabase = getInputRepeat(inputMessage)
151  ntry += 1
152 
153  while True:
154  since = getInput('',
155  '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
156  if not since:
157  since = None
158  break
159  else:
160  try:
161  since = int(since)
162  break
163  except ValueError:
164  logging.error('The since value has to be an integer or empty (null).')
165 
166  userText = getInput('',
167  '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
168 
169  destinationTags = {}
170  while True:
171  destinationTag = getInput('',
172  '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
173  if not destinationTag:
174  if len(destinationTags) == 0:
175  logging.error('There must be at least one destination tag.')
176  continue
177  break
178 
179  if destinationTag in destinationTags:
180  logging.warning(
181  'You already added this destination tag. Overwriting the previous one with this new one.')
182 
183  destinationTags[destinationTag] = {
184  }
185 
186  metadata = {
187  'destinationDatabase': destinationDatabase,
188  'destinationTags': destinationTags,
189  'inputTag': inputTag,
190  'since': since,
191  'userText': userText,
192  }
193 
194  metadata = json.dumps(metadata, sort_keys=True, indent=4)
195  print '\nThis is the generated metadata:\n%s' % metadata
196 
197  if getInput('n',
198  '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() == 'y':
199  break
200  logging.info('Saving generated metadata in %s...', metadataFilename)
201  with open(metadataFilename, 'wb') as metadataFile:
202  metadataFile.write(metadata)
def uploadConditions.testTier0Upload ( )

Definition at line 842 of file uploadConditions.py.

References uploadTier0Files().

842 
843 def testTier0Upload():
844 
845  global defaultNetrcHost
846 
847  (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
848 
849  filenames = ['testFiles/localSqlite-top2']
850 
851  uploadTier0Files(filenames, username, password, cookieFileName = None)
852 
def uploadConditions.uploadAllFiles (   options,
  arguments 
)

Definition at line 613 of file uploadConditions.py.

References authenticateUser(), getInput(), and runWizard().

Referenced by main().

613 
614 def uploadAllFiles(options, arguments):
615 
616  ret = {}
617  ret['status'] = 0
618 
619  # Check that we can read the data and metadata files
620  # If the metadata file does not exist, start the wizard
621  for filename in arguments:
622  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
623  basename = os.path.basename(basepath)
624  dataFilename = '%s.db' % basepath
625  metadataFilename = '%s.txt' % basepath
626 
627  logging.info('Checking %s...', basename)
628 
629  # Data file
630  try:
631  with open(dataFilename, 'rb') as dataFile:
632  pass
633  except IOError as e:
634  errMsg = 'Impossible to open SQLite data file %s' %dataFilename
635  logging.error( errMsg )
636  ret['status'] = -3
637  ret['error'] = errMsg
638  return ret
639 
640  # Check the data file
641  empty = True
642  try:
643  dbcon = sqlite3.connect( dataFilename )
644  dbcur = dbcon.cursor()
645  dbcur.execute('SELECT * FROM IOV')
646  rows = dbcur.fetchall()
647  for r in rows:
648  empty = False
649  dbcon.close()
650  if empty:
651  errMsg = 'The input SQLite data file %s contains no data.' %dataFilename
652  logging.error( errMsg )
653  ret['status'] = -4
654  ret['error'] = errMsg
655  return ret
656  except Exception as e:
657  errMsg = 'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
658  logging.error( errMsg )
659  ret['status'] = -5
660  ret['error'] = errMsg
661  return ret
662 
663  # Metadata file
664  try:
665  with open(metadataFilename, 'rb') as metadataFile:
666  pass
667  except IOError as e:
668  if e.errno != errno.ENOENT:
669  errMsg = 'Impossible to open file %s (for other reason than not existing)' %metadataFilename
670  logging.error( errMsg )
671  ret['status'] = -4
672  ret['error'] = errMsg
673  return ret
674 
675  if getInput('y', '\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() != 'y':
676  errMsg = 'Metadata file %s does not exist' %metadataFilename
677  logging.error( errMsg )
678  ret['status'] = -5
679  ret['error'] = errMsg
680  return ret
681  # Wizard
682  runWizard(basename, dataFilename, metadataFilename)
683 
684  # Upload files
685  try:
686  dropBox = ConditionsUploader(options.hostname, options.urlTemplate)
687 
688  # Authentication
689  if not authenticateUser(dropBox, options):
690  logging.error("Error authenticating user. Aborting.")
691  return { 'status' : -2, 'error' : "Error authenticating user. Aborting." }
692 
693  # At this point we must be authenticated
694  dropBox._checkForUpdates()
695 
696  results = {}
697  for filename in arguments:
698  backend = options.backend
699  basepath = filename.rsplit('.db', 1)[0].rsplit('.txt', 1)[0]
700  metadataFilename = '%s.txt' % basepath
701  with open(metadataFilename, 'rb') as metadataFile:
702  metadata = json.load( metadataFile )
703  # When dest db = prep the hostname has to be set to dev.
704  forceHost = False
705  destDb = metadata['destinationDatabase']
706  if destDb.startswith('oracle://cms_orcon_prod') or destDb.startswith('oracle://cms_orcoff_prep'):
707  if destDb.startswith('oracle://cms_orcoff_prep'):
708  dropBox.setHost( defaultDevHostname )
709  dropBox.signInAgain()
710  forceHost = True
711  results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
712  if forceHost:
713  # set back the hostname to the original global setting
714  dropBox.setHost( options.hostname )
715  dropBox.signInAgain()
716  else:
717  results[filename] = False
718  logging.error("DestinationDatabase %s is not valid. Skipping the upload." %destDb)
719  if not results[filename]:
720  if ret['status']<0:
721  ret['status'] = 0
722  ret['status'] += 1
723  ret['files'] = results
724  logging.debug("all files processed, logging out now.")
725 
726  dropBox.signOut()
727 
728  except HTTPError as e:
729  logging.error('got HTTP error: %s', str(e))
730  return { 'status' : -1, 'error' : str(e) }
731 
732  return ret
def uploadConditions.uploadTier0Files (   filenames,
  username,
  password,
  cookieFileName = None 
) 
Uploads a bunch of files coming from Tier0.
This has the following requirements:
    * Username/Password based authentication.
    * Uses the online backend.
    * Ignores errors related to the upload/content (e.g. duplicated file).

Definition at line 733 of file uploadConditions.py.

Referenced by testTier0Upload().

733 
734 def uploadTier0Files(filenames, username, password, cookieFileName = None):
735  '''Uploads a bunch of files coming from Tier0.
736  This has the following requirements:
737  * Username/Password based authentication.
738  * Uses the online backend.
739  * Ignores errors related to the upload/content (e.g. duplicated file).
740  '''
741 
742  dropBox = ConditionsUploader()
743 
744  dropBox.signIn(username, password)
745 
746  for filename in filenames:
747  try:
748  result = dropBox.uploadFile(filename, backend = 'test')
749  except HTTPError as e:
750  if e.code == 400:
751  # 400 Bad Request: This is an exception related to the upload
752  # being wrong for some reason (e.g. duplicated file).
753  # Since for Tier0 this is not an issue, continue
754  logging.error('HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
755  continue
756 
757  # In any other case, re-raise.
758  raise
759 
760  #-toDo: add a flag to say if we should retry or not. So far, all retries are done server-side (Tier-0),
761  # if we flag as failed any retry would not help and would result in the same error (e.g.
762  # when a file with an identical hash is uploaded again)
763  #-review(2015-09-25): get feedback from tests at Tier-0 (action: AP)
764 
765  if not result: # dropbox reported an error when uploading, do not retry.
766  logging.error('Error from dropbox, upload-related, skipping.')
767  continue
768 
769  dropBox.signOut()
770 

Variable Documentation

string uploadConditions.__author__ = 'Andreas Pfeiffer'

Definition at line 6 of file uploadConditions.py.

string uploadConditions.__copyright__ = 'Copyright 2015, CERN CMS'

Definition at line 7 of file uploadConditions.py.

list uploadConditions.__credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']

Definition at line 8 of file uploadConditions.py.

string uploadConditions.__email__ = 'andreas.pfeiffer@cern.ch'

Definition at line 11 of file uploadConditions.py.

string uploadConditions.__license__ = 'Unknown'

Definition at line 9 of file uploadConditions.py.

string uploadConditions.__maintainer__ = 'Andreas Pfeiffer'

Definition at line 10 of file uploadConditions.py.

int uploadConditions.__version__ = 1

Definition at line 12 of file uploadConditions.py.

string uploadConditions.CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'

Definition at line 221 of file uploadConditions.py.

string uploadConditions.defaultBackend = 'online'

Definition at line 27 of file uploadConditions.py.

string uploadConditions.defaultDevHostname = 'cms-conddb-dev.cern.ch'

Definition at line 29 of file uploadConditions.py.

string uploadConditions.defaultHostname = 'cms-conddb-prod.cern.ch'

Definition at line 28 of file uploadConditions.py.

string uploadConditions.defaultNetrcHost = 'ConditionUploader'

Definition at line 32 of file uploadConditions.py.

string uploadConditions.defaultTemporaryFile = 'upload.tar.bz2'

Definition at line 31 of file uploadConditions.py.

string uploadConditions.defaultUrlTemplate = 'https://%s/cmsDbUpload/'

Definition at line 30 of file uploadConditions.py.

string uploadConditions.defaultWorkflow = 'offline'

Definition at line 33 of file uploadConditions.py.

Generated for CMSSW Reference Manual by doxygen 1.8.5