2 '''Script that uploads to the new CMS conditions uploader.
3 Adapted to the new infrastructure from v6 of the upload.py script for the DropBox from Miguel Ojeda.
6 __author__ =
'Andreas Pfeiffer'
7 __copyright__ =
'Copyright 2015, CERN CMS'
8 __credits__ = [
'Giacomo Govi',
'Salvatore Di Guida',
'Miguel Ojeda',
'Andreas Pfeiffer']
9 __license__ =
'Unknown'
10 __maintainer__ =
'Andreas Pfeiffer'
11 __email__ =
'andreas.pfeiffer@cern.ch'
27 defaultBackend =
'online'
28 defaultHostname =
'cms-conddb-prod.cern.ch'
29 defaultDevHostname =
'cms-conddb-dev.cern.ch'
30 defaultUrlTemplate =
'https://%s/cmsDbUpload/'
31 defaultTemporaryFile =
'upload.tar.bz2'
32 defaultNetrcHost =
'ConditionUploader'
33 defaultWorkflow =
'offline'
45 '''Like raw_input() but with a default and automatic strip().
48 answer = raw_input(prompt)
52 return default.strip()
56 '''Like getInput() but tailored to get target workflows (synchronization options).
60 workflow =
getInput(defaultWorkflow, prompt)
62 if workflow
in frozenset([
'offline',
'hlt',
'express',
'prompt',
'pcl']):
65 logging.error(
'Please specify one of the allowed workflows. See above for the explanation on each of them.')
69 '''Makes the user choose from a list of options.
76 return optionsList[int(index)]
78 logging.error(
'Please specify an index of the list (i.e. integer).')
80 logging.error(
'The index you provided is not in the given list.')
84 '''Like raw_input() but repeats if nothing is provided and automatic strip().
88 answer = raw_input(prompt)
92 logging.error(
'You need to provide a value.')
95 def runWizard(basename, dataFilename, metadataFilename):
97 print '''\nWizard for metadata for %s
99 I will ask you some questions to fill the metadata file. For some of the questions there are defaults between square brackets (i.e. []), leave empty (i.e. hit Enter) to use them.''' % basename
103 dataConnection = sqlite3.connect(dataFilename)
104 dataCursor = dataConnection.cursor()
105 dataCursor.execute(
'select name from sqlite_master where type == "table"')
106 tables = set(zip(*dataCursor.fetchall())[0])
110 dataCursor.execute(
'select NAME from TAG')
115 inputTags = dataCursor.fetchall()
116 if len(inputTags) == 0:
118 inputTags = zip(*inputTags)[0]
123 if len(inputTags) == 0:
124 print '\nI could not find any input tag in your data file, but you can still specify one manually.'
127 '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. BeamSpotObject_ByRun\ninputTag: ')
130 print '\nI found the following input tags in your SQLite data file:'
131 for (index, inputTag)
in enumerate(inputTags):
132 print ' %s) %s' % (index, inputTag)
135 '\nWhich is the input tag (i.e. the tag to be read from the SQLite data file)?\ne.g. 0 (you select the first in the list)\ninputTag [0]: ')
137 destinationDatabase =
''
139 while ( destinationDatabase !=
'oracle://cms_orcon_prod/CMS_CONDITIONS' and destinationDatabase !=
'oracle://cms_orcoff_prep/CMS_CONDITIONS' ):
142 '\nWhich is the destination database where the tags should be exported? \nPossible choices: oracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \ndestinationDatabase: '
145 '\nPlease choose one of the two valid destinations: \noracle://cms_orcon_prod/CMS_CONDITIONS (for prod) or oracle://cms_orcoff_prep/CMS_CONDITIONS (for prep) \
146 \ndestinationDatabase: '
148 raise Exception(
'No valid destination chosen. Bailing out...')
154 '\nWhich is the given since? (if not specified, the one from the SQLite data file will be taken -- note that even if specified, still this may not be the final since, depending on the synchronization options you select later: if the synchronization target is not offline, and the since you give is smaller than the next possible one (i.e. you give a run number earlier than the one which will be started/processed next in prompt/hlt/express), the DropBox will move the since ahead to go to the first safe run instead of the value you gave)\ne.g. 1234\nsince []: ')
163 logging.error(
'The since value has to be an integer or empty (null).')
166 '\nWrite any comments/text you may want to describe your request\ne.g. Muon alignment scenario for...\nuserText []: ')
171 '\nWhich is the next destination tag to be added (leave empty to stop)?\ne.g. BeamSpotObjects_PCL_byRun_v0_offline\ndestinationTag []: ')
172 if not destinationTag:
173 if len(destinationTags) == 0:
174 logging.error(
'There must be at least one destination tag.')
178 if destinationTag
in destinationTags:
180 'You already added this destination tag. Overwriting the previous one with this new one.')
182 destinationTags[destinationTag] = {
186 'destinationDatabase': destinationDatabase,
187 'destinationTags': destinationTags,
188 'inputTag': inputTag,
190 'userText': userText,
193 metadata = json.dumps(metadata, sort_keys=
True, indent=4)
194 print '\nThis is the generated metadata:\n%s' % metadata
197 '\nIs it fine (i.e. save in %s and *upload* the conditions if this is the latest file)?\nAnswer [n]: ' % metadataFilename).lower() ==
'y':
199 logging.info(
'Saving generated metadata in %s...', metadataFilename)
200 with open(metadataFilename,
'wb')
as metadataFile:
201 metadataFile.write(metadata)
204 '''A common HTTP exception.
206 self.code is the response HTTP code as an integer.
207 self.response is the response body (i.e. page).
216 self.
args = (response.split(
'<p>')[1].
split(
'</p>')[0], )
221 CERN_SSO_CURL_CAPATH =
'/etc/pki/tls/certs'
224 '''Class used for querying URLs using the HTTP protocol.
227 retryCodes = frozenset([502, 503])
234 self.curl.setopt(self.curl.COOKIEFILE,
'')
239 self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
240 self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
247 '''Returns the list of cookies.
249 return self.curl.getinfo(self.curl.INFO_COOKIELIST)
254 self.curl.setopt(self.curl.COOKIELIST,
'ALL')
258 '''Allows to set a base URL which will be prefixed to all the URLs
259 that will be queried later.
265 '''Allows to set a proxy.
267 self.curl.setopt(self.curl.PROXY, proxy)
271 '''Allows to set a timeout.
273 self.curl.setopt(self.curl.TIMEOUT, timeout)
277 '''Allows to set retries.
279 The retries are a sequence of the seconds to wait per retry.
281 The retries are done on:
282 * PyCurl errors (includes network problems, e.g. not being able
283 to connect to the host).
284 * 502 Bad Gateway (for the moment, to avoid temporary
285 Apache-CherryPy issues).
286 * 503 Service Temporarily Unavailable (for when we update
295 self.curl.setopt(pycurl.URL, url)
296 self.curl.setopt(pycurl.VERBOSE, 0)
306 self.curl.setopt(pycurl.HTTPHEADER, [
'Accept: application/json'])
308 self.curl.setopt(self.curl.HTTPGET, 0)
310 response = cStringIO.StringIO()
311 self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
312 self.curl.setopt(pycurl.USERPWD,
'%s:%s' % (username, password) )
314 logging.debug(
'going to connect to server at: %s' % url )
317 code = self.curl.getinfo(pycurl.RESPONSE_CODE)
318 logging.debug(
'got: %s ', str(code))
321 self.
token = json.loads( response.getvalue() )[
'token']
322 except Exception
as e:
323 logging.error(
'http::getToken> got error from server: %s ', str(e) )
324 if 'No JSON object could be decoded' in str(e):
326 logging.error(
"error getting token: %s", str(e))
329 logging.debug(
'token: %s', self.
token)
330 logging.debug(
'returning: %s', response.getvalue())
332 return response.getvalue()
334 def query(self, url, data = None, files = None, keepCookies = True):
335 '''Queries a URL, optionally with some data (dictionary).
337 If no data is specified, a GET request will be used.
338 If some data is specified, a POST request will be used.
340 If files is specified, it must be a dictionary like data but
341 the values are filenames.
343 By default, cookies are kept in-between requests.
345 A HTTPError exception is raised if the response's HTTP code is not 200.
354 data4log = copy.copy(data)
356 if 'password' in data4log.keys():
357 data4log[
'password'] =
'*'
362 logging.debug(
'Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
364 time.sleep(retries.pop(0))
367 self.curl.setopt(self.curl.URL, url)
368 self.curl.setopt(self.curl.HTTPGET, 1)
371 self.curl.setopt(pycurl.USERPWD,
'%s:""' % ( str(self.
token), ) )
372 self.curl.setopt(pycurl.HTTPHEADER, [
'Accept: application/json'])
374 if data
is not None or files
is not None:
380 finalData.update(data)
382 if files
is not None:
383 for (key, fileName)
in files.items():
384 finalData[key] = (self.curl.FORM_FILE, fileName)
385 self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
387 self.curl.setopt(pycurl.VERBOSE, 0)
389 response = cStringIO.StringIO()
390 self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
393 code = self.curl.getinfo(self.curl.RESPONSE_CODE)
395 if code
in self.
retryCodes and len(retries) > 0:
396 logging.debug(
'Retrying since we got the %s error code...', code)
400 raise HTTPError(code, response.getvalue())
402 return response.getvalue()
404 except pycurl.error
as e:
405 if len(retries) == 0:
407 logging.debug(
'Retrying since we got the %s pycurl exception...', str(e))
412 tarInfo = tarFile.gettarinfo(fileobj = fileobj, arcname = arcname)
414 tarInfo.uid = tarInfo.gid = tarInfo.mtime = 0
415 tarInfo.uname = tarInfo.gname =
'root'
416 tarFile.addfile(tarInfo, fileobj)
419 '''Upload conditions to the CMS conditions uploader service.
422 def __init__(self, hostname = defaultHostname, urlTemplate = defaultUrlTemplate):
436 if socket.getfqdn().strip().endswith(
'.cms'):
437 self.http.setProxy(
'https://cmsproxy.cms:3128/')
439 '''Signs in the server.
442 logging.info(
'%s: Signing in user %s ...', self.
hostname, username)
444 self.
token = self.http.getToken(username, password)
445 except Exception
as e:
446 logging.error(
"Caught exception when trying to get token for user %s from %s: %s" % (username, self.
hostname, str(e)) )
450 logging.error(
"could not get token for user %s from %s" % (username, self.
hostname) )
453 logging.debug(
"got: '%s'", str(self.
token) )
462 '''Signs out the server.
465 logging.info(
'%s: Signing out...', self.
hostname)
471 '''Updates this script, if a new version is found.
474 logging.debug(
'%s: Checking if a newer version of this script is available ...', self.
hostname)
475 version = int(self.http.query(
'getUploadScriptVersion'))
477 if version <= __version__:
478 logging.debug(
'%s: Script is up-to-date.', self.
hostname)
481 logging.info(
'%s: Updating to a newer version (%s) than the current one (%s): downloading ...', self.
hostname, version, __version__)
483 uploadScript = self.http.query(
'getUploadScript')
487 logging.info(
'%s: ... saving the new version ...', self.
hostname)
488 with open(sys.argv[0],
'wb')
as f:
489 f.write(uploadScript)
491 logging.info(
'%s: ... executing the new version...', self.
hostname)
492 os.execl(sys.executable, *([sys.executable] + sys.argv))
495 def uploadFile(self, filename, backend = defaultBackend, temporaryFile = defaultTemporaryFile):
496 '''Uploads a file to the dropBox.
498 The filename can be without extension, with .db or with .txt extension.
499 It will be stripped and then both .db and .txt files are used.
502 basepath = filename.rsplit(
'.db', 1)[0].rsplit(
'.txt', 1)[0]
503 basename = os.path.basename(basepath)
505 logging.debug(
'%s: %s: Creating tar file for upload ...', self.
hostname, basename)
508 tarFile = tarfile.open(temporaryFile,
'w:bz2')
510 with open(
'%s.db' % basepath,
'rb')
as data:
512 except Exception
as e:
513 msg =
'Error when creating tar file. \n'
514 msg +=
'Please check that you have write access to the directory you are running,\n'
515 msg +=
'and that you have enough space on this disk (df -h .)\n'
519 with tempfile.NamedTemporaryFile()
as metadata:
520 with open(
'%s.txt' % basepath,
'rb')
as originalMetadata:
521 json.dump(json.load(originalMetadata), metadata, sort_keys =
True, indent = 4)
528 logging.debug(
'%s: %s: Calculating hash...', self.
hostname, basename)
530 fileHash = hashlib.sha1()
531 with open(temporaryFile,
'rb')
as f:
533 data = f.read(4 * 1024 * 1024)
536 fileHash.update(data)
538 fileHash = fileHash.hexdigest()
539 fileInfo = os.stat(temporaryFile)
540 fileSize = fileInfo.st_size
542 logging.debug(
'%s: %s: Hash: %s', self.
hostname, basename, fileHash)
544 logging.info(
'%s: %s: Uploading file (%s, size %s) to the %s backend...', self.
hostname, basename, fileHash, fileSize, backend)
545 os.rename(temporaryFile, fileHash)
547 ret = self.http.query(
'uploadFile',
550 'fileName': basename,
554 'uploadedFile': fileHash,
557 except Exception
as e:
558 logging.error(
'Error from uploading: %s' % str(e))
559 ret = json.dumps( {
"status": -1,
"upload" : {
'itemStatus' : { basename : {
'status':
'failed',
'info':str(e)}}},
"error" : str(e)} )
563 statusInfo = json.loads(ret)[
'upload']
564 logging.debug(
'upload returned: %s', statusInfo )
569 for tag, info
in statusInfo[
'itemStatus'].
items():
570 logging.debug(
'checking tag %s, info %s', tag, str(json.dumps(info, indent=4,sort_keys=
True)) )
571 if 'ok' in info[
'status'].lower() :
573 logging.info(
'tag %s successfully uploaded', tag)
574 if 'skip' in info[
'status'].lower() :
575 skippedTags.append( tag )
576 logging.warning(
'found tag %s to be skipped. reason: \n ... \t%s ', tag, info[
'info'])
577 if 'fail' in info[
'status'].lower() :
578 failedTags.append( tag )
579 logging.error(
'found tag %s failed to upload. reason: \n ... \t%s ', tag, info[
'info'])
581 if len(okTags) > 0: logging.info (
"tags sucessfully uploaded: %s ", str(okTags) )
582 if len(skippedTags) > 0: logging.warning(
"tags SKIPped to upload : %s ", str(skippedTags) )
583 if len(failedTags) > 0: logging.error (
"tags FAILed to upload : %s ", str(failedTags) )
585 fileLogURL =
'https://%s/logs/dropBox/getFileLog?fileHash=%s'
586 logging.info(
'file log at: %s', fileLogURL % (self.
hostname,fileHash))
594 (username, account, password) = netrc.netrc().authenticators(options.netrcHost)
598 'netrc entry "%s" not found: if you wish not to have to retype your password, you can add an entry in your .netrc file. However, beware of the risks of having your password stored as plaintext. Instead.',
602 defaultUsername = getpass.getuser()
603 if defaultUsername
is None:
604 defaultUsername =
'(not found)'
606 username =
getInput(defaultUsername,
'\nUsername [%s]: ' % defaultUsername)
607 password = getpass.getpass(
'Password: ')
610 return dropBox.signIn(username, password)
620 for filename
in arguments:
621 basepath = filename.rsplit(
'.db', 1)[0].rsplit(
'.txt', 1)[0]
622 basename = os.path.basename(basepath)
623 dataFilename =
'%s.db' % basepath
624 metadataFilename =
'%s.txt' % basepath
626 logging.info(
'Checking %s...', basename)
630 with open(dataFilename,
'rb')
as dataFile:
633 errMsg =
'Impossible to open SQLite data file %s' %dataFilename
634 logging.error( errMsg )
636 ret[
'error'] = errMsg
642 dbcon = sqlite3.connect( dataFilename )
643 dbcur = dbcon.cursor()
644 dbcur.execute(
'SELECT * FROM IOV')
645 rows = dbcur.fetchall()
650 errMsg =
'The input SQLite data file %s contains no data.' %dataFilename
651 logging.error( errMsg )
653 ret[
'error'] = errMsg
655 except Exception
as e:
656 errMsg =
'Check on input SQLite data file %s failed: %s' %(dataFilename,str(e))
657 logging.error( errMsg )
659 ret[
'error'] = errMsg
664 with open(metadataFilename,
'rb')
as metadataFile:
667 if e.errno != errno.ENOENT:
668 errMsg =
'Impossible to open file %s (for other reason than not existing)' %metadataFilename
669 logging.error( errMsg )
671 ret[
'error'] = errMsg
674 if getInput(
'y',
'\nIt looks like the metadata file %s does not exist. Do you want me to create it and help you fill it?\nAnswer [y]: ' % metadataFilename).lower() !=
'y':
675 errMsg =
'Metadata file %s does not exist' %metadataFilename
676 logging.error( errMsg )
678 ret[
'error'] = errMsg
681 runWizard(basename, dataFilename, metadataFilename)
689 logging.error(
"Error authenticating user. Aborting.")
690 return {
'status' : -2,
'error' :
"Error authenticating user. Aborting." }
693 dropBox._checkForUpdates()
696 for filename
in arguments:
697 backend = options.backend
698 basepath = filename.rsplit(
'.db', 1)[0].rsplit(
'.txt', 1)[0]
699 metadataFilename =
'%s.txt' % basepath
700 with open(metadataFilename,
'rb')
as metadataFile:
701 metadata = json.load( metadataFile )
704 destDb = metadata[
'destinationDatabase']
705 if destDb.startswith(
'oracle://cms_orcon_prod')
or destDb.startswith(
'oracle://cms_orcoff_prep'):
706 if destDb.startswith(
'oracle://cms_orcoff_prep'):
707 dropBox.setHost( defaultDevHostname )
708 dropBox.signInAgain()
710 results[filename] = dropBox.uploadFile(filename, options.backend, options.temporaryFile)
713 dropBox.setHost( options.hostname )
714 dropBox.signInAgain()
716 results[filename] =
False
717 logging.error(
"DestinationDatabase %s is not valid. Skipping the upload." %destDb)
718 if not results[filename]:
722 ret[
'files'] = results
723 logging.debug(
"all files processed, logging out now.")
727 except HTTPError
as e:
728 logging.error(
'got HTTP error: %s', str(e))
729 return {
'status' : -1,
'error' : str(e) }
734 '''Uploads a bunch of files coming from Tier0.
735 This has the following requirements:
736 * Username/Password based authentication.
737 * Uses the online backend.
738 * Ignores errors related to the upload/content (e.g. duplicated file).
743 dropBox.signIn(username, password)
745 for filename
in filenames:
747 result = dropBox.uploadFile(filename, backend =
'test')
748 except HTTPError
as e:
753 logging.error(
'HTTP Exception 400 Bad Request: Upload-related, skipping. Message: %s', e)
765 logging.error(
'Error from dropbox, upload-related, skipping.')
775 parser = optparse.OptionParser(usage =
776 'Usage: %prog [options] <file> [<file> ...]\n'
779 parser.add_option(
'-d',
'--debug',
783 help =
'Switch on printing debug information. Default: %default',
786 parser.add_option(
'-b',
'--backend',
788 default = defaultBackend,
789 help =
'dropBox\'s backend to upload to. Default: %default',
792 parser.add_option(
'-H',
'--hostname',
794 default = defaultHostname,
795 help =
'dropBox\'s hostname. Default: %default',
798 parser.add_option(
'-u',
'--urlTemplate',
799 dest =
'urlTemplate',
800 default = defaultUrlTemplate,
801 help =
'dropBox\'s URL template. Default: %default',
804 parser.add_option(
'-f',
'--temporaryFile',
805 dest =
'temporaryFile',
806 default = defaultTemporaryFile,
807 help =
'Temporary file that will be used to store the first tar file. Note that it then will be moved to a file with the hash of the file as its name, so there will be two temporary files created in fact. Default: %default',
810 parser.add_option(
'-n',
'--netrcHost',
812 default = defaultNetrcHost,
813 help =
'The netrc host (machine) from where the username and password will be read. Default: %default',
816 (options, arguments) = parser.parse_args()
818 if len(arguments) < 1:
822 logLevel = logging.INFO
824 logLevel = logging.DEBUG
826 format =
'[%(asctime)s] %(levelname)s: %(message)s',
832 if not results.has_key(
'status'):
833 print 'Unexpected error.'
835 ret = results[
'status']
837 print "upload ended with code: %s" %ret
844 global defaultNetrcHost
846 (username, account, password) = netrc.netrc().authenticators(defaultNetrcHost)
848 filenames = [
'testFiles/localSqlite-top2']
853 if __name__ ==
'__main__':
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run