uploads.uploader Class Reference

Inheritance diagram for uploads.uploader:

Public Member Functions
def	__init__
def	check_response_for_error_key
def	close_upload_session
def	exit_upload
def	filter_iovs_by_fcsr
def	get_all_hashes
def	get_fcsr_from_server
def	get_hashes_to_send
def	get_tag_dictionary
def	get_upload_session_id
def	send_blob
def	send_metadata
def	send_payloads
def	upload
def	write_server_side_log

Public Attributes
	data_to_send
	hashes_with_no_local_payload
	input_tag
	metadata_source
	server_side_log_file
	sqlite_file_name
	upload_log_file_name
	upload_session_id

Private Attributes
	_debug
	_handle
	_log_data
	_outputter
	_SERVICE_URL
	_testing
	_verbose

Detailed Description

Upload session controller - creates, tracks, and deletes upload sessions on the server.

Definition at line 81 of file uploads.py.

Constructor & Destructor Documentation

def uploads.uploader.__init__	(		self,
			metadata_source = None,
			debug = False,
			verbose = False,
			testing = False,
			server = "https://cms-conddb-dev.cern.ch/cmsDbCondUpload/",
			kwargs
	)

Upload constructor:
Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded.

Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used.

Note: default value of service_url should be changed for production.

Definition at line 86 of file uploads.py.

    def __init__(self, metadata_source=None, debug=False, verbose=False, testing=False, server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", **kwargs):
        """
        Upload constructor:
        Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded.

        Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used.

        Note: default value of service_url should be changed for production.
        """
        # set private variables
        self._debug = debug
        self._verbose = verbose
        self._testing = testing
        # initialise server-side log data as empty string - will be replaced when we get a response back from the server
        self._log_data = ""
        self._SERVICE_URL = server
        self.upload_session_id = None

        # set up client-side log file
        self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id()
        self._handle = open(self.upload_log_file_name, "a")

        # set up client-side logging object
        self._outputter = output(verbose=verbose, log_handle=self._handle)
        self._outputter.write("Using server instance at '%s'." % self._SERVICE_URL)

        # expect a CondDBFW data_source object for metadata_source
        if metadata_source == None:
            # no upload metadat has been given - we cannot continue with the upload
            self.exit_upload("A source of metadata must be given so CondDBFW knows how to upload conditions.")
        else:
            # set up global metadata source variable
            self.metadata_source = metadata_source.data()

        # check for the destination tag
        # this is required whatever type of upload we're performing
        if self.metadata_source.get("destinationTags") == None:
            self.exit_upload("No destination Tag was given.")
        else:
            if type(self.metadata_source.get("destinationTags")) == dict and list(self.metadata_source.get("destinationTags").keys())[0] == None:
                self.exit_upload("No destination Tag was given.")

        # make sure a destination database was given
        if self.metadata_source.get("destinationDatabase") == None:
            self.exit_upload("No destination database was given.")

        # get Conditions metadata
        if self.metadata_source.get("sourceDB") == None and self.metadata_source.get("hashToUse") == None:
            """
            If we have neither an sqlite file nor the command line data
            """
            self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\
                            + "\nSee --help for command line argument information.")
        elif self.metadata_source.get("sourceDB") != None:
            """
            We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source
            We now extract the Tag and IOV data from SQLite.  It is added to the dictionary for sending over HTTPs later.
            """

            # make sure we have an input tag to look for in the source db
            self.input_tag = metadata_source.data().get("inputTag")
            if self.input_tag == None:
                self.exit_upload("No input Tag name was given.")

            # set empty dictionary to contain Tag and IOV data from SQLite
            result_dictionary = {}
            self.sqlite_file_name = self.metadata_source["sourceDB"]
            if not(os.path.isfile(self.sqlite_file_name)):
                self.exit_upload("SQLite file '%s' given doesn't exist." % self.sqlite_file_name)
            sqlite_con = querying.connect("sqlite://%s" % os.path.abspath(self.sqlite_file_name))

            self._outputter.write("Getting Tag and IOVs from SQLite database.")

            # query for Tag, check for existence, then convert to dictionary
            tag = sqlite_con.tag(name=self.input_tag)
            if tag == None:
                self.exit_upload("The source Tag '%s' you gave was not found in the SQLite file." % self.input_tag)
            tag = tag.as_dicts(convert_timestamps=True)

            # query for IOVs, check for existence, then convert to dictionaries
            iovs = sqlite_con.iov(tag_name=self.input_tag)
            if iovs == None:
                self.exit_upload("No IOVs found in the SQLite file given for Tag '%s'." % self.input_tag)
            iovs = iovs.as_dicts(convert_timestamps=True)
            iovs = [iovs] if type(iovs) != list else iovs

            """
            Finally, get the list of all Payload hashes of IOVs,
            then compute the list of hashes for which there is no Payload for
            this is used later to decide if we can continue the upload if the Payload was not found on the server.
            """
            iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag)
            if iovs_for_hashes.__class__ == data_sources.json_list:
                hashes_of_iovs = iovs_for_hashes.get_members("payload_hash").data()
            else:
                hashes_of_iovs = [iovs_for_hashes.payload_hash]
            self.hashes_with_no_local_payload = [payload_hash for payload_hash in hashes_of_iovs if sqlite_con.payload(hash=payload_hash) == None]

            # close session open on SQLite database file
            sqlite_con.close_session()

        elif metadata_source.data().get("hashToUse") != None:
            """
            Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments).
            We now use Tag and IOV data from command line.  It is added to the dictionary for sending over HTTPs later.
            """

            # set empty dictionary to contain Tag and IOV data from command line
            result_dictionary = {}

            now = to_timestamp(datetime.utcnow())
            # tag dictionary will be taken from the server
            # this does not require any authentication
            tag = self.get_tag_dictionary()
            self.check_response_for_error_key(tag)
            iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\
                    "insertion_time" : now}]

            # hashToUse cannot be stored locally (no sqlite file is given), so register it as not found
            self.hashes_with_no_local_payload = [self.metadata_source["hashToUse"]]

            # Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side

        # if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0
        if tag["time_type"] == "Run":
            for (i, iov) in enumerate(iovs):
                iovs[i]["since"] = iovs[i]["since"] << 32

        result_dictionary = {"inputTagData" : tag, "iovs" : iovs}

        # add command line arguments to dictionary
        # remembering that metadata_source is a json_dict object
        result_dictionary.update(metadata_source.data())

        # store in instance variable
        self.data_to_send = result_dictionary

        # if the since doesn't exist, take the first since from the list of IOVs
        if result_dictionary.get("since") == None:
            result_dictionary["since"] = sorted(iovs, key=lambda iov : iov["since"])[0]["since"]
        elif self.data_to_send["inputTagData"]["time_type"] == "Run":
            # Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing
            self.data_to_send["since"] = self.data_to_send["since"] << 32

        """
        TODO - Settle on a single destination tag format.
        """
        # look for deprecated metadata entries - give warnings
        # Note - we only really support this format
        try:
            if type(result_dictionary["destinationTags"]) == dict:
                self._outputter.write("WARNING: Multiple destination tags in a single metadata source is deprecated.")
        except Exception as e:
            self._outputter.write("ERROR: %s" % str(e))