upload_popcon Namespace Reference

Classes
class	CERNSSOError
class	DropBox
class	HTTP
class	HTTPError

Functions
def	_getCERNSSOCookies
def	addToTarFile
def	getInput
def	getInputChoose
def	getInputRepeat
def	getInputWorkflow

Variables
string	__author__ = 'Miguel Ojeda'
string	__copyright__ = 'Copyright 2012, CERN CMS'
list	__credits__ = ['Giacomo Govi', 'Salvatore Di Guida', 'Miguel Ojeda', 'Andreas Pfeiffer']
string	__email__ = 'mojedasa@cern.ch'
string	__license__ = 'Unknown'
string	__maintainer__ = 'Miguel Ojeda'
int	__version__ = 6
string	defaultBackend = 'online'
string	defaultHostname = 'cms-conddb-prod.cern.ch'
string	defaultNetrcHost = 'DropBox'
string	defaultTemporaryFile = 'upload.tar.bz2'
string	defaultUrlTemplate = 'https://%s/dropBox/'
string	defaultWorkflow = 'offline'

Function Documentation

def upload_popcon._getCERNSSOCookies (   url,   secureTarget = True,   secure = True  )

private

Returns the required CERN SSO cookies for a URL using Kerberos.

They can be used with any HTTP client (libcurl, wget, urllib...).

If you wish to make secure SSL connections to the CERN SSO
(i.e. verify peers/hosts), you may need to install the CERN-CA-certs package.
Use secure == False to skip this (i.e. this is the same as curl
-k/--insecure). Not recommended: tell users to install them or use lxplus6.

The same way, if you have a self-signed certificate in your target URL
you can use secureTarget == False as well. Note that this option
is provided in order to be able to use a secure SSL connection to CERN SSO,
even if the connection to your target URL is not secure. Note that
you will probably need the CERN-CA-certs package after you get a certificate
signed by the CERN CA (https://cern.ch/ca), even if you did not need it
for the CERN SSO.

Note that this method *does* a query to the given URL if successful.

This was implemented outside the HTTP class for two main reasons:

    * The only thing needed to use CERN SSO is the cookies, therefore
      this function is useful alone as well (e.g. as a simple replacement
      of the cern-get-sso-cookie script or as a Python port of
      the WWW::CERNSSO::Auth Perl package -- this one does not write
      any file and can be used in-memory, by the way).

    * We do not need to use the curl handler of the HTTP class.
      This way we do not overwrite any options in that one and we use
      only a temporary one here for getting the cookie.

TODO: Support also Certificate/Key authentication.
TODO: Support also Username/Password authentication.
TODO: Review the error paths.
TODO: Why PERLSESSID was used in the original code?
TODO: Retry if timeouts are really common (?)

Definition at line 53 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.addCERNSSOCookies().

def _getCERNSSOCookies(url, secureTarget = True, secure = True):
    '''Returns the required CERN SSO cookies for a URL using Kerberos.

    They can be used with any HTTP client (libcurl, wget, urllib...).

    If you wish to make secure SSL connections to the CERN SSO
    (i.e. verify peers/hosts), you may need to install the CERN-CA-certs package.
    Use secure == False to skip this (i.e. this is the same as curl
    -k/--insecure). Not recommended: tell users to install them or use lxplus6.

    The same way, if you have a self-signed certificate in your target URL
    you can use secureTarget == False as well. Note that this option
    is provided in order to be able to use a secure SSL connection to CERN SSO,
    even if the connection to your target URL is not secure. Note that
    you will probably need the CERN-CA-certs package after you get a certificate
    signed by the CERN CA (https://cern.ch/ca), even if you did not need it
    for the CERN SSO.

    Note that this method *does* a query to the given URL if successful.

    This was implemented outside the HTTP class for two main reasons:

        * The only thing needed to use CERN SSO is the cookies, therefore
          this function is useful alone as well (e.g. as a simple replacement
          of the cern-get-sso-cookie script or as a Python port of
          the WWW::CERNSSO::Auth Perl package -- this one does not write
          any file and can be used in-memory, by the way).

        * We do not need to use the curl handler of the HTTP class.
          This way we do not overwrite any options in that one and we use
          only a temporary one here for getting the cookie.

    TODO: Support also Certificate/Key authentication.
    TODO: Support also Username/Password authentication.
    TODO: Review the error paths.
    TODO: Why PERLSESSID was used in the original code?
    TODO: Retry if timeouts are really common (?)
    '''

    def perform():
        response = cStringIO.StringIO()
        curl.setopt(curl.WRITEFUNCTION, response.write)
        curl.perform()
        code = curl.getinfo(curl.RESPONSE_CODE)
        response = response.getvalue()
        effectiveUrl = curl.getinfo(curl.EFFECTIVE_URL)
        return (code, response, effectiveUrl)

    # These constants and the original code came from the official CERN
    # cern-get-sso-cookie script and WWW::CERNSSO::Auth Perl package.
    VERSION = '0.4.2'
    CERN_SSO_CURL_USER_AGENT_KRB = 'curl-sso-kerberos/%s' % VERSION
    CERN_SSO_CURL_AUTHERR = 'HTTP Error 401.2 - Unauthorized'
    CERN_SSO_CURL_ADFS_EP = '/adfs/ls/auth'
    CERN_SSO_CURL_ADFS_SIGNIN = 'wa=wsignin1.0'
    CERN_SSO_CURL_CAPATH = '/etc/pki/tls/certs'

    logging.debug('secureTarget = %s', secureTarget)
    logging.debug('secure = %s', secure)

    curl = pycurl.Curl()

    # Store the cookies in memory, which we will retreive later on
    curl.setopt(curl.COOKIEFILE, '')

    # This should not be needed, but sometimes requests hang 'forever'
    curl.setopt(curl.TIMEOUT, 10)
    curl.setopt(curl.CONNECTTIMEOUT, 10)

    # Ask curl to use Kerberos5 authentication
    curl.setopt(curl.USERAGENT, CERN_SSO_CURL_USER_AGENT_KRB)
    curl.setopt(curl.HTTPAUTH, curl.HTTPAUTH_GSSNEGOTIATE)
    curl.setopt(curl.USERPWD, ':')

    # Follow location (and send the password along to other hosts,
    # although we do not really send any password)
    curl.setopt(curl.FOLLOWLOCATION, 1)
    curl.setopt(curl.UNRESTRICTED_AUTH, 1)

    # We do not need the headers
    curl.setopt(curl.HEADER, 0)

    # The target server has a valid certificate
    if secureTarget:
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        curl.setopt(curl.CAPATH, CERN_SSO_CURL_CAPATH)
    else:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    # Fetch the url
    logging.debug('Connecting to %s', url)
    curl.setopt(curl.URL, url)
    (code, response, effectiveUrl) = perform()

    if CERN_SSO_CURL_ADFS_EP not in effectiveUrl:
        raise CERNSSOError('Not behind SSO or we already have the cookie.')

    # Do the manual redirection to the IDP
    logging.debug('Redirected to IDP %s', effectiveUrl)

    # The CERN SSO servers have a valid certificate
    if secure:
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        curl.setopt(curl.CAPATH, CERN_SSO_CURL_CAPATH)
    else:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    curl.setopt(curl.URL, effectiveUrl)
    (code, response, effectiveUrl) = perform()

    if CERN_SSO_CURL_AUTHERR in response:
        raise CERNSSOError('Authentication error: Redirected to IDP Authentication error %s' % effectiveUrl)

    match = re.search('form .+?action="([^"]+)"', response)
    if not match:
        raise CERNSSOError('Something went wrong: could not find the expected redirection form (do you have a valid Kerberos ticket? -- see klist and kinit).')

    # Do the JavaScript redirection via the form to the SP
    spUrl = match.groups()[0]
    logging.debug('Redirected (via form) to SP %s', spUrl)

    formPairs = re.findall('input type="hidden" name="([^"]+)" value="([^"]+)"', response)

    # Microsoft ADFS produces broken encoding in auth forms:
    # '<' and '"' are encoded as '&lt;' and '&quot;' however
    # '>' is *not* encoded. Does not matter here though, we just decode.
    htmlParser = HTMLParser.HTMLParser()
    formPairs = [(x[0], htmlParser.unescape(x[1])) for x in formPairs]

    # The target server has a valid certificate
    if secureTarget:
        curl.setopt(curl.SSL_VERIFYPEER, 1)
        curl.setopt(curl.SSL_VERIFYHOST, 2)
        curl.setopt(curl.CAPATH, CERN_SSO_CURL_CAPATH)
    else:
        curl.setopt(curl.SSL_VERIFYPEER, 0)
        curl.setopt(curl.SSL_VERIFYHOST, 0)

    curl.setopt(curl.URL, spUrl)
    curl.setopt(curl.POSTFIELDS, urllib.urlencode(formPairs))
    curl.setopt(curl.POST, 1)
    (code, response, effectiveUrl) = perform()

    if CERN_SSO_CURL_ADFS_SIGNIN in effectiveUrl:
        raise CERNSSOError('Something went wrong: still on the auth page.')

    # Return the cookies
    return curl.getinfo(curl.INFO_COOKIELIST)