CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Groups Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
upload_popcon.HTTP Class Reference
Inheritance diagram for upload_popcon.HTTP:

Public Member Functions

def __init__
 
def discardCookies
 
def getCookies
 
def getToken
 
def query
 
def setBaseUrl
 
def setProxy
 
def setRetries
 
def setTimeout
 

Public Attributes

 baseUrl
 
 curl
 
 retries
 
 token
 

Static Public Attributes

tuple retryCodes = frozenset([502, 503])
 

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 64 of file upload_popcon.py.

Constructor & Destructor Documentation

def upload_popcon.HTTP.__init__ (   self)

Definition at line 70 of file upload_popcon.py.

References upload_popcon.HTTP.setBaseUrl(), and upload_popcon.HTTP.setRetries().

70 
71  def __init__(self):
72  self.setBaseUrl()
73  self.setRetries()
74 
75  self.curl = pycurl.Curl()
76  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
77 
78  #-toDo: make sure we have the right options set here to use ssl
79  #-review(2015-09-25): check and see - action: AP
80  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
81  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
82  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
83 
84  self.baseUrl = None
85 
86  self.token = None
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:117
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:98
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.curl
curl
Definition: upload_popcon.py:74
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
upload_popcon.HTTP.__init__
def __init__
Definition: upload_popcon.py:70

Member Function Documentation

def upload_popcon.HTTP.discardCookies (   self) 
Discards cookies.

Definition at line 92 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

92 
93  def discardCookies(self):
94  '''Discards cookies.
95  '''
96  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
97 
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:92
def upload_popcon.HTTP.getCookies (   self) 
Returns the list of cookies.

Definition at line 87 of file upload_popcon.py.

87 
88  def getCookies(self):
89  '''Returns the list of cookies.
90  '''
91  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
upload_popcon.HTTP.getCookies
def getCookies
Definition: upload_popcon.py:87
def upload_popcon.HTTP.getToken (   self,
  username,
  password 
)

Definition at line 132 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, str, and upload_popcon.HTTP.token.

132 
133  def getToken(self, username, password):
134 
135  url = self.baseUrl + 'token'
136 
137  self.curl.setopt(pycurl.URL, url)
138  self.curl.setopt(pycurl.VERBOSE, 0)
139 
140  #-toDo: check if/why these are needed ...
141  #-ap: hmm ...
142  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
143  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
144  #-end hmmm ...
145  #-review(2015-09-25): check and see - action: AP
146 
147 
148  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
149  # self.curl.setopt( self.curl.POST, {})
150  self.curl.setopt(self.curl.HTTPGET, 0)
151 
152  response = cStringIO.StringIO()
153  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
154  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
155 
156  logging.debug('going to connect to server at: %s' % url )
157 
158  self.curl.perform()
159  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
160  logging.debug('got: %s ', str(code))
161 
162  try:
163  self.token = json.loads( response.getvalue() )['token']
164  except Exception as e:
165  logging.error('http::getToken> got error from server: %s ', str(e) )
166  if 'No JSON object could be decoded' in str(e):
167  return None
168  logging.error("error getting token: %s", str(e))
169  return None
170 
171  logging.debug('token: %s', self.token)
172  logging.debug('returning: %s', response.getvalue())
173 
174  return response.getvalue()
upload_popcon.HTTP.getToken
def getToken
Definition: upload_popcon.py:132
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
str
#define str(s)
Definition: TestProcessor.cc:53
def upload_popcon.HTTP.query (   self,
  url,
  data = None,
  files = None,
  keepCookies = True 
) 
Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 175 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.discardCookies(), upload_popcon.HTTP.retries, upload_popcon.HTTP.retryCodes, str, and upload_popcon.HTTP.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

175 
176  def query(self, url, data = None, files = None, keepCookies = True):
177  '''Queries a URL, optionally with some data (dictionary).
178 
179  If no data is specified, a GET request will be used.
180  If some data is specified, a POST request will be used.
181 
182  If files is specified, it must be a dictionary like data but
183  the values are filenames.
184 
185  By default, cookies are kept in-between requests.
186 
187  A HTTPError exception is raised if the response's HTTP code is not 200.
188  '''
189 
190  if not keepCookies:
191  self.discardCookies()
192 
193  url = self.baseUrl + url
194 
195  # make sure the logs are safe ... at least somewhat :)
196  data4log = copy.copy(data)
197  if data4log:
198  if 'password' in data4log.keys():
199  data4log['password'] = '*'
200 
201  retries = [0] + list(self.retries)
202 
203  while True:
204  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
205 
206  time.sleep(retries.pop(0))
207 
208  try:
209  self.curl.setopt(self.curl.URL, url)
210  self.curl.setopt(self.curl.HTTPGET, 1)
211 
212  # from now on we use the token we got from the login
213  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
214  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
215 
216  if data is not None or files is not None:
217  # If there is data or files to send, use a POST request
218 
219  finalData = {}
220 
221  if data is not None:
222  finalData.update(data)
223 
224  if files is not None:
225  for (key, fileName) in files.items():
226  finalData[key] = (self.curl.FORM_FILE, fileName)
227  self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
228 
229  self.curl.setopt(pycurl.VERBOSE, 0)
230 
231  response = cStringIO.StringIO()
232  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
233  self.curl.perform()
234 
235  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
236 
237  if code in self.retryCodes and len(retries) > 0:
238  logging.debug('Retrying since we got the %s error code...', code)
239  continue
240 
241  if code != 200:
242  raise HTTPError(code, response.getvalue())
243 
244  return response.getvalue()
245 
246  except pycurl.error as e:
247  if len(retries) == 0:
248  raise e
249  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
250 
251 # common/http.py end
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:92
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
upload_popcon.HTTPError
Definition: upload_popcon.py:44
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:130
upload_popcon.HTTP.query
def query
Definition: upload_popcon.py:175
upload_popcon.HTTP.retryCodes
tuple retryCodes
Definition: upload_popcon.py:68
str
#define str(s)
Definition: TestProcessor.cc:53
def upload_popcon.HTTP.setBaseUrl (   self,
  baseUrl = '' 
) 
Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 98 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

98 
99  def setBaseUrl(self, baseUrl = ''):
100  '''Allows to set a base URL which will be prefixed to all the URLs
101  that will be queried later.
102  '''
103  self.baseUrl = baseUrl
104 
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:98
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
def upload_popcon.HTTP.setProxy (   self,
  proxy = '' 
) 
Allows to set a proxy.

Definition at line 105 of file upload_popcon.py.

105 
106  def setProxy(self, proxy = ''):
107  '''Allows to set a proxy.
108  '''
109  self.curl.setopt(self.curl.PROXY, proxy)
110 
upload_popcon.HTTP.setProxy
def setProxy
Definition: upload_popcon.py:105
def upload_popcon.HTTP.setRetries (   self,
  retries = () 
) 
Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 117 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

117 
118  def setRetries(self, retries = ()):
119  '''Allows to set retries.
120 
121  The retries are a sequence of the seconds to wait per retry.
122 
123  The retries are done on:
124  * PyCurl errors (includes network problems, e.g. not being able
125  to connect to the host).
126  * 502 Bad Gateway (for the moment, to avoid temporary
127  Apache-CherryPy issues).
128  * 503 Service Temporarily Unavailable (for when we update
129  the frontends).
130  '''
131  self.retries = retries
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:117
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:130
def upload_popcon.HTTP.setTimeout (   self,
  timeout = 0 
) 
Allows to set a timeout.

Definition at line 111 of file upload_popcon.py.

111 
112  def setTimeout(self, timeout = 0):
113  '''Allows to set a timeout.
114  '''
115  self.curl.setopt(self.curl.TIMEOUT, timeout)
116 
upload_popcon.HTTP.setTimeout
def setTimeout
Definition: upload_popcon.py:111

Member Data Documentation

upload_popcon.HTTP.baseUrl

Definition at line 83 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setBaseUrl(), and uploadConditions.HTTP.setBaseUrl().

upload_popcon.HTTP.curl

Definition at line 74 of file upload_popcon.py.

upload_popcon.HTTP.retries

Definition at line 130 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

tuple upload_popcon.HTTP.retryCodes = frozenset([502, 503])
static

Definition at line 68 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

upload_popcon.HTTP.token

Definition at line 85 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), uploadConditions.ConditionsUploader.setHost(), uploadConditions.ConditionsUploader.signIn(), upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

Generated for CMSSW Reference Manual by doxygen 1.8.5