CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
upload_popcon.HTTP Class Reference
Inheritance diagram for upload_popcon.HTTP:

Public Member Functions

def __init__
 
def discardCookies
 
def getCookies
 
def getToken
 
def query
 
def setBaseUrl
 
def setProxy
 
def setRetries
 
def setTimeout
 

Public Attributes

 baseUrl
 
 curl
 
 retries
 
 token
 

Static Public Attributes

tuple retryCodes = frozenset([502, 503])
 

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 63 of file upload_popcon.py.

Constructor & Destructor Documentation

def upload_popcon.HTTP.__init__ (   self)

Definition at line 69 of file upload_popcon.py.

References upload_popcon.HTTP.setBaseUrl(), and upload_popcon.HTTP.setRetries().

69 
70  def __init__(self):
71  self.setBaseUrl()
72  self.setRetries()
73 
74  self.curl = pycurl.Curl()
75  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
76 
77  #-toDo: make sure we have the right options set here to use ssl
78  #-review(2015-09-25): check and see - action: AP
79  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
80  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
81  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
82 
83  self.baseUrl = None
84 
85  self.token = None
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:116
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:97
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:82
upload_popcon.HTTP.curl
curl
Definition: upload_popcon.py:73
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:84
upload_popcon.HTTP.__init__
def __init__
Definition: upload_popcon.py:69

Member Function Documentation

def upload_popcon.HTTP.discardCookies (   self) 
Discards cookies.

Definition at line 91 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

91 
92  def discardCookies(self):
93  '''Discards cookies.
94  '''
95  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
96 
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:91
def upload_popcon.HTTP.getCookies (   self) 
Returns the list of cookies.

Definition at line 86 of file upload_popcon.py.

86 
87  def getCookies(self):
88  '''Returns the list of cookies.
89  '''
90  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
upload_popcon.HTTP.getCookies
def getCookies
Definition: upload_popcon.py:86
def upload_popcon.HTTP.getToken (   self,
  username,
  password 
)

Definition at line 131 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, cond::TagInfo_t.token, and upload_popcon.HTTP.token.

131 
132  def getToken(self, username, password):
133 
134  url = self.baseUrl + 'token'
135 
136  self.curl.setopt(pycurl.URL, url)
137  self.curl.setopt(pycurl.VERBOSE, 0)
138 
139  #-toDo: check if/why these are needed ...
140  #-ap: hmm ...
141  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
142  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
143  #-end hmmm ...
144  #-review(2015-09-25): check and see - action: AP
145 
146 
147  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
148  # self.curl.setopt( self.curl.POST, {})
149  self.curl.setopt(self.curl.HTTPGET, 0)
150 
151  response = cStringIO.StringIO()
152  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
153  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
154 
155  logging.debug('going to connect to server at: %s' % url )
156 
157  self.curl.perform()
158  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
159  logging.debug('got: %s ', str(code))
160 
161  try:
162  self.token = json.loads( response.getvalue() )['token']
163  except Exception as e:
164  logging.error('http::getToken> got error from server: %s ', str(e) )
165  if 'No JSON object could be decoded' in str(e):
166  return None
167  logging.error("error getting token: %s", str(e))
168  return None
169 
170  logging.debug('token: %s', self.token)
171  logging.debug('returning: %s', response.getvalue())
172 
173  return response.getvalue()
upload_popcon.HTTP.getToken
def getToken
Definition: upload_popcon.py:131
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:82
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:84
def upload_popcon.HTTP.query (   self,
  url,
  data = None,
  files = None,
  keepCookies = True 
) 
Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 174 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.discardCookies(), list(), upload_popcon.HTTP.retries, upload_popcon.HTTP.retryCodes, cond::TagInfo_t.token, and upload_popcon.HTTP.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

174 
175  def query(self, url, data = None, files = None, keepCookies = True):
176  '''Queries a URL, optionally with some data (dictionary).
177 
178  If no data is specified, a GET request will be used.
179  If some data is specified, a POST request will be used.
180 
181  If files is specified, it must be a dictionary like data but
182  the values are filenames.
183 
184  By default, cookies are kept in-between requests.
185 
186  A HTTPError exception is raised if the response's HTTP code is not 200.
187  '''
188 
189  if not keepCookies:
190  self.discardCookies()
191 
192  url = self.baseUrl + url
193 
194  # make sure the logs are safe ... at least somewhat :)
195  data4log = copy.copy(data)
196  if data4log:
197  if 'password' in data4log.keys():
198  data4log['password'] = '*'
199 
200  retries = [0] + list(self.retries)
201 
202  while True:
203  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
204 
205  time.sleep(retries.pop(0))
206 
207  try:
208  self.curl.setopt(self.curl.URL, url)
209  self.curl.setopt(self.curl.HTTPGET, 1)
210 
211  # from now on we use the token we got from the login
212  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
213  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
214 
215  if data is not None or files is not None:
216  # If there is data or files to send, use a POST request
217 
218  finalData = {}
219 
220  if data is not None:
221  finalData.update(data)
222 
223  if files is not None:
224  for (key, fileName) in files.items():
225  finalData[key] = (self.curl.FORM_FILE, fileName)
226  self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
227 
228  self.curl.setopt(pycurl.VERBOSE, 0)
229 
230  response = cStringIO.StringIO()
231  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
232  self.curl.perform()
233 
234  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
235 
236  if code in self.retryCodes and len(retries) > 0:
237  logging.debug('Retrying since we got the %s error code...', code)
238  continue
239 
240  if code != 200:
241  raise HTTPError(code, response.getvalue())
242 
243  return response.getvalue()
244 
245  except pycurl.error as e:
246  if len(retries) == 0:
247  raise e
248  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
249 
250 # common/http.py end
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:91
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:82
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:84
upload_popcon.HTTPError
Definition: upload_popcon.py:43
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:129
upload_popcon.HTTP.query
def query
Definition: upload_popcon.py:174
upload_popcon.HTTP.retryCodes
tuple retryCodes
Definition: upload_popcon.py:67
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def upload_popcon.HTTP.setBaseUrl (   self,
  baseUrl = '' 
) 
Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 97 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

97 
98  def setBaseUrl(self, baseUrl = ''):
99  '''Allows to set a base URL which will be prefixed to all the URLs
100  that will be queried later.
101  '''
102  self.baseUrl = baseUrl
103 
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:97
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:82
def upload_popcon.HTTP.setProxy (   self,
  proxy = '' 
) 
Allows to set a proxy.

Definition at line 104 of file upload_popcon.py.

104 
105  def setProxy(self, proxy = ''):
106  '''Allows to set a proxy.
107  '''
108  self.curl.setopt(self.curl.PROXY, proxy)
109 
upload_popcon.HTTP.setProxy
def setProxy
Definition: upload_popcon.py:104
def upload_popcon.HTTP.setRetries (   self,
  retries = () 
) 
Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 116 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

116 
117  def setRetries(self, retries = ()):
118  '''Allows to set retries.
119 
120  The retries are a sequence of the seconds to wait per retry.
121 
122  The retries are done on:
123  * PyCurl errors (includes network problems, e.g. not being able
124  to connect to the host).
125  * 502 Bad Gateway (for the moment, to avoid temporary
126  Apache-CherryPy issues).
127  * 503 Service Temporarily Unavailable (for when we update
128  the frontends).
129  '''
130  self.retries = retries
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:116
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:129
def upload_popcon.HTTP.setTimeout (   self,
  timeout = 0 
) 
Allows to set a timeout.

Definition at line 110 of file upload_popcon.py.

110 
111  def setTimeout(self, timeout = 0):
112  '''Allows to set a timeout.
113  '''
114  self.curl.setopt(self.curl.TIMEOUT, timeout)
115 
upload_popcon.HTTP.setTimeout
def setTimeout
Definition: upload_popcon.py:110

Member Data Documentation

upload_popcon.HTTP.baseUrl

Definition at line 82 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setBaseUrl(), and uploadConditions.HTTP.setBaseUrl().

upload_popcon.HTTP.curl

Definition at line 73 of file upload_popcon.py.

upload_popcon.HTTP.retries

Definition at line 129 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

tuple upload_popcon.HTTP.retryCodes = frozenset([502, 503])
static

Definition at line 67 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

upload_popcon.HTTP.token

Definition at line 84 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

Generated for CMSSW Reference Manual by doxygen 1.8.5