test
CMS 3D CMS Logo
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Pages
List of all members | Public Member Functions | Public Attributes | Static Public Attributes
upload_popcon.HTTP Class Reference
Inheritance diagram for upload_popcon.HTTP:

Public Member Functions

def __init__
 
def discardCookies
 
def getCookies
 
def getToken
 
def query
 
def setBaseUrl
 
def setProxy
 
def setRetries
 
def setTimeout
 

Public Attributes

 baseUrl
 
 curl
 
 retries
 
 token
 

Static Public Attributes

tuple retryCodes = frozenset([502, 503])
 

Detailed Description

Class used for querying URLs using the HTTP protocol.

Definition at line 64 of file upload_popcon.py.

Constructor & Destructor Documentation

def upload_popcon.HTTP.__init__ (   self)

Definition at line 70 of file upload_popcon.py.

References upload_popcon.HTTP.setBaseUrl(), and upload_popcon.HTTP.setRetries().

70 
71  def __init__(self):
72  self.setBaseUrl()
73  self.setRetries()
74 
75  self.curl = pycurl.Curl()
76  self.curl.setopt(self.curl.COOKIEFILE, '') # in memory
77 
78  #-toDo: make sure we have the right options set here to use ssl
79  #-review(2015-09-25): check and see - action: AP
80  # self.curl.setopt(self.curl.SSL_VERIFYPEER, 1)
81  self.curl.setopt(self.curl.SSL_VERIFYPEER, 0)
82  self.curl.setopt(self.curl.SSL_VERIFYHOST, 2)
83 
84  self.baseUrl = None
85 
86  self.token = None
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:117
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:98
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.curl
curl
Definition: upload_popcon.py:74
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
upload_popcon.HTTP.__init__
def __init__
Definition: upload_popcon.py:70

Member Function Documentation

def upload_popcon.HTTP.discardCookies (   self) 
Discards cookies.

Definition at line 92 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

92 
93  def discardCookies(self):
94  '''Discards cookies.
95  '''
96  self.curl.setopt(self.curl.COOKIELIST, 'ALL')
97 
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:92
def upload_popcon.HTTP.getCookies (   self) 
Returns the list of cookies.

Definition at line 87 of file upload_popcon.py.

87 
88  def getCookies(self):
89  '''Returns the list of cookies.
90  '''
91  return self.curl.getinfo(self.curl.INFO_COOKIELIST)
upload_popcon.HTTP.getCookies
def getCookies
Definition: upload_popcon.py:87
def upload_popcon.HTTP.getToken (   self,
  username,
  password 
)

Definition at line 132 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, cond::TagInfo_t.token, and upload_popcon.HTTP.token.

132 
133  def getToken(self, username, password):
134 
135  url = self.baseUrl + 'token'
136 
137  self.curl.setopt(pycurl.URL, url)
138  self.curl.setopt(pycurl.VERBOSE, 0)
139 
140  #-toDo: check if/why these are needed ...
141  #-ap: hmm ...
142  # self.curl.setopt(pycurl.DNS_CACHE_TIMEOUT, 0)
143  # self.curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
144  #-end hmmm ...
145  #-review(2015-09-25): check and see - action: AP
146 
147 
148  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
149  # self.curl.setopt( self.curl.POST, {})
150  self.curl.setopt(self.curl.HTTPGET, 0)
151 
152  response = cStringIO.StringIO()
153  self.curl.setopt(pycurl.WRITEFUNCTION, response.write)
154  self.curl.setopt(pycurl.USERPWD, '%s:%s' % (username, password) )
155 
156  logging.debug('going to connect to server at: %s' % url )
157 
158  self.curl.perform()
159  code = self.curl.getinfo(pycurl.RESPONSE_CODE)
160  logging.debug('got: %s ', str(code))
161 
162  try:
163  self.token = json.loads( response.getvalue() )['token']
164  except Exception as e:
165  logging.error('http::getToken> got error from server: %s ', str(e) )
166  if 'No JSON object could be decoded' in str(e):
167  return None
168  logging.error("error getting token: %s", str(e))
169  return None
170 
171  logging.debug('token: %s', self.token)
172  logging.debug('returning: %s', response.getvalue())
173 
174  return response.getvalue()
upload_popcon.HTTP.getToken
def getToken
Definition: upload_popcon.py:132
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
def upload_popcon.HTTP.query (   self,
  url,
  data = None,
  files = None,
  keepCookies = True 
) 
Queries a URL, optionally with some data (dictionary).

If no data is specified, a GET request will be used.
If some data is specified, a POST request will be used.

If files is specified, it must be a dictionary like data but
the values are filenames.

By default, cookies are kept in-between requests.

A HTTPError exception is raised if the response's HTTP code is not 200.

Definition at line 175 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl, upload_popcon.HTTP.discardCookies(), list(), upload_popcon.HTTP.retries, upload_popcon.HTTP.retryCodes, cond::TagInfo_t.token, and upload_popcon.HTTP.token.

Referenced by production_tasks.BaseDataset.run(), and edmIntegrityCheck.IntegrityCheck.test().

175 
176  def query(self, url, data = None, files = None, keepCookies = True):
177  '''Queries a URL, optionally with some data (dictionary).
178 
179  If no data is specified, a GET request will be used.
180  If some data is specified, a POST request will be used.
181 
182  If files is specified, it must be a dictionary like data but
183  the values are filenames.
184 
185  By default, cookies are kept in-between requests.
186 
187  A HTTPError exception is raised if the response's HTTP code is not 200.
188  '''
189 
190  if not keepCookies:
191  self.discardCookies()
192 
193  url = self.baseUrl + url
194 
195  # make sure the logs are safe ... at least somewhat :)
196  data4log = copy.copy(data)
197  if data4log:
198  if 'password' in data4log.keys():
199  data4log['password'] = '*'
200 
201  retries = [0] + list(self.retries)
202 
203  while True:
204  logging.debug('Querying %s with data %s and files %s (retries left: %s, current sleep: %s)...', url, data4log, files, len(retries), retries[0])
205 
206  time.sleep(retries.pop(0))
207 
208  try:
209  self.curl.setopt(self.curl.URL, url)
210  self.curl.setopt(self.curl.HTTPGET, 1)
211 
212  # from now on we use the token we got from the login
213  self.curl.setopt(pycurl.USERPWD, '%s:""' % ( str(self.token), ) )
214  self.curl.setopt(pycurl.HTTPHEADER, ['Accept: application/json'])
215 
216  if data is not None or files is not None:
217  # If there is data or files to send, use a POST request
218 
219  finalData = {}
220 
221  if data is not None:
222  finalData.update(data)
223 
224  if files is not None:
225  for (key, fileName) in files.items():
226  finalData[key] = (self.curl.FORM_FILE, fileName)
227  self.curl.setopt( self.curl.HTTPPOST, finalData.items() )
228 
229  self.curl.setopt(pycurl.VERBOSE, 0)
230 
231  response = cStringIO.StringIO()
232  self.curl.setopt(self.curl.WRITEFUNCTION, response.write)
233  self.curl.perform()
234 
235  code = self.curl.getinfo(self.curl.RESPONSE_CODE)
236 
237  if code in self.retryCodes and len(retries) > 0:
238  logging.debug('Retrying since we got the %s error code...', code)
239  continue
240 
241  if code != 200:
242  raise HTTPError(code, response.getvalue())
243 
244  return response.getvalue()
245 
246  except pycurl.error as e:
247  if len(retries) == 0:
248  raise e
249  logging.debug('Retrying since we got the %s pycurl exception...', str(e))
250 
251 # common/http.py end
upload_popcon.HTTP.discardCookies
def discardCookies
Definition: upload_popcon.py:92
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
upload_popcon.HTTP.token
token
Definition: upload_popcon.py:85
upload_popcon.HTTPError
Definition: upload_popcon.py:44
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:130
upload_popcon.HTTP.query
def query
Definition: upload_popcon.py:175
upload_popcon.HTTP.retryCodes
tuple retryCodes
Definition: upload_popcon.py:68
list
How EventSelector::AcceptEvent() decides whether to accept an event for output otherwise it is excluding the probing of A single or multiple positive and the trigger will pass if any such matching triggers are PASS or EXCEPTION[A criterion thatmatches no triggers at all is detected and causes a throw.] A single negative with an expectation of appropriate bit checking in the decision and the trigger will pass if any such matching triggers are FAIL or EXCEPTION A wildcarded negative criterion that matches more than one trigger in the trigger list("!*","!HLTx*"if it matches 2 triggers or more) will accept the event if all the matching triggers are FAIL.It will reject the event if any of the triggers are PASS or EXCEPTION(this matches the behavior of"!*"before the partial wildcard feature was incorporated).Triggers which are in the READY state are completely ignored.(READY should never be returned since the trigger paths have been run
def upload_popcon.HTTP.setBaseUrl (   self,
  baseUrl = '' 
) 
Allows to set a base URL which will be prefixed to all the URLs
that will be queried later.

Definition at line 98 of file upload_popcon.py.

References upload_popcon.HTTP.baseUrl.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

98 
99  def setBaseUrl(self, baseUrl = ''):
100  '''Allows to set a base URL which will be prefixed to all the URLs
101  that will be queried later.
102  '''
103  self.baseUrl = baseUrl
104 
upload_popcon.HTTP.setBaseUrl
def setBaseUrl
Definition: upload_popcon.py:98
upload_popcon.HTTP.baseUrl
baseUrl
Definition: upload_popcon.py:83
def upload_popcon.HTTP.setProxy (   self,
  proxy = '' 
) 
Allows to set a proxy.

Definition at line 105 of file upload_popcon.py.

105 
106  def setProxy(self, proxy = ''):
107  '''Allows to set a proxy.
108  '''
109  self.curl.setopt(self.curl.PROXY, proxy)
110 
upload_popcon.HTTP.setProxy
def setProxy
Definition: upload_popcon.py:105
def upload_popcon.HTTP.setRetries (   self,
  retries = () 
) 
Allows to set retries.

The retries are a sequence of the seconds to wait per retry.

The retries are done on:
    * PyCurl errors (includes network problems, e.g. not being able
      to connect to the host).
    * 502 Bad Gateway (for the moment, to avoid temporary
      Apache-CherryPy issues).
    * 503 Service Temporarily Unavailable (for when we update
      the frontends).

Definition at line 117 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.__init__(), and uploadConditions.HTTP.__init__().

117 
118  def setRetries(self, retries = ()):
119  '''Allows to set retries.
120 
121  The retries are a sequence of the seconds to wait per retry.
122 
123  The retries are done on:
124  * PyCurl errors (includes network problems, e.g. not being able
125  to connect to the host).
126  * 502 Bad Gateway (for the moment, to avoid temporary
127  Apache-CherryPy issues).
128  * 503 Service Temporarily Unavailable (for when we update
129  the frontends).
130  '''
131  self.retries = retries
upload_popcon.HTTP.setRetries
def setRetries
Definition: upload_popcon.py:117
upload_popcon.HTTP.retries
retries
Definition: upload_popcon.py:130
def upload_popcon.HTTP.setTimeout (   self,
  timeout = 0 
) 
Allows to set a timeout.

Definition at line 111 of file upload_popcon.py.

111 
112  def setTimeout(self, timeout = 0):
113  '''Allows to set a timeout.
114  '''
115  self.curl.setopt(self.curl.TIMEOUT, timeout)
116 
upload_popcon.HTTP.setTimeout
def setTimeout
Definition: upload_popcon.py:111

Member Data Documentation

upload_popcon.HTTP.baseUrl

Definition at line 83 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.HTTP.setBaseUrl(), and uploadConditions.HTTP.setBaseUrl().

upload_popcon.HTTP.curl

Definition at line 74 of file upload_popcon.py.

upload_popcon.HTTP.retries

Definition at line 130 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

tuple upload_popcon.HTTP.retryCodes = frozenset([502, 503])
static

Definition at line 68 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.query(), and uploadConditions.HTTP.query().

upload_popcon.HTTP.token

Definition at line 85 of file upload_popcon.py.

Referenced by upload_popcon.HTTP.getToken(), uploadConditions.HTTP.getToken(), upload_popcon.HTTP.query(), uploadConditions.HTTP.query(), upload_popcon.ConditionsUploader.signOut(), and uploadConditions.ConditionsUploader.signOut().

Generated for CMSSW Reference Manual by doxygen 1.8.5