I'm trying to refactor some code in which many HTTP requests are made using the requests module. Many of these requests have (partially) the same headers, so I would like to 'pre-fill' these using Session objects.
However, I'm having difficulty making multiple inheritance work in this context. Here is what I've tried:
import requests, time
requestbin_URL = 'http://requestb.in/1nsaz9y1' # For testing only; remains usable for 48 hours
auth_token = 'asdlfjkwoieur182932385' # Fake authorization token
class AuthorizedSession(requests.Session):
def __init__(self, auth_token):
super(AuthorizedSession, self).__init__()
self.auth_token = auth_token
self.headers.update({'Authorization': 'token=' + self.auth_token})
class JSONSession(requests.Session):
def __init__(self):
super(JSONSession, self).__init__()
self.headers.update({'content-type': 'application/json'})
class AuthorizedJSONSession(AuthorizedSession, JSONSession):
def __init__(self, auth_token):
AuthorizedSession.__init__(self, auth_token=auth_token)
JSONSession.__init__(self)
""" These two commented-out requests work as expected """
# with JSONSession() as s:
# response = s.post(requestbin_URL, data={"ts" : time.time()})
# with AuthorizedSession(auth_token=auth_token) as s:
# response = s.post(requestbin_URL, data={"key1" : "value1"})
""" This one doesn't """
with AuthorizedJSONSession(auth_token=auth_token) as s:
response = s.post(requestbin_URL, data={"tag" : "some_tag_name"})
If I inspect the result of the last request at http://requestb.in/1nsaz9y1?inspect, I see the following:
It seems like the Content-Type field is correctly set to application/json; however, I don't see an Authorization header with the fake authentication token. How can I combine the AuthorizedSession and JSONSession classes to see both?
I've found that the request works if I define AuthorizedJSONSession more simply as follows:
class AuthorizedJSONSession(AuthorizedSession, JSONSession):
def __init__(self, auth_token):
super(AuthorizedJSONSession, self).__init__(auth_token=auth_token)
The resulting request now has updated both the Authorization and Content-Type headers:
I've understood that when a class inherits from multiple classes which in turn inherit from the same base class, then Python is 'smart enough' to simply use super to initialize.
Related
I'm trying to work with a third party API and I am having problems with sending the request when using the requests or even urllib.request.
Somehow when I use http.client I am successful sending and receiving the response I need.
To make life easier for me, I created an API class below:
class API:
def get_response_data(self, response: http.client.HTTPResponse) -> dict:
"""Get the response data."""
response_body = response.read()
response_data = json.loads(response_body.decode("utf-8"))
return response_data
The way I use it is like this:
api = API()
rest_api_host = "api.app.com"
connection = http.client.HTTPSConnection(rest_api_host)
token = "my_third_party_token"
data = {
"token":token
}
payload = json.loads(data)
headers = {
# some headers
}
connection.request("POST", "/some/endpoint/", payload, headers)
response = connection.getresponse()
response_data = api.get_response_data(response) # I get a dictionary response
This workflow works for me. Now I just want to write a test for the get_response_data method.
How do I instantiate a http.client.HTTPResponse with the desired output to be tested?
For example:
from . import API
from unittest import TestCase
class APITestCase(TestCase):
"""API test case."""
def setUp(self) -> None:
super().setUp()
api = API()
def test_get_response_data_returns_expected_response_data(self) -> None:
"""get_response_data() method returns expected response data in http.client.HTTPResponse"""
expected_response_data = {"token": "a_secret_token"}
# I want to do something like this
response = http.client.HTTPResponse(expected_response_data)
self.assertEqual(api.get_response_data(response), expected_response_data)
How can I do this?
From the http.client docs it says:
class http.client.HTTPResponse(sock, debuglevel=0, method=None, url=None)
Class whose instances are returned upon successful connection. Not instantiated directly by user.
I tried looking at socket for the sock argument in the instantiation but honestly, I don't understand it.
I tried reading the docs in
https://docs.python.org/3/library/http.client.html#http.client.HTTPResponse
https://docs.python.org/3/library/socket.html
Searched the internet on "how to test http.client.HTTPResponse" but I haven't found the answer I was looking for.
I am trying to implement a type of custom authentication by using aiohttp something like the example in this link but I also need request body. Here is an example for requests:
class CustomAuth(AuthBase):
def __init__(self, secretkey):
self.secretkey = secretkey
def get_hash(self, request):
if request.body:
data = request.body.decode('utf-8')
else:
data = "{}"
signature = hmac.new(
str.encode(self.secretkey),
msg=str.encode(data),
digestmod=hashlib.sha256
).hexdigest().upper()
return signature
def __call__(self, request):
request.headers["CUSTOM-AUTH"] = self.get_hash(request)
return request
I've looked into tracing and BasicAuth but they are useless in my situation. On on_request_start request body is not ready, on on_request_chunk_sent headers have already been sent. A solution like BasicAuth don't have access the request data at all.
Do you have any idea?
Thanks in advance.
I have created a custom class, which push my logs to splunk, but somehow it is not working. Here is the class.
class Splunk(logging.StreamHandler):
def __init__(self, url, token):
super().__init__()
self.url = url
self.headers = {f'Authorization': f'Splunk {token}'}
self.propagate = False
def emit(self, record):
mydata = dict()
mydata['sourcetype'] = 'mysourcetype'
mydata['event'] = record.__dict__
response = requests.post(self.url, data=json.dumps(mydata), headers=self.headers)
return response
I call the class from my logger class, somehow like this (adding additional handler), so that it can log on console along with send to splunk
if splunk_config is not None:
splunk_handler = Splunk(splunk_config["url"], splunk_config["token"])
self.default_logger.addHandler(splunk_handler)
But somehow, I am not able to see any logs in splunk. Though I can see the logs in console.
When I try to run the strip down version of above logic from python3 terminal, it is successful.
import requests
import json
url = 'myurl'
token = 'mytoken'
headers = {'Authorization': 'Splunk mytoken'}
propagate = False
mydata = dict()
mydata['sourcetype'] = 'mysourcetype'
mydata['event'] = {'name': 'root', 'msg': 'this is a sample message'}
response = requests.post(url, data=json.dumps(mydata), headers=headers)
print(response.text)
Things I have already tried, making my dictionary data as JSON serializable using below link but it didn't helped.
https://pynative.com/make-python-class-json-serializable/
Any other things to try ?
I've successfully used this Python Class for Sending Events to Splunk HTTP Event Collector instead of writing a dedicated class
https://github.com/georgestarcher/Splunk-Class-httpevent
Advantage is that it implements batchEvent() and flushBatch() methods to submit multiple events at once across multiple threads.
The example here should get you started:
https://github.com/georgestarcher/Splunk-Class-httpevent/blob/master/example.py
If this answers your question, take a moment to accept the answer. This can be done by clicking on the check mark beside the answer to toggle it from greyed out to filled in!
The default Python xmlrpc.client.Transport (can be used with xmlrpc.client.ServerProxy) does not retain cookies, which are sometimes needed for cookie based logins.
For example, the following proxy, when used with the TapaTalk API (for which the login method uses cookies for authentication), will give a permission error when trying to modify posts.
proxy = xmlrpc.client.ServerProxy(URL, xmlrpc.client.Transport())
There are some solutions for Python 2 on the net, but they aren't compatible with Python 3.
How can I use a Transport that retains cookies?
Existing answer from GermainZ works only for HTTP. After a lot of time fighting with it, there is HTTPS adaptation. Note the context option which is crucial.
class CookiesTransport(xmlrpc.client.SafeTransport):
"""A SafeTransport (HTTPS) subclass that retains cookies over its lifetime."""
# Note context option - it's required for success
def __init__(self, context=None):
super().__init__(context=context)
self._cookies = []
def send_headers(self, connection, headers):
if self._cookies:
connection.putheader("Cookie", "; ".join(self._cookies))
super().send_headers(connection, headers)
def parse_response(self, response):
# This check is required if in some responses we receive no cookies at all
if response.msg.get_all("Set-Cookie"):
for header in response.msg.get_all("Set-Cookie"):
cookie = header.split(";", 1)[0]
self._cookies.append(cookie)
return super().parse_response(response)
The reason for it is that ServerProxy doesn't respect context option related to transport, if transport is specified, so we need to use it directly in Transport constructor.
Usage:
import xmlrpc.client
import ssl
transport = CookiesTransport(context=ssl._create_unverified_context())
# Note the closing slash in address as well, very important
server = xmlrpc.client.ServerProxy("https://<api_link>/", transport=transport)
# do stuff with server
server.myApiFunc({'param1': 'x', 'param2': 'y'})
This is a simple Transport subclass that will retain all cookies:
class CookiesTransport(xmlrpc.client.Transport):
"""A Transport subclass that retains cookies over its lifetime."""
def __init__(self):
super().__init__()
self._cookies = []
def send_headers(self, connection, headers):
if self._cookies:
connection.putheader("Cookie", "; ".join(self._cookies))
super().send_headers(connection, headers)
def parse_response(self, response):
for header in response.msg.get_all("Set-Cookie"):
cookie = header.split(";", 1)[0]
self._cookies.append(cookie)
return super().parse_response(response)
Usage:
proxy = xmlrpc.client.ServerProxy(URL, CookiesTransport())
Since xmlrpc.client in Python 3 has better suited hooks for this, it's much simpler than an equivalent Python 2 version.
I am using Google App Engine's Protocol RPC library. I want to get the headers for a request and check that a certain header exists. I can't figure out how to get the requests headers?
The code basically looks like this:
class MyService(remote.Service):
#remote.method(MyRequest, MyResponse)
def my_request(self, request):
# TODO: Check that header exists in request
The passed in request object is of the type 'MyRequest' and doesn't have any header information attached to it.
There is a special method initialize_request_state that allows you to access all of the requests headers.
class MyService(remote.Service):
def initialize_request_state(self, state):
self.headers = state.headers
#remote.method(MyRequest, MyResponse)
def my_request(self, request):
logging.debug(self.headers)