I'm trying to do what I feel like should be a pretty simple mock in python.
I have a tree that looks like this:
- src
- auth
- authenticator.py
- http_client
- auth_client.py
- tests
- test_authenticator.py
My authenticator.py looks like:
import sys
sys.path.append("src/http_client")
import auth_client
def authenticate_request(headers="", env="dev"):
# logic...
auth_client.do_auth(headers)
# logic...
Then auth_client.py looks like:
import json
import requests
def do_auth(headers):
url = "https://FAKE_URL.com"
r = requests.get(url, headers)
return r.json()
And my test_authenticator.py has a test that looks like
import mock
import sys
sys.path.append("src/auth")
sys.path.append("src/http_client")
import authenticator as auth
import auth_client
#mock.patch('auth_client.do_auth')
def test_good(self, mock_request):
headers = FAKE_HEADERS
mock_request.return_value = '{\"response:\": 200}'
body, statusCode = auth.authenticate_request(
headers=headers)
self.assertEqual(200, statusCode)
# self.assertEqual(body, "Token must have kid")
def mock_request():
return "{\"response\":200}"
But I can't seem to actually mock the function. When I run my tests with pytest I get in the coverage that it's hitting auth_client even though I don't want it to.
How do I properly mock this function?
Thanks!
I have following function that I want to write unit test for
def get_user_data(user, session):
'''Given a github user, gets it data'''
url = f'https://api.github.com/users/{user}'
response = session.get(url)
json_response = response.json()
return json_response["login"]
This is how I am testing the above function
from unittest.mock import Mock, MagicMock
from requests import Session
response_payload = {"login": "agrawalo"}
fake_session = MagicMock(spec=Session)
fake_session.get.return_value.get_json.return_value = response_payload
If you notice the code above, instead of mocking "json" method on a response object I have created a mock for "get_json" method. This ends up creating get_json method on a response object. And now when I call
get_user_data("agrawalo", fake_session)
I get
<MagicMock name='mock.get().json().__getitem__()' id='139783470311800'>
Whereas I expect the above unittest to fail with NoAttributeError.
In python 3.8 there is a way to fix above problem using "seal" from unitest.mock.
# from python 3.8
from unittest.mock import seal
seal(fake_session)
get_user_data("agrawalo", fake_session)
How can I do this in python 2.7?
I want to add a referer while retrieving data from the web but this is not working on my python2 referer request.add_header('Referer', 'https://www.python.org').
My Url.txt content
https://www.python.org/about/
https://stackoverflow.com/questions
https://docs.python.org/2.7/
These are my codes
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import urllib2
import threading
import time
import requests
max_thread = 5
urllist = open("Url.txt").readlines()
def url_connect(url):
try :
request = urllib2.Request(url)
request.add_header('Referer', 'https://www.python.org')
request.add_header('User-agent', 'Mozilla/5.0')
goo = re.findall('<title>(.*?)</title>', urllib2.urlopen(url.replace(' ','')).read())[0]
print '\n' + goo.decode("utf-8")
with open('SaveMyDataFile.txt', 'ab') as f:
f.write(goo + "\n")
except Exception as Errors:
pass
for i in urllist:
i = i.strip()
if i.startswith("http"):
while threading.activeCount() >= max_thread:
time.sleep(0.1)
threading.Thread(target=url_connect, args=(i,)).start()
Looks to me the problem is in your call to urlopen. You call it with the url and not with the request.
From https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
Open the URL url, which can be either a string or a Request object.
You need to pass urllib.urlopen() the Request object that you just built--you're currently not doing anything with that.
I use google safe browsing API. So I tested this simple code:
from safebrowsinglookup import SafebrowsinglookupClient
class TestMe:
def testMe(self):
self.key='my_Valid_Key_Here'
self.client=SafebrowsinglookupClient(self.key)
self.response=self.client.lookup('http://www.google.com')
print(self.response)
if __name__=="__main__":
TM=TestMe()
TM.testMe()
I always get this whatever the website I test:
{'website_I_tried','error'}
Note that I had to change some lines in the source code after I installed this API because it was written in Python 2 and I am using Python 3.4.1. How can I resolve this problem?
Update:
To understand why the above problem occured to me, I run this code:
from safebrowsinglookup import SafebrowsinglookupClient
class TestMe:
def testMe(self):
self.key = 'my_key_here'
self.client=SafebrowsinglookupClient(self.key,debug=1)
urls = ['http://www.google.com/','http://addonrock.ru/Debugger.js/']
self.results = self.client.lookup(*urls)
print(self.results['http://www.google.com/'])
if __name__ == "__main__":
TM=TestMe()
TM.testMe()
Now, I got this message:
BODY:
2
http://www.google.com/
http://addonrock.ru/Debugger.js/
URL: https://sb-ssl.google.com/safebrowsing/api/lookup?client=python&apikey=ABQIAAAAAU6Oj8JFgQpt0AXtnVwBYxQYl9AeQCxMD6irIIDtWpux_GHGQQ&appver=0.1&pver=3.0
Unexpected server response
name 'urllib2' is not defined
error
error
The library doesn't support Python3.x.
In this case, you can either make it support Python3 (there is also an opened pull request for Python3 Compatibility), or make the request to "Google Safebrowsing API" manually.
Here's an example using requests:
import requests
key = 'your key here'
URL = "https://sb-ssl.google.com/safebrowsing/api/lookup?client=api&apikey={key}&appver=1.0&pver=3.0&url={url}"
def is_safe(key, url):
response = requests.get(URL.format(key=key, url=url))
return response.text != 'malware'
print(is_safe(key, 'http://addonrock.ru/Debugger.js/')) # prints False
print(is_safe(key, 'http://google.com')) # prints True
Just the same, but without third-party packages (using urllib.request):
from urllib.request import urlopen
key = 'your key here'
URL = "https://sb-ssl.google.com/safebrowsing/api/lookup?client=python&apikey={key}&appver=1.0&pver=3.0&url={url}"
def is_safe(key, url):
response = urlopen(URL.format(key=key, url=url)).read().decode("utf8")
return response != 'malware'
print(is_safe(key, 'http://addonrock.ru/Debugger.js/')) # prints False
print(is_safe(key, 'http://google.com')) # prints True
I am writing an application that performs REST operations using Kenneth Reitz's requests library and I'm struggling to find a nice way to unit test these applications, because requests provides its methods via module-level methods.
What I want is the ability to synthesize the conversation between the two sides; provide a series of request assertions and responses.
It is in fact a little strange that the library has a blank page about end-user unit testing, while targeting user-friendliness and ease of use. There's however an easy-to-use library by Dropbox, unsurprisingly called responses. Here is its intro post. It says they've failed to employ httpretty, while stating no reason of the fail, and written a library with similar API.
import unittest
import requests
import responses
class TestCase(unittest.TestCase):
#responses.activate
def testExample(self):
responses.add(**{
'method' : responses.GET,
'url' : 'http://example.com/api/123',
'body' : '{"error": "reason"}',
'status' : 404,
'content_type' : 'application/json',
'adding_headers' : {'X-Foo': 'Bar'}
})
response = requests.get('http://example.com/api/123')
self.assertEqual({'error': 'reason'}, response.json())
self.assertEqual(404, response.status_code)
If you use specifically requests try httmock. It's wonderfully simple and elegant:
from httmock import urlmatch, HTTMock
import requests
# define matcher:
#urlmatch(netloc=r'(.*\.)?google\.com$')
def google_mock(url, request):
return 'Feeling lucky, punk?'
# open context to patch
with HTTMock(google_mock):
# call requests
r = requests.get('http://google.com/')
print r.content # 'Feeling lucky, punk?'
If you want something more generic (e.g. to mock any library making http calls) go for httpretty.
Almost as elegant:
import requests
import httpretty
#httpretty.activate
def test_one():
# define your patch:
httpretty.register_uri(httpretty.GET, "http://yipit.com/",
body="Find the best daily deals")
# use!
response = requests.get('http://yipit.com')
assert response.text == "Find the best daily deals"
HTTPretty is far more feature-rich - it offers also mocking status code, streaming responses, rotating responses, dynamic responses (with a callback).
You could use a mocking library such as Mocker to intercept the calls to the requests library and return specified results.
As a very simple example, consider this class which uses the requests library:
class MyReq(object):
def doSomething(self):
r = requests.get('https://api.github.com', auth=('user', 'pass'))
return r.headers['content-type']
Here's a unit test that intercepts the call to requests.get and returns a specified result for testing:
import unittest
import requests
import myreq
from mocker import Mocker, MockerTestCase
class MyReqTests(MockerTestCase):
def testSomething(self):
# Create a mock result for the requests.get call
result = self.mocker.mock()
result.headers
self.mocker.result({'content-type': 'mytest/pass'})
# Use mocker to intercept the call to requests.get
myget = self.mocker.replace("requests.get")
myget('https://api.github.com', auth=('user', 'pass'))
self.mocker.result(result)
self.mocker.replay()
# Now execute my code
r = myreq.MyReq()
v = r.doSomething()
# and verify the results
self.assertEqual(v, 'mytest/pass')
self.mocker.verify()
if __name__ == '__main__':
unittest.main()
When I run this unit test I get the following result:
.
----------------------------------------------------------------------
Ran 1 test in 0.004s
OK
Missing from these answers is requests-mock.
From their page:
>>> import requests
>>> import requests_mock
As a context manager:
>>> with requests_mock.mock() as m:
... m.get('http://test.com', text='data')
... requests.get('http://test.com').text
...
'data'
Or as a decorator:
>>> #requests_mock.mock()
... def test_func(m):
... m.get('http://test.com', text='data')
... return requests.get('http://test.com').text
...
>>> test_func()
'data'
using mocker like in srgerg's answer:
def replacer(method, endpoint, json_string):
from mocker import Mocker, ANY, CONTAINS
mocker = Mocker()
result = mocker.mock()
result.json()
mocker.count(1, None)
mocker.result(json_string)
replacement = mocker.replace("requests." + method)
replacement(CONTAINS(endpoint), params=ANY)
self.mocker.result(result)
self.mocker.replay()
For the requests library, this would intercept the request by method and endpoint you're hitting and replace the .json() on the response with the json_string passed in.
If you break out your response handler/parser into a separate function, you can work with requests.Response objects directly, without needing to mock the client-server interaction.
Code under test
from xml.dom import minidom
from requests.models import Response
def function_under_test(s3_response: Response):
doc = minidom.parseString(s3_response.text)
return (
s3_response.status_code,
doc.getElementsByTagName('Code').item(0).firstChild.data,
)
Test code
import unittest
from io import BytesIO
class Test(unittest.TestCase):
def test_it(self):
s3_response = Response()
s3_response.status_code = 404
s3_response.raw = BytesIO(b"""<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>NoSuchKey</Code>
<Message>The resource you requested does not exist</Message>
<Resource>/mybucket/myfoto.jpg</Resource>
<RequestId>4442587FB7D0A2F9</RequestId>
</Error>
""")
parsed_response = function_under_test(s3_response)
self.assertEqual(404, parsed_response[0])
self.assertEqual("NoSuchKey", parsed_response[1])
There's a library for this, if you want to write your test server with Flask: requests-flask-adaptor
You just have to be careful with the order of imports when monkeypatching.