I am trying to write pytest to test the following method by mocking the database. How to mock the database connection without actually connecting to the real database server.I tried with sample test case. I am not sure if that is right way to do it. Please correct me if I am wrong.
//fetch.py
import pymssql
def cur_fetch(query):
with pymssql.connect('host', 'username', 'password') as conn:
with conn.cursor(as_dict=True) as cursor:
cursor.execute(query)
response = cursor.fetchall()
return response
//test_fetch.py
import mock
from unittest.mock import MagicMock, patch
from .fetch import cur_fetch
def test_cur_fetch():
with patch('fetch.pymssql', autospec=True) as mock_pymssql:
mock_cursor = mock.MagicMock()
test_data = [{'password': 'secret', 'id': 1}]
mock_cursor.fetchall.return_value = MagicMock(return_value=test_data)
mock_pymssql.connect.return_value.cursor.return_value.__enter__.return_value = mock_cursor
x = cur_fetch({})
assert x == None
The result is:
AssertionError: assert <MagicMock name='pymssql.connect().__enter__().cursor().__enter__().fetchall()' id='2250972950288'> == None
Please help.
Trying to mock out a module is difficult. Mocking a method call is simple. Rewrite your test like this:
import unittest.mock as mock
import fetch
def test_cur_tech():
with mock.patch('fetch.pymssql.connect') as mock_connect:
mock_conn = mock.MagicMock()
mock_cursor = mock.MagicMock()
mock_connect.return_value.__enter__.return_value = mock_conn
mock_conn.cursor.return_value.__enter__.return_value = mock_cursor
mock_cursor.fetchall.return_value = [{}]
res = fetch.cur_fetch('select * from table')
assert mock_cursor.execute.call_args.args[0] == 'select * from table'
assert res == [{}]
In the above code we're explicitly mocking pymyssql.connect, and
providing appropriate fake context managers to make the code in
cur_fetch happy.
You could simplify things a bit if cur_fetch received the connection
as an argument, rather than calling pymssql.connect itself.
Related
I am currently trying to write unit tests for my python code using Moto & #mock_dynamodb2 . So far it's been working for me to test my "successful operation" test cases. But I'm having trouble getting it to work for my "failure cases".
In my test code I have:
#mock_dynamodb2
class TestClassUnderTestExample(unittest.TestCase):
def setUp(self):
ddb = boto3.resource("dynamodb", "us-east-1")
self.table = ddb.create_table(<the table definition)
self.example_under_test = ClassUnderTestExample(ddb)
def test_some_thing_success(self):
expected_response = {<some value>}
assert expected_response = self.example_under_test.write_entry(<some value>)
def test_some_thing_success(self):
response = self.example_under_test.write_entry(<some value>)
# How to assert exception is thrown by forcing put item to fail?
The TestClassUnderTestExample would look something like this:
class ClassUnderTestExample:
def __init__(self, ddb_resource=None):
if not ddb_resource:
ddb_resource = boto3.resource('dynamodb')
self.table = ddb_resource.Table(.....)
def write_entry(some_value)
ddb_item = <do stuff with some_value to create sanitized item>
response = self.table.put_item(
Item=ddb_item
)
if pydash.get(response, "ResponseMetadata.HTTPStatusCode") != 200:
raise SomeCustomErrorType("Unexpected response from DynamoDB when attempting to PutItem")
return ddb_item
I've been completely stuck when it comes to actually mocking the .put_item operation to return a non-success value so that I can test that the ClassUnderTestExample will handle it as expected and throw the custom error. I've tried things like deleting the table before running the test, but that just throws an exception when getting the table rather than an executed PutItem with an error code.
I've also tried putting a patch for pydash or for the table above the test but I must be doing something wrong. I can't find anything in moto's documentation. Any help would be appreciated!
The goal of Moto is to completely mimick AWS' behaviour, including how to behave when the user supplies erroneous inputs. In other words, a call to put_item() that fails against AWS, would/should also fail against Moto.
There is no build-in way to force an error response on a valid input.
It's difficult to tell from your example how this can be forced, but it looks like it's worth playing around with this line to create an invalid input:
ddb_item = <do stuff with some_value to create sanitized item>
Yes, you can. Use mocking for this. Simple and runnable example:
from unittest import TestCase
from unittest.mock import Mock
from uuid import uuid4
import boto3
from moto import mock_dynamodb2
def create_user_table(table_name: str) -> dict:
return dict(
TableName=table_name,
KeySchema=[
{
'AttributeName': 'id',
'KeyType': 'HASH'
},
],
AttributeDefinitions=[
{
'AttributeName': 'id',
'AttributeType': 'S'
},
],
BillingMode='PAY_PER_REQUEST'
)
class UserRepository:
table_name = 'users'
def __init__(self, ddb_resource):
if not ddb_resource:
ddb_resource = boto3.resource('dynamodb')
self.table = ddb_resource.Table(self.table_name)
def create_user(self, username):
return self.table.put_item(Item={'id': str(uuid4), 'username': username})
#mock_dynamodb2
class TestUserRepository(TestCase):
def setUp(self):
ddb = boto3.resource("dynamodb", "us-east-1")
self.table = ddb.create_table(**create_user_table('users'))
self.test_user_repo = UserRepository(ddb)
def tearDown(self):
self.table.delete()
def test_some_thing_success(self):
user = self.test_user_repo.create_user(username='John')
assert len(self.table.scan()['Items']) == 1
def test_some_thing_failure(self):
self.test_user_repo.table = table = Mock()
table.put_item.side_effect = Exception('Boto3 Exception')
with self.assertRaises(Exception) as exc:
self.test_user_repo.create_user(username='John')
self.assertTrue('Boto3 Exception' in exc.exception)
I am new to pytest and wanted to add the below 3 methods for unit test coverage without actually using a real mongo db instance but rather mock it.
Could try using a real db instance but it isn't recommended.
Request for an example on how to mock mongodb client and get a document
import os
import logging
import urllib.parse
from dotenv import load_dotenv
from pymongo import MongoClient
from logger import *
load_dotenv()
def getMongoConnection():
userName = urllib.parse.quote_plus(os.getenv("USER_NAME"))
password = urllib.parse.quote_plus(os.getenv("PASSWORD"))
hostName1_port = os.getenv("HOST_NAME1")
hostName2_port = os.getenv("HOST_NAME2")
hostName3_port = os.getenv("HOST_NAME3")
authSourceDatabase = os.getenv("AUTH_SOURCE_DATABASE")
replicaSet = os.getenv("REPLICA_SET")
connectTimeoutMS = "1000"
socketTimeoutMS = "30000"
maxPoolSize = "100"
try:
client = MongoClient('mongodb://'+userName+':'+password+'#'+hostName1_port+','+hostName2_port+','+hostName3_port+'/'+authSourceDatabase+'?ssl=true&replicaSet='+replicaSet +
'&authSource='+authSourceDatabase+'&retryWrites=true&w=majority&connectTimeoutMS='+connectTimeoutMS+'&socketTimeoutMS='+socketTimeoutMS+'&maxPoolSize='+maxPoolSize)
return client
except Exception as e:
logging.error("Error while connecting to mongoDB.")
return False
def connectToDBCollection(client, databaseName, collectionName):
db = client[databaseName]
collection = db[collectionName]
return collection
def getDoc(bucketName, databaseName, collectionName):
try:
client = getMongoConnection()
if client != False:
collection = connectToDBCollection(
client, databaseName, collectionName)
return collection.find_one({'bucket': bucketName})
except Exception as e:
logging.error("An exception occurred while fetching doc, error is ", e)
Edit : (Tried using below code and was able to cover most of the cases but seeing an error)
def test_mongo():
db_conn = mongomock.MongoClient()
assert isinstance(getMongoConnection(), MongoClient)
def test_connect_mongo():
return connectToDBCollection(mongomock.MongoClient(), "sampleDB", "sampleCollection")
//trying to cover exception block for getMongoConnection()
def test_exception():
with pytest.raises(Exception) as excinfo:
getMongoConnection()
assert str(excinfo.value) == False
def test_getDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.find_one({'_id': 1})
assert stored_obj == getDoc("bucket", "db", "collection")
def test_createDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.insert_one({'_id': 1})
assert stored_obj == createDoc("bucket", "db", "collection")
def test_updateDoc():
collection = mongomock.MongoClient().db.collection
stored_obj = collection.replace_one({'_id': 1}, {'_id': 2})
assert stored_obj == updateDoc(
{'_id': 1}, {'$set': {'_id': 2}}, "db", "collection")
Errors :
test_exception - Failed: DID NOT RAISE <class 'Exception'>
test_createDoc - TypeError: not all arguments converted during string formatting
AssertionError: assert <pymongo.results.UpdateResult object at 0x7fc0e835a400> == <pymongo.results.UpdateResult object at 0x7fc0e8211900>
Looks like MongoClient is a nested dict with databaseName and collectionName or implemented with a key accessor.
You could mock the client first with
import unittest
mocked_collection = unittest.mock.MagicMock()
# mock the find_one method
mocked_collection.find_one.return_value = {'data': 'collection_find_one_result'}
mocked_client = unittest.mock.patch('pymongo.MongoClient').start()
mocked_client.return_value = {
'databaseName': {'collectionname': mocked_collection}
}
Maybe try a specialized mocking library like MongoMock?
In particular the last example using #mongomock.patch looks like it can be relevant for your code.
I am a beginner when it comes to writing tests and mocking.
I have a created two modules. One module object (Site) creates another object from my second module (Item) on init. The Item object makes a call to an API endpoint to get some data using requests.
I want to Mock the API call I am making so I can test things like a bad response and importantly have control over the response data.
I have simplified my code and put below. When I run the test I get back the actual response data and not what I have mocked.
I have a feeling I am not putting my Mock in the right place. Also, I have seen lots of people saying to use #unittest.patch annotation. I am not clear if I should be using that here.
So I am looking for how to get _get_range_detail to actually return a Mocked response from requests and also just general feedback on if it looks like I am approaching this the right way.
# hello_world.py
from mymodule.site import Site
sites = [
dict(
name="site1",
ranges=[
"range1",
"range2"
]
)
]
site_object = Site(sites[0]['name'], sites[0]['ranges'])
for i in site_object.get_ranges_objects():
print(i.range_detail)
# site.py
from mymodule.item import Item
class Site:
def __init__(self, name, ranges):
self.name = name
self.ranges = ranges
self.ranges_objects = []
for my_range in ranges:
self.ranges_objects.append(Item(my_range))
def get_ranges_objects(self):
return self.ranges_objects
# item.py
import requests
class Item:
def __init__(self, range_name):
self.range_name = range_name
self.range_detail = self._get_range_detail(self.range_name)
def _get_range_detail(self, range_name):
uri = "https://postman-echo.com/get?some_cool_value=real_value"
try:
r = requests.get(uri)
if r.status_code == 200:
return r.json()['args']
else:
return None
except Exception as e:
print(e)
exit(1)
# test_site.py
import pytest
from mymodule.site import Site
from unittest import mock
from mymodule.item import requests
def test_get_ranges_objects():
sites = [
dict(
name="site1",
ranges=[
"range1",
"range2"
]
)
]
requests = mock.Mock()
requests.status_code = 200
requests.json.return_value = {
'args': {'some_mock_value': 'mocky'}
}
site_object = Site(sites[0]['name'], sites[0]['ranges'])
assert site_object.name == "site1"
assert isinstance(site_object.ranges_objects, list)
assert site_object.ranges_objects[0].range_detail == dict(some_mock_value='mocky')
You can use pytest-mock. it makes mocking in pytest simple. (pip install pytest-mock)
You should replace requests.get. Simply
requests_get = mock.patch('requests.get').start()
If you use pytest-mock,
requests_get = mocker.patch('requests.get')
Rewrote test case using pytest-mock
# test_site.py
import pytest
from mymodule.site import Site
from unittest import mock
#pytest.fixture
def requests_get(mocker):
requests_get = mocker.patch('requests.get')
yield requests_get
def test_get_ranges_objects(mocker, requests_get):
response = mocker.MagicMock()
response.status_code = 200
response.json.return_value = {'args': {'some_mock_value': 'mocky'}}
requests_get.return_value = response
sites = [
dict(
name="site1",
ranges=[
"range1",
"range2"
]
)
]
site_object = Site(sites[0]['name'], sites[0]['ranges'])
assert site_object.name == "site1"
assert isinstance(site_object.ranges_objects, list)
assert site_object.ranges_objects[0].range_detail == {'some_mock_value': 'mocky'}
I'm trying to write unit test for some_func function in some_func.py below. I do not want to connect to any database during this tests and I want to mock out any calls to DB.
Since the actual db calls are somewhat nested, i'm not able to get this to work. How do i patch any db interaction in this case?
db_module.py
import MySQLdb
from random_module.config.config_loader import config
from random_module.passwd_util import get_creds
class MySQLConn:
_conn = None
def __init__(self):
self._conn = self._get_rds_connection()
def get_conn(self):
return self._conn
def _get_rds_connection(self):
"""
Returns a conn object after establishing connection with the MySQL database
:return: obj: conn
"""
try:
logger.info("Establishing connection with MySQL")
username, password = get_creds(config['mysql_dev_creds'])
connection = MySQLdb.connect(
host=config['host'],
user=username,
passwd=password,
db=config['db_name'],
port=int(config['db_port']))
connection.autocommit = True
except Exception as err:
logger.error("Unable to establish connection to MySQL")
raise ConnectionAbortedError(err)
if (connection):
logger.info("Connection to MySQL successful")
return connection
else:
return None
db_mysql = MySQLConn()
some_func.py
from random_module.utils.db_module import db_mysql
def some_func():
try:
db_conn = db_mysql.get_conn()
db_cursor = db_conn.cursor()
results = db_cursor.execute("SELECT * FROM some_table")
results = db_cursor.fetchall()
result_set = []
for row in results:
result_set.insert(i, row['x'])
result_set.insert(i, row['y'])
except Exception:
logger.error("some error")
return result_set
dir struct -
src
├──pkg
| ├── common
| |___ some_func.py
| |___ __init__.py
|
| ├── utils
| |___ db_module.py
| |___ __init__.py
|
| __init__.py
You need to mock this line: db_conn = db_mysql.get_conn()
The return value of the get_conn method is what you're interested in.
from random_module.utils.db_module import db_mysql
#mock.patch.object(db_mysql, 'get_conn')
def test_some_func(self, mock_get):
mock_conn = mock.MagicMock()
mock_get.return_value = mock_conn
mock_cursor = mock.MagicMock()
mock_conn.cursor.return_value = mock_cursor
expect = ...
result = some_func()
self.assertEqual(expect, result)
self.assertTrue(mock_cursor.execute.called)
As you can see there is a lot of complexity setting up these mocks. That is because you are instantiating objects inside of your function. A better approach would be to refactor your code to inject the cursor because the cursor is the only thing relevant to this function. An even better approach would be to create a database fixture to test the function actually interacts with the database correctly.
You should mock db_mysql in some_module.py and then assert that the expected calls were made after some_func() executes.
from unittest TestCase
from unittest.mock import patch
from some_module import some_func
class SomeFuncTest(TestCase):
#patch('some_module.db_mysql')
def test_some_func(self, mock_db):
result_set = some_func()
mock_db.get_conn.assert_called_once_with()
mock_cursor = mock_db.get_conn.return_value
mock_cursor.assert_called_once_with()
mock_cursor.execute.assert_called_once_with("SELECT * FROM some_table")
mock_cursor.fetchall.return_value = [
{'x': 'foo1', 'y': 'bar1'},
{'x': 'foo2', 'y': 'bar2'}
]
mock_cursor.fetchall.assert_called_once_with()
self.assertEqual(result_set, ['foo1', 'bar1', 'foo2', 'bar2'])
I have test:
class MyTests(TestCase):
def setUp(self):
self.myclient = MyClient()
#mock.patch('the_file.requests.json')
def test_myfunc(self, mock_item):
mock_item.return_value = [
{'itemId': 1},
{'itemId': 2},
]
item_ids = self.myclient.get_item_ids()
self.assertEqual(item_ids, [1, 2])
in the file I have
import requests
class MyClient(object):
def get_product_info(self):
response = requests.get(PRODUCT_INFO_URL)
return response.json()
My goal is to mock get_product_info() to return the return_value data in the test. I have tried mocking requests.json and requests.get.json, both error out on no attribute, I've mocked the_file.MyClient.get_product_info which doesn't cause error but doesn't work, it returns the real data.
How can I mock this get_product_info which uses requests library?
You should be able to just patch get_product_info().
from unittest.mock import patch
class MyClient(object):
def get_product_info(self):
return 'x'
with patch('__main__.MyClient.get_product_info', return_value='z'):
client = MyClient()
info = client.get_product_info()
print('Info is {}'.format(info))
# >> Info is z
Just switch __main__ to the name of your module. You might also find patch.object useful.