I am trying to mock GetDatabaseConnection but it is still running the code within it.
class GetDatabaseConnection:
resp_dict = json.loads(get_secret())
endpoint = resp_dict.get('host')
username = resp_dict.get('username')
password = resp_dict.get('password')
database_name = resp_dict.get('dbname')
port = resp_dict.get('port')
connection = pymysql.connect(host=endpoint, user=username, passwd=password, db=database_name, port=port)
cursor = connection.cursor()
Here is the test I have written to try to mock the class.
#mock.patch("lambda_function.GetDatabaseConnection")
def test_mock_simple_class(mock_class):
mock_class.return_value = "test"
But I get the following error
test_lambda_function.py::TestPreSignUp::test_mock_simple_class FAILED [100%]
test_lambda_function.py:151 (TestPreSignUp.test_mock_simple_class)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1334: in patched
with self.decoration_helper(patched,
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py:117: in __enter__
return next(self.gen)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1316: in decoration_helper
arg = exit_stack.enter_context(patching)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py:429: in enter_context
result = _cm_type.__enter__(cm)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1389: in __enter__
self.target = self.getter()
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1564: in <lambda>
getter = lambda: _importer(target)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1236: in _importer
thing = __import__(import_path)
../lambda_function.py:42: in <module>
class GetDatabaseConnection():
../lambda_function.py:43: in GetDatabaseConnection
resp_dict = json.loads(get_secret())
You are experiencing the same problem as in this other question about importing without executing the class - python. Since you designed your class to have attributes that would execute the call to pymysql, then they would be executed right away even when the file is just imported (e.g. during mock patching) without even creating an instance of GetDatabaseConnection.
src.py
import pymysql
class GetDatabaseConnection:
connection = pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80)
test_src.py
from unittest import mock
# This will read your class. And this will run your <pymsql> commands even without a running patch yet.
import src # or <from src import GetDatabaseConnection>
# This will also read your class with or without the import above. And this will run your <pymsql> commands even without the patch yet.
#mock.patch('src.GetDatabaseConnection')
def test_try1():
assert True
Output
$ pytest -q -rP
================================================================================================= ERRORS ==================================================================================================
______________________________________________________________________________________ ERROR collecting test_samp.py
E pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '127.0.0.1' ([Errno 111] Connection refused)")
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1 error in 0.22s
Solution 1:
Redesign you class to not execute class-level logic by putting them inside class methods.
class GetDatabaseConnection:
def __init__(self):
self.connection = pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80)
from unittest import mock
#mock.patch('src.GetDatabaseConnection')
def test_try(mock_class):
assert True
$ pytest -q -rP
================================================================================================= PASSES ==================================================================================================
1 passed in 0.03s
Solution 2:
Transform your GetDatabaseConnection into an ordinary function:
def getDatabaseConnection():
return {
"connection": pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80),
}
from unittest import mock
#mock.patch('src.getDatabaseConnection')
def test_try(mock_func):
assert True
Output same as Solution 1
Solution 3:
Not recommendable. Don't import the file directly without a running patch. Thus, don't patch the class GetDatabaseConnection too to avoid reading from the file and executing pymysql. First, you need to patch pymysql before importing the file. This is very hard to maintain and would break if one of your source code files import the file containing the class GetDatabaseConnection.
from unittest import mock
#mock.patch('pymysql.connect')
def test_try(mock_pymsql):
import src # Or <from src import GetDatabaseConnection>
print(f"{src.GetDatabaseConnection.connection=}") # This will be the patched version
assert True
Output:
$ pytest -q -rP
================================================================================================= PASSES ==================================================================================================
________________________________________________________________________________________________ test_try _________________________________________________________________________________________________
------------------------------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------------------------------
src.GetDatabaseConnection.connection=<MagicMock name='connect()' id='139647536711760'>
1 passed in 0.03s
Related
I have a file called redis_db.py which has code to connect to redis
import os
import redis
import sys
class Database:
def __init__(self, zset_name):
redis_host = os.environ.get('REDIS_HOST', '127.0.0.1')
redis_port = os.environ.get('REDIS_PORT', 6379)
self.db = redis.StrictRedis(host=redis_host, port=redis_port)
self.zset_name = zset_name
def add(self, key):
try:
self.db.zadd(self.zset_name, {key: 0})
except redis.exceptions.ConnectionError:
print("Unable to connect to redis host.")
sys.exit(0)
I have another file called app.py which is like this
from flask import Flask
from redis_db import Database
app = Flask(__name__)
db = Database('zset')
#app.route('/add_word/word=<word>')
def add_word(word):
db.add(word)
return ("{} added".format(word))
if __name__ == '__main__':
app.run(host='0.0.0.0', port='8080')
Now I am writing unit test for add_word function like this
import unittest
import sys
import os
from unittest import mock
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../api/")
from api import app # noqa: E402
class Testing(unittest.TestCase):
def test_add_word(self):
with mock.patch('app.Database') as mockdb:
mockdb.return_value.add.return_value = ""
result = app.add_word('shivam')
self.assertEqual(result, 'shivam word added.')
Issue I am facing is that even though I am mocking the db method call it is still calling the actual method in the class instead of returning mocked values and during testing I am getting error with message Unable to connect to redis host..
Can someone please help me in figuring out how can I mock the redis database calls.
I am using unittest module
The issue is that db is defined on module import, so the mock.patch does not affect the db variable. Either you move the instantiation of
db in the add_word(word) function or you patch db instead of Database, e.g.
def test_add_word():
with mock.patch('api.app.db') as mockdb:
mockdb.add = mock.MagicMock(return_value="your desired return value")
result = app.add_word('shivam')
print(result)
Note that the call of add_word has to be in the with block, otherwise the unmocked version is used.
I am trying to write a unittest for a function named search_ldap(), which searches an LDAP server given a particular username. Here is the function definition in utils.py (note: I'm using Python 3):
from ldap3 import Server, Connection
def search_ldap(username):
result = ()
baseDN = "o=Universiteit van Tilburg,c=NL"
searchFilter = '(uid={})'.format(username)
attributes = ['givenName', 'cn', 'employeeNumber', 'mail']
try:
server = Server('ldap.example.com', use_ssl=True)
conn = Connection(server, auto_bind=True)
conn.search(baseDN, searchFilter, attributes=attributes)
for a in attributes:
result += (conn.response[0]['attributes'][a][0], )
except Exception:
raise LDAPError('Error in LDAP query')
return result
Of course I don't want to actually connect to ldap.example.com during testing, so I've decided to use Python's mock object library to mock the Server() and Connection() classes in my unittests. Here is the test code:
from unittest import mock
from django.test import TestCase
class LdapTest(TestCase):
#mock.patch('ldap3.Server')
#mock.patch('ldap3.Connection')
def test_search_ldap(self, mockConnection, mockServer):
from .utils import search_ldap
search_ldap('username')
self.assertTrue(mockServer.called)
self.assertTrue(mockConnection.called)
This test simply asserts that the mocked Server and Connection objects are instantiated. However, thay don't, because when I run the tests with ./manage.py test I receive the following error:
Creating test database for alias 'default'...
F.
======================================================================
FAIL: test_search_ldap (uvt_user.tests.LdapTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/lib/python3.4/unittest/mock.py", line 1142, in patched
return func(*args, **keywargs)
File "/home/jj/projects/autodidact/uvt_user/tests.py", line 28, in test_search_ldap
self.assertTrue(mockServer.called)
AssertionError: False is not true
----------------------------------------------------------------------
Ran 2 tests in 0.030s
FAILED (failures=1)
Destroying test database for alias 'default'...
Why are my tests failing? How can I successfully mock ldap3's Server and Connection classes?
To mock class you should provide it's fake implementation with required methods. For example:
class FakeServer:
def call():
pass
class LdapTest(TestCase):
#mock.patch('ldap3.Server', FakeServer)
def test_search_ldap(self):
<do you checks here>
With patch() it is important that you patch objects in the namespace where they are looked up. This is explained in the Where to patch section of the documentation. There is a fundamental difference between doing
from ldap3 import Server
server = Server()
and
import ldap3
server = ldap3.Server()
In the first case (also the case in the original question), the name "Server" belongs to the current module. In the second case, the name "Server" belongs to the ldap3 module where it is defined. The following Django unittest patches the correct "Server" and "Connection" names and should work as intended:
from unittest import mock
from django.test import TestCase
class LdapTest(TestCase):
#mock.patch('appname.utils.Server')
#mock.patch('appname.utils.Connection')
def test_search_ldap(self, mockConnection, mockServer):
from .utils import search_ldap
search_ldap('username')
self.assertTrue(mockServer.called)
self.assertTrue(mockConnection.called)
I am trying to test the Class BluetoothClient which connects to a BluetoothSocket. To avoid using real sockets I just want to test that the connect() method from the socket is called with the right parameters. Using mock.patch to replace the imported bluetooth module in my bluetooth_control module doesn't work out like expected.
As I see it, the connect() method is called but the assertion tells me otherwise.
Code:
Unit Under Test (bluetooth_control.py):
import bluetooth
class BluetoothClient(object):
def __init__(self):
self.address="98:D3:31:B2:EF:32"
self.port=1
def establishConnection(self):
self.createSocket()
self.connect()
def createSocket(self):
self.sock=bluetooth.BluetoothSocket( bluetooth.RFCOMM )
def connect(self):
print "connect: sock="+str(self.sock)
self.sock.connect((self.address, self.port))
Test (bluetooth_control_test.py):
import unittest
import mock
import bluetooth_control
import bluetooth
class TestShelf(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.bc = bluetooth_control.BluetoothClient()
print "setUp"
def tearDown(self):
self.shelf = None
print "tearDown"
#mock.patch('bluetooth_control.bluetooth')
def testEstablishConnection(self,mock_bluetooth):
self.bc.establishConnection()
print "testEstablishConnection sock="+str(self.bc.sock)
mock_bluetooth.connect().assert_called_with(self.bc.sock,("98:D3:31:B2:EF:32",1))
if __name__ == "__main__":
unittest.main()
Output:
setUp
connect: sock=<MagicMock name='bluetooth.BluetoothSocket()' id='140433322111504'>
testEstablishConnection sock=<MagicMock name='bluetooth.BluetoothSocket()' id='140433322111504'>
FtearDown
======================================================================
FAIL: testEstablishConnection (__main__.TestShelf)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/mock.py", line 1201, in patched
return func(*args, **keywargs)
File "bluetooth_control_test.py", line 21, in testEstablishConnection
mock_bluetooth.connect().assert_called_with(self.bc.sock,("98:D3:31:B2:EF:32",1))
File "/usr/lib/python2.7/site-packages/mock.py", line 831, in assert_called_with
raise AssertionError('Expected call: %s\nNot called' % (expected,))
AssertionError: Expected call: mock(<MagicMock name='bluetooth.BluetoothSocket()' id='140433322111504'>, ('98:D3:31:B2:EF:32', 1))
Not called
----------------------------------------------------------------------
Ran 1 test in 0.003s
FAILED (failures=1)
After looking at the problem again two days later I found the stupid mistakes I made. I had to patch the actual method and remove the falsely added brackets at the assertion.
I am not going to delete this question so that maybe it will help someone to avoid these mistakes.
Test
import unittest
import mock
import bluetooth_control
import bluetooth
class TestShelf(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.bc = bluetooth_control.BluetoothClient()
print "setUp"
def tearDown(self):
self.shelf = None
print "tearDown"
#mock.patch('bluetooth_control.bluetooth.BluetoothSocket.connect')
def testEstablishConnection(self,mock_connect):
self.bc.establishConnection()
print "testEstablishConnection sock="+str(self.bc.sock)
mock_connect.assert_called_with(("98:D3:31:B2:EF:32",1))
if __name__ == "__main__":
unittest.main()
Having a weird problem with Python's unittest and PyMongo. The test randomly succeeds or fails:
import unittest
from pymongo import Connection
from tractor import Tractor
class TestTractor(unittest.TestCase):
def setUp(self):
self.tractor = Tractor(1)
self.mongo = Connection()
self.db = self.mongo.tractor
self.db.classes.remove({'name': {'$regex':'^test_'}})
self.action_class_id = self.db.classes.insert({'name': 'test_action',
'metaclass': 'action'})
self.object_class_id = self.db.classes.insert({'name': 'test_object',
'metaclass': 'object'})
def tearDown(self):
self.tractor = None
def test_create_class(self):
cid1 = self.tractor.create_action_class('test_create_action_class')
cid2 = self.tractor.create_object_class('test_create_object_class')
self.assertNotEqual(cid1, None)
self.assertNotEqual(cid2, None)
action_obj = self.db.classes.find_one({'_id': cid1})
object_obj = self.db.classes.find_one({'_id': cid2})
self.assertNotEqual(cid1, cid2)
self.assertEqual(action_obj['_id'], cid1)
self.assertEqual(object_obj['_id'], cid2)
self.assertEqual(action_obj['name'], 'test_create_action_class')
self.assertEqual(object_obj['name'], 'test_create_object_class')
Class being tested:
from pymongo import Connection
from pymongo.objectid import ObjectId
class Tractor(object):
def __init__(self, uid):
self.uid = uid
self.mongo = Connection()
self.db = self.mongo.tractor
# Classes
def create_action_class(self, name):
return self.db.classes.insert({'name': name,
'attributes': [],
'metaclass': 'action'})
def create_object_class(self, name):
return self.db.classes.insert({'name': name,
'attributes': [],
'metaclass': 'object'})
Random behavior:
silver#aregh-6930-lnx ~/projects/traction/tractor $ python -m unittest discover
......ssEssssssssss
======================================================================
ERROR: test_create_class (tests.test_tractor.TestTractor)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/silver/projects/traction/tractor/tests/test_tractor.py", line 64, in test_create_class
self.assertEqual(action_obj['_id'], cid1)
TypeError: 'NoneType' object is not subscriptable
----------------------------------------------------------------------
Ran 19 tests in 0.023s
FAILED (errors=1, skipped=12)
...
silver#aregh-6930-lnx ~/projects/traction/tractor $ python -m unittest discover
......ss.ssssssssss
----------------------------------------------------------------------
Ran 19 tests in 0.015s
OK (skipped=12)
These two results randomly happen for the same test as I rerun the test without changing anything neither in the class nor in the test.
All of this runs on my machine and I know for sure that while running the test, nobody else tinkers neither with MongoDB nor with the code.
What gives?
I strongly suspect the problem here is that you are not using "safe" mode for your writes.
By default MongoDB uses "fire and forget" mode. This means that the insert command is sent to the server, but the driver doesn't check for any server responses.
When you switch to "safe" mode, the driver will send the insert command and it will then send a second command getLastError. This second command will return when the server has actually committed the write.
Again, by default you are running in "fire and forget" mode, so there is indeed a potential race condition here. For unit tests you will need to run with "safe" mode on.
The function signature for insert is defined here. However, you should also be able to make the change at the Connection level so that each connection to the DB uses "safe" mode by default.
I want to use nose to test an application that I am writing using twisted and txmongo. I can't even get simple use cases like the following working:
from nose.twistedtools import reactor, deferred, threaded_reactor
import logging
from twisted.internet import defer
import txmongo
log = logging.getLogger("common.test.test_db")
conn = txmongo.lazyMongoConnectionPool('localhost', 27017, 4)
#deferred()
def test_mongo():
tdb = conn.test
#defer.inlineCallbacks
def cb(oid):
assert oid
obj = yield tdb.test.find({"_id":oid})
log.error("In callback")
assert obj
d = tdb.test.save({"s":1, "b":2})
d.addCallback(cb)
return d
However, this always return the following:
E
======================================================================
ERROR: common.test.test_db.test_mongo
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Volumes/Users/jce/.pyenv/celery/lib/python2.6/site-packages/nose/case.py", line 186, in runTest
self.test(*self.arg)
File "/Volumes/Users/jce/.pyenv/celery/lib/python2.6/site-packages/nose/twistedtools.py", line 138, in errback
failure.raiseException()
File "/Volumes/Users/jce/.pyenv/celery/lib/python2.6/site-packages/twisted/python/failure.py", line 326, in raiseException
raise self.type, self.value, self.tb
RuntimeWarning: not connected
----------------------------------------------------------------------
Ran 1 test in 0.006s
FAILED (errors=1)
I tried manually adding a threaded_reactor() call, but it didn't help.
edit
I removed the "lazy" connections, and modified the code, and now it works... I'm still curious as to why the "lazy" didn't work. The working code is as follows:
dbconn = txmongo.MongoConnectionPool('localhost', 27017, 4)
#deferred()
def test_mongo():
#defer.inlineCallbacks
def cb(conn):
tdb = conn.test
oid = yield tdb.test.save({"s":1, "b":2})
assert oid
log.error(str(oid))
obj = yield tdb.test.find({"_id":oid})
assert obj
log.error(str(obj))
dbconn.addCallback(cb)
return dbconn
MongoConnectionPool will return a deferred, which is fired when the connection is established passing the connection handler as argument to the callback. You should conn = yield MongoConnectionPool().
lazyMongoConnectionPool will return the connection handler directly, without waiting for the connection to be established.
Lazy is usually used by web servers and other services that doesn't require immediate connection when your service starts. If you want to do so, don't use the lazy method.