I have a file called redis_db.py which has code to connect to redis
import os
import redis
import sys
class Database:
def __init__(self, zset_name):
redis_host = os.environ.get('REDIS_HOST', '127.0.0.1')
redis_port = os.environ.get('REDIS_PORT', 6379)
self.db = redis.StrictRedis(host=redis_host, port=redis_port)
self.zset_name = zset_name
def add(self, key):
try:
self.db.zadd(self.zset_name, {key: 0})
except redis.exceptions.ConnectionError:
print("Unable to connect to redis host.")
sys.exit(0)
I have another file called app.py which is like this
from flask import Flask
from redis_db import Database
app = Flask(__name__)
db = Database('zset')
#app.route('/add_word/word=<word>')
def add_word(word):
db.add(word)
return ("{} added".format(word))
if __name__ == '__main__':
app.run(host='0.0.0.0', port='8080')
Now I am writing unit test for add_word function like this
import unittest
import sys
import os
from unittest import mock
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../api/")
from api import app # noqa: E402
class Testing(unittest.TestCase):
def test_add_word(self):
with mock.patch('app.Database') as mockdb:
mockdb.return_value.add.return_value = ""
result = app.add_word('shivam')
self.assertEqual(result, 'shivam word added.')
Issue I am facing is that even though I am mocking the db method call it is still calling the actual method in the class instead of returning mocked values and during testing I am getting error with message Unable to connect to redis host..
Can someone please help me in figuring out how can I mock the redis database calls.
I am using unittest module
The issue is that db is defined on module import, so the mock.patch does not affect the db variable. Either you move the instantiation of
db in the add_word(word) function or you patch db instead of Database, e.g.
def test_add_word():
with mock.patch('api.app.db') as mockdb:
mockdb.add = mock.MagicMock(return_value="your desired return value")
result = app.add_word('shivam')
print(result)
Note that the call of add_word has to be in the with block, otherwise the unmocked version is used.
Related
This is my inside flaskapp/__init__.py for creating my Flask app, with it I can access Session inside any module in the package, by just importing Session from flaskapp/db.py:
import os
from flask import Flask
def create_app(test_config=None):
app = Flask(__name__, instance_relative_config=True)
app.config.from_mapping(
SECRET_KEY=b'some_secret_key',
DEBUG=True,
SQLALCHEMY_DATABASE_URI=f'sqlite:///{os.path.join(app.instance_path, "flaskapp.sqlite")}',
)
if test_config is None:
app.config.from_pyfile('config.py', silent=True)
else:
app.config.from_mapping(test_config)
try:
os.makedirs(app.instance_path)
except OSError:
pass
with app.app_context():
from flaskapp.routes import home, auth
from flaskapp.db import init_db
init_db()
return app
This is my flaskapp/db.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from flask import current_app
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
Session = sessionmaker(bind=engine)
def init_db():
import flaskapp.models as models
models.Base.metadata.create_all(engine)
def drop_db():
import flaskapp.models as models
models.Base.metadata.drop_all(engine)
With that, I can access the database in any other module, for example, flaskapp/auth.py:
from flask import flash, session
from sqlalchemy import select
from flaskapp.db import Session
from flaskapp.models import User
def log_user(username: str, password: str) -> bool:
with Session() as db_session:
stmt1 = select(User).where(User.username == username)
query_user = db_session.execute(stmt1).first()
if not query_user:
flash('Some error message')
# Some password verifications and other things
session['username'] = query_user[0].username
flash('Successfully logged in')
return True
Until that point, I don't have any problem, the problem comes when I try to do unit testing with unittest, I can't set the test environment and I don't know how can I use the Session object defined in flaskapp/db.py for testing in a separate database. Everything I've tried until now gets me an error, this is my tests/__init__.py:
import unittest
from flaskapp import create_app
from flaskapp.db import Session
from flaskapp.models import User
class BaseTestClass(unittest.TestCase):
def setUp(self):
self.app = create_app(test_config={
'TESTING': True,
'DEBUG': True,
'APP_ENV': 'testing',
# I pass the test database URI expecting the engine to use it
'SQLALCHEMY_DATABASE_URI': 'sqlite:///testdb.sqlite',
})
self.client = self.app.test_client()
# Context
with self.app.app_context():
self.populate_db()
def tearDown(self):
pass
def populate_db(self):
with Session() as db_session:
db_session.add(User(
username='Harry',
email='harry#yahoo.es',
password = 'Harry123.'
))
db_session.commit()
When I try to use the Session object inside populate_db() I get this error:
=====================================================================
ERROR: tests.test_auth (unittest.loader._FailedTest.tests.test_auth)
----------------------------------------------------------------------
ImportError: Failed to import test module: tests.test_auth
Traceback (most recent call last):
File "/usr/local/lib/python3.11/unittest/loader.py", line 407, in _find_test_path
module = self._get_module_from_name(name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/unittest/loader.py", line 350, in _get_module_from_name
__import__(name)
File "/home/chus/usb/soft/proy/SAGC/tests/test_auth.py", line 1, in <module>
from flaskapp.db import Session
File "/home/chus/usb/soft/proy/SAGC/flaskapp/db.py", line 5, in <module>
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 316, in __get__
obj = instance._get_current_object() # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 513, in _get_current_object
raise RuntimeError(unbound_message) from None
RuntimeError: Working outside of application context.
This typically means that you attempted to use functionality that needed
the current application. To solve this, set up an application context
with app.app_context(). See the documentation for more information.
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
Can someone help me out? I've tried everything to make the tests use the app context in order to populate the test database and make queries but it appears to be conflicting with it being created in create_app() or something.
Firsly, the error you posted is about tests/test_auth.py which you did not include.
However, your code is tightly coupling the flask app and the db.py module with the use of flask.current_app to obtain the engine url.
So the engine creation will only work if there is an app context, which is probably absent when you do a top level import (line 1) of from flaskapp.db import Session in tests/test_auth.py.
A solution could be creating the engine and sessionmaker (or scoped_session) with your create_app and injecting the engine or Session in your functions (look up dependency injection principle):
def init_db(engine):
models.Base.metadata.create_all(engine)
def drop_db(engine):
models.Base.metadata.drop_all(engine)
def log_user(session, username: str, password: str) -> bool:
stmt1 = select(User).where(User.username == username)
query_user = session.execute(stmt1).first()
# skipping the rest
Or look into Flask-SQLAlchemy.
I am trying to mock GetDatabaseConnection but it is still running the code within it.
class GetDatabaseConnection:
resp_dict = json.loads(get_secret())
endpoint = resp_dict.get('host')
username = resp_dict.get('username')
password = resp_dict.get('password')
database_name = resp_dict.get('dbname')
port = resp_dict.get('port')
connection = pymysql.connect(host=endpoint, user=username, passwd=password, db=database_name, port=port)
cursor = connection.cursor()
Here is the test I have written to try to mock the class.
#mock.patch("lambda_function.GetDatabaseConnection")
def test_mock_simple_class(mock_class):
mock_class.return_value = "test"
But I get the following error
test_lambda_function.py::TestPreSignUp::test_mock_simple_class FAILED [100%]
test_lambda_function.py:151 (TestPreSignUp.test_mock_simple_class)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1334: in patched
with self.decoration_helper(patched,
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py:117: in __enter__
return next(self.gen)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1316: in decoration_helper
arg = exit_stack.enter_context(patching)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py:429: in enter_context
result = _cm_type.__enter__(cm)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1389: in __enter__
self.target = self.getter()
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1564: in <lambda>
getter = lambda: _importer(target)
/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/unittest/mock.py:1236: in _importer
thing = __import__(import_path)
../lambda_function.py:42: in <module>
class GetDatabaseConnection():
../lambda_function.py:43: in GetDatabaseConnection
resp_dict = json.loads(get_secret())
You are experiencing the same problem as in this other question about importing without executing the class - python. Since you designed your class to have attributes that would execute the call to pymysql, then they would be executed right away even when the file is just imported (e.g. during mock patching) without even creating an instance of GetDatabaseConnection.
src.py
import pymysql
class GetDatabaseConnection:
connection = pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80)
test_src.py
from unittest import mock
# This will read your class. And this will run your <pymsql> commands even without a running patch yet.
import src # or <from src import GetDatabaseConnection>
# This will also read your class with or without the import above. And this will run your <pymsql> commands even without the patch yet.
#mock.patch('src.GetDatabaseConnection')
def test_try1():
assert True
Output
$ pytest -q -rP
================================================================================================= ERRORS ==================================================================================================
______________________________________________________________________________________ ERROR collecting test_samp.py
E pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on '127.0.0.1' ([Errno 111] Connection refused)")
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1 error in 0.22s
Solution 1:
Redesign you class to not execute class-level logic by putting them inside class methods.
class GetDatabaseConnection:
def __init__(self):
self.connection = pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80)
from unittest import mock
#mock.patch('src.GetDatabaseConnection')
def test_try(mock_class):
assert True
$ pytest -q -rP
================================================================================================= PASSES ==================================================================================================
1 passed in 0.03s
Solution 2:
Transform your GetDatabaseConnection into an ordinary function:
def getDatabaseConnection():
return {
"connection": pymysql.connect(host="127.0.0.1", user='username', passwd="password", db='database_name', port=80),
}
from unittest import mock
#mock.patch('src.getDatabaseConnection')
def test_try(mock_func):
assert True
Output same as Solution 1
Solution 3:
Not recommendable. Don't import the file directly without a running patch. Thus, don't patch the class GetDatabaseConnection too to avoid reading from the file and executing pymysql. First, you need to patch pymysql before importing the file. This is very hard to maintain and would break if one of your source code files import the file containing the class GetDatabaseConnection.
from unittest import mock
#mock.patch('pymysql.connect')
def test_try(mock_pymsql):
import src # Or <from src import GetDatabaseConnection>
print(f"{src.GetDatabaseConnection.connection=}") # This will be the patched version
assert True
Output:
$ pytest -q -rP
================================================================================================= PASSES ==================================================================================================
________________________________________________________________________________________________ test_try _________________________________________________________________________________________________
------------------------------------------------------------------------------------------ Captured stdout call -------------------------------------------------------------------------------------------
src.GetDatabaseConnection.connection=<MagicMock name='connect()' id='139647536711760'>
1 passed in 0.03s
I am quite new to unit testing and relatively new to RESTful API development as well. I am wondering how to do unit test for functions inside Resource class in flask restful? I can do unit test for the endpoint's response but I don't know how to do testing for the individual functions inside the endpoint's controller class.
Below is my application code. It has 3 files including test:
api.py
controller_foo.py
test_controller_foo.py
# api.py
from flask import Flask
from flask_restful import Api
from .controller_foo import ControllerFoo
def create_app(config=None):
app = Flask(__name__)
app.config['ENV'] ='development'
return app
application = app = create_app()
api = Api(app)
api.add_resource(ControllerFoo, '/ctrl')
if __name__ == "__main__":
app.run(debug=True)
# controller_foo.py
from flask_restful import Resource
from flask import request
class ControllerFoo(Resource):
"""
basically flask-restful's Resource method is a wrapper for flask's MethodView
"""
def post(self):
request_data = self.handle_request()
response = self.process_request(request_data)
return response
def handle_request(self):
json = request.get_json()
return json
def process_request(self, data):
# do some stuffs here
return {'foo': 'bar'}
I am using unittest
# test_controller_foo.py
import unittest
from api import app
from .controller_foo import ControllerFoo
# initiating class to try testing but I don't know how to start
ctrl = ControllerFoo()
class ControllerFooTestCase(unittest.TestCase):
def setUp(self):
self.app = app
self.app.config['TESTING'] = True
self.client = app.test_client()
self.payload = {'its': 'empty'}
def tearDown(self):
pass
def test_get_response(self):
response = self.client.post('/ctrl', json=self.payload)
expected_resp = {
'foo': 'bar'
}
self.assertEqual(response.status_code, 200)
self.assertDictEqual(response.get_json(), expected_resp)
if __name__ == "__main__":
unittest.main()
I want to know how to properly do unit test for handle_request and process_request function
EDIT: Fixing out my buggy code. Thanks Laurent LAPORTE for the highlights.
There are several bugs in your code, so this is not easy to explain.
First of all, the recommended way to do testing with Flask (and Flask-Restful) is to use PyTest instead of unittest, because it is easier to setup and use.
Take a look at the documentation: Testing Flask Applications.
But, you can start with unittest…
note: you can have a confusion with your app module and the app instance in that module. So, to avoid it, I imported the module. Another good practice is to name your test module against the tested module: "app.py" => "test_app.py". You can also have a confusion with the controller module and the controller instance. The best practice is to use a more precise name, like "controller_foo" or something else…
Here is a working unit test:
# test_app.py
import unittest
import app
class ControllerTestCase(unittest.TestCase):
def setUp(self):
self.app = app.app
self.app.config['TESTING'] = True
self.client = self.app.test_client()
self.payload = {'its': 'empty'}
def test_get_response(self):
response = self.client.post('/ctrl', json=self.payload)
expected_resp = {'foo': 'bar'}
self.assertEqual(response.status_code, 200)
self.assertDictEqual(response.get_json(), expected_resp)
if __name__ == "__main__":
unittest.main()
As you can see, I also fixed the posted URL, in your application, the URL is "/ctrl", not "controller".
At this point, the test can run, but you have another error:
Ran 1 test in 0.006s
FAILED (errors=1)
Error
Traceback (most recent call last):
...
TypeError: process_request() takes 1 positional argument but 2 were given
If you take a look at your process_request() method, you can see that you missed the self parameter. Change it like this.
def process_request(self, data):
# do some stuffs here
return {'foo': 'bar'}
Your test should pass.
But, that not the right way to implement Flask-Restful controolers. Read the doc and use get and post methods…
I have the following endpoint,
#developer_blueprint.route("/init_db", methods=["POST"])
def initialize_database():
try:
upload_data(current_app)
logger.debug("Database entries upload.")
return jsonify({"result": "Database entries uploaded."}), 201
except Exception as e:
return jsonify({"error": str(e)})
def upload_data(app):
with open("src/core/data/data.json") as data_file:
data = json.load(data_file)
try:
current_app.db.put(("somenamespace", "test", "default"), data, None)
except Exception as e:
raise e
I'm trying to figure out how to unit test this (we need to get coverage on our code).
Do I just mock up app.db? How can I do that?
Any suggestions would be appreciated.
It is not uncommon to mock database calls for unit testing using something like unittest.mock and then run Aerospike in a container or VM for end-to-end testing.
However, keep in mind that the Aerospike Python client library is written in C for better performance and thus it is not easy to do partial patching (aka "monkey patching"). For example, you will get a TypeError: can't set attributes of built-in/extension type if you try to simply patch out aerospike.Client.put.
One approach is to create a mock client object to replace or sub-class the Aerospike client object. The implementation of this mock object depends on your code and the cases you are testing for.
Take the following example code in which app.db is an instance of the Aerospike client library:
# example.py
import aerospike
import json
class App(object):
db = None
def __init__(self):
config = {'hosts': [('127.0.0.1', 3000)]}
self.db = aerospike.client(config).connect()
def upload_data(app):
with open("data.json") as data_file:
data = json.load(data_file)
try:
app.db.put(("ns1", "test", "default"), data, None)
except Exception as e:
raise e
if __name__ == "__main__":
app = App()
upload_data(app)
In writing unit tests for the upload_data function let's assume you want to test for a success case which is determined to mean that the put method is called and no exceptions are raised:
# test.py
from unittest import TestCase, main
from unittest.mock import PropertyMock, patch
from example import App, upload_data
from aerospike import Client, exception
class MockClient(Client):
def __init__(self, *args, **kwargs):
pass
def put(self, *args, **kwargs):
return 0
class ExampleTestCase(TestCase):
def test_upload_data_success(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = client = MockClient()
app = App()
with patch.object(client, 'put') as put_mock:
upload_data(app)
put_mock.assert_called()
if __name__ == '__main__':
main()
In the test_upload_data_success method the App.db property is patched with the MockClient class instead of the aerospike.Client class. The put method of the MockClient instance is also patched so that it can be asserted that the put method gets called after upload_data is called.
To test that an exception raised by the Aerospike client is re-raised from the upload_data function, the MockClient class can be modified to raise an exception explicitly:
# test.py
from unittest import TestCase, main
from unittest.mock import PropertyMock, patch
from example import App, upload_data
from aerospike import Client, exception
class MockClient(Client):
def __init__(self, *args, **kwargs):
self.put_err = None
if 'put_err' in kwargs:
self.put_err = kwargs['put_err']
def put(self, *args, **kwargs):
if self.put_err:
raise self.put_err
else:
return 0
class ExampleTestCase(TestCase):
def test_upload_data_success(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = client = MockClient()
app = App()
with patch.object(client, 'put') as put_mock:
upload_data(app)
put_mock.assert_called()
def test_upload_data_error(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = MockClient(put_err=exception.AerospikeError)
app = App()
with self.assertRaises(exception.AerospikeError):
upload_data(app)
if __name__ == '__main__':
main()
I'm having trouble creating a global function accessible from within all classes. I receive an error from within user.py that says:
NameError: global name 'connectCentral' is not defined
Here is my current code.
project/model/__ init __.py:
"""The application's model objects"""
import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy import engine_from_config
from pylons import config
import central
#Establish an on-demand connection to the central database
def connectCentral():
engine = engine_from_config(config, 'sqlalchemy.central.')
central.engine = engine
central.Session.configure(bind=engine)
project/model/user.py
import project.model
class User(object):
def emailExists(self):
try:
connectCentral()
emails = central.Session.query(User).filter_by(email=self.email).count()
if (emails > 0):
return False
else:
return True
except NameError:
self.errors['email'] = 'E-Mail not set'
return False
Am I missing an import? Is there a better way to do this?
Thanks.
You need to qualify the name with the module (or package) it's in, so:
try:
project.model.connectCentral()
etc.