I have tried unsuccesfully to save fetched API data to sqlite database in a Flask app. I have used requests.get() to extract external API data to dataframe. The function "extract_to_df_race" works when i test it in Jupyter Notebook. I have placed try-except statements to print error messages to console. Since, there were no error messages logged in console, I initally presume that the data has been successfully fetched and save to database. However, upon checking the database, none of the records have been saved.
I have used a custom Flask command to execute the 'historical_records' function to one-off load the database.
Are there any better methods of debugging that i could try?
app/api/log.py
from app import app
from app.models import Race, db
from app.utils import *
import click
#app.cli.command()
def historical_records():
seasons = [2015]
races_round = range(1,5)
df_races = extract_to_df_race('results', seasons, races_round)
save_races_to_db(df_races, db)
def save_races_to_db(df_races, db):
for idx,row in df_races.iterrows():
r = Race()
r.url = df_races.loc[idx,"url"]
r.season = df_races.loc[idx,"season"]
r.raceName = df_races.loc[idx,"raceName"]
db.session.add(r)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
eprint(str(e))
To execute historical_records function from virtual environment, i ran "export FLASK_APP=app/api/log.py", then "flask historical_records"
app/utils.py
from __future__ import print_function
import requests
import json
import pandas as pd
import datetime
import sys
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def extract_to_df_race(results_type, seasons, races_round):
df_races = pd.DataFrame()
if results_type == 'results':
for s in seasons:
for r in races_round:
try:
response = requests.get(API_URL)
response.raise_for_status()
dictionary = response.content
dictionary = json.loads(dictionary)
races = transform_func(dictionary, s, r)
df_races = pd.concat([df_races, races])
except requests.exceptions.HTTPError as err:
eprint(err)
sys.exit(1)
return df_races
Races model
class Race(db.Model, Serializer):
__tablename__ = 'races'
raceId = db.Column(db.Integer, primary_key=True)
url = db.Column(db.String(50), unique=True)
season = db.Column(db.Integer)
raceName = db.Column(db.String(50))
def __init__(self, **kwargs):
super(Race, self).__init__(**kwargs)
Related
Description of the problem
I wanted to use ormar with SQLite for my project but ran into the problem that ormar doesn't save changes to the database. Although everything seems to be done according to the documentation. (Additionally, I used the faker to generate unique names to fill db and loguru to logging)
My code
import asyncio
from databases import Database
from faker import Faker
from loguru import logger as log
from ormar import ModelMeta, Model, Integer, String
from sqlalchemy import create_engine, MetaData
fake = Faker()
DB_PATH = 'sqlite:///db.sqlite3'
database = Database(DB_PATH)
metadata = MetaData()
class BaseMeta(ModelMeta):
database = database
metadata = metadata
class User(Model):
class Meta(BaseMeta):
tablename = 'users'
id: int = Integer(primary_key=True)
name: str = String(max_length=64)
# Also I tried without `with_connect` function, but it also doesn't work
async def with_connect(function):
async with database:
await function()
async def create():
return f"User created: {await User(name=fake.name()).save()}"
# Also I tried this: `User.objects.get_or_create(name=fake.name())`
# but it also doesn't work:
async def read():
return f"All data from db: {await User.objects.all()}"
async def main():
log.info(await create())
log.info(await create())
log.info(await read())
if __name__ == '__main__':
engine = create_engine(DB_PATH)
metadata.drop_all(engine)
metadata.create_all(engine)
try:
asyncio.run(with_connect(main))
finally:
metadata.drop_all(engine)
Results
As a result, I expected that after each run of the code, data would be printed during previous runs. That is so that the created objects are saved to the file db.sqlite3.
The actual result is that after each run of the code, only the data generated during that run is printed.
Conclusion
Why is the data not saved to the database file? Maybe I misunderstood how ORMs work?
I am studying the "Cosmic Python" book and chapter 6 explains how to use the Unit of Work pattern to change the interaction with the database/repository.
Chapter 6 of the book can be accessed here:
https://www.cosmicpython.com/book/chapter_06_uow.html
The code provided by the author is the following:
from __future__ import annotations
import abc
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm.session import Session
from allocation import config
from allocation.adapters import repository
class AbstractUnitOfWork(abc.ABC):
products: repository.AbstractRepository
def __enter__(self) -> AbstractUnitOfWork:
return self
def __exit__(self, *args):
self.rollback()
#abc.abstractmethod
def commit(self):
raise NotImplementedError
#abc.abstractmethod
def rollback(self):
raise NotImplementedError
DEFAULT_SESSION_FACTORY = sessionmaker(bind=create_engine(
config.get_postgres_uri(),
isolation_level="REPEATABLE READ",
))
class SqlAlchemyUnitOfWork(AbstractUnitOfWork):
def __init__(self, session_factory=DEFAULT_SESSION_FACTORY):
self.session_factory = session_factory
def __enter__(self):
self.session = self.session_factory() # type: Session
self.products = repository.SqlAlchemyRepository(self.session)
return super().__enter__()
def __exit__(self, *args):
super().__exit__(*args)
self.session.close()
def commit(self):
self.session.commit()
def rollback(self):
self.session.rollback()
I am trying to test my endpoints on Flask but I could not make it rollback the data inserted after each test.
To solve that I tried to install the package pytest-flask-sqlalchemy but with the following error:
'SqlAlchemyUnitOfWork' object has no attribute 'engine'
I do not quite understand how pytest-flask-sqlalchemy works and I have no clue on how to make the Unit of Work rollback transactions after a test.
Is it possible to make it work the way the author implemented it?
Edited
It is possible to replicate my situation through the following repository:
https://github.com/Santana94/CosmicPythonRollbackTest
You should get that the test is not rolling back previous actions by cloning it and running make all.
Finally, I got to make the rollback functionality happen after every test.
I got that working when I saw a package called pytest-postgresql implementing it on itself. I just made my adjustments to make tests rollback the database data that I was working with. For that, I just had to implement this function on conftest.py:
#pytest.fixture(scope='function')
def db_session():
engine = create_engine(config.get_postgres_uri(), echo=False, poolclass=NullPool)
metadata.create_all(engine)
pyramid_basemodel.Session = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
pyramid_basemodel.bind_engine(
engine, pyramid_basemodel.Session, should_create=True, should_drop=True)
yield pyramid_basemodel.Session
transaction.commit()
metadata.drop_all(engine)
After that, I had to place the db_session as a parameter of a test if I wanted to rollback transactions:
#pytest.mark.usefixtures('postgres_db')
#pytest.mark.usefixtures('restart_api')
def test_happy_path_returns_202_and_batch_is_allocated(db_session):
orderid = random_orderid()
sku, othersku = random_sku(), random_sku('other')
earlybatch = random_batchref(1)
laterbatch = random_batchref(2)
otherbatch = random_batchref(3)
api_client.post_to_add_batch(laterbatch, sku, 100, '2011-01-02')
api_client.post_to_add_batch(earlybatch, sku, 100, '2011-01-01')
api_client.post_to_add_batch(otherbatch, othersku, 100, None)
r = api_client.post_to_allocate(orderid, sku, qty=3)
assert r.status_code == 202
r = api_client.get_allocation(orderid)
assert r.ok
assert r.json() == [
{'sku': sku, 'batchref': earlybatch},
]
It is possible to check out the requirements for that and other aspects of that implementation on my GitHub repository.
https://github.com/Santana94/CosmicPythonRollbackTest
I have a file called redis_db.py which has code to connect to redis
import os
import redis
import sys
class Database:
def __init__(self, zset_name):
redis_host = os.environ.get('REDIS_HOST', '127.0.0.1')
redis_port = os.environ.get('REDIS_PORT', 6379)
self.db = redis.StrictRedis(host=redis_host, port=redis_port)
self.zset_name = zset_name
def add(self, key):
try:
self.db.zadd(self.zset_name, {key: 0})
except redis.exceptions.ConnectionError:
print("Unable to connect to redis host.")
sys.exit(0)
I have another file called app.py which is like this
from flask import Flask
from redis_db import Database
app = Flask(__name__)
db = Database('zset')
#app.route('/add_word/word=<word>')
def add_word(word):
db.add(word)
return ("{} added".format(word))
if __name__ == '__main__':
app.run(host='0.0.0.0', port='8080')
Now I am writing unit test for add_word function like this
import unittest
import sys
import os
from unittest import mock
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../api/")
from api import app # noqa: E402
class Testing(unittest.TestCase):
def test_add_word(self):
with mock.patch('app.Database') as mockdb:
mockdb.return_value.add.return_value = ""
result = app.add_word('shivam')
self.assertEqual(result, 'shivam word added.')
Issue I am facing is that even though I am mocking the db method call it is still calling the actual method in the class instead of returning mocked values and during testing I am getting error with message Unable to connect to redis host..
Can someone please help me in figuring out how can I mock the redis database calls.
I am using unittest module
The issue is that db is defined on module import, so the mock.patch does not affect the db variable. Either you move the instantiation of
db in the add_word(word) function or you patch db instead of Database, e.g.
def test_add_word():
with mock.patch('api.app.db') as mockdb:
mockdb.add = mock.MagicMock(return_value="your desired return value")
result = app.add_word('shivam')
print(result)
Note that the call of add_word has to be in the with block, otherwise the unmocked version is used.
I have the following endpoint,
#developer_blueprint.route("/init_db", methods=["POST"])
def initialize_database():
try:
upload_data(current_app)
logger.debug("Database entries upload.")
return jsonify({"result": "Database entries uploaded."}), 201
except Exception as e:
return jsonify({"error": str(e)})
def upload_data(app):
with open("src/core/data/data.json") as data_file:
data = json.load(data_file)
try:
current_app.db.put(("somenamespace", "test", "default"), data, None)
except Exception as e:
raise e
I'm trying to figure out how to unit test this (we need to get coverage on our code).
Do I just mock up app.db? How can I do that?
Any suggestions would be appreciated.
It is not uncommon to mock database calls for unit testing using something like unittest.mock and then run Aerospike in a container or VM for end-to-end testing.
However, keep in mind that the Aerospike Python client library is written in C for better performance and thus it is not easy to do partial patching (aka "monkey patching"). For example, you will get a TypeError: can't set attributes of built-in/extension type if you try to simply patch out aerospike.Client.put.
One approach is to create a mock client object to replace or sub-class the Aerospike client object. The implementation of this mock object depends on your code and the cases you are testing for.
Take the following example code in which app.db is an instance of the Aerospike client library:
# example.py
import aerospike
import json
class App(object):
db = None
def __init__(self):
config = {'hosts': [('127.0.0.1', 3000)]}
self.db = aerospike.client(config).connect()
def upload_data(app):
with open("data.json") as data_file:
data = json.load(data_file)
try:
app.db.put(("ns1", "test", "default"), data, None)
except Exception as e:
raise e
if __name__ == "__main__":
app = App()
upload_data(app)
In writing unit tests for the upload_data function let's assume you want to test for a success case which is determined to mean that the put method is called and no exceptions are raised:
# test.py
from unittest import TestCase, main
from unittest.mock import PropertyMock, patch
from example import App, upload_data
from aerospike import Client, exception
class MockClient(Client):
def __init__(self, *args, **kwargs):
pass
def put(self, *args, **kwargs):
return 0
class ExampleTestCase(TestCase):
def test_upload_data_success(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = client = MockClient()
app = App()
with patch.object(client, 'put') as put_mock:
upload_data(app)
put_mock.assert_called()
if __name__ == '__main__':
main()
In the test_upload_data_success method the App.db property is patched with the MockClient class instead of the aerospike.Client class. The put method of the MockClient instance is also patched so that it can be asserted that the put method gets called after upload_data is called.
To test that an exception raised by the Aerospike client is re-raised from the upload_data function, the MockClient class can be modified to raise an exception explicitly:
# test.py
from unittest import TestCase, main
from unittest.mock import PropertyMock, patch
from example import App, upload_data
from aerospike import Client, exception
class MockClient(Client):
def __init__(self, *args, **kwargs):
self.put_err = None
if 'put_err' in kwargs:
self.put_err = kwargs['put_err']
def put(self, *args, **kwargs):
if self.put_err:
raise self.put_err
else:
return 0
class ExampleTestCase(TestCase):
def test_upload_data_success(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = client = MockClient()
app = App()
with patch.object(client, 'put') as put_mock:
upload_data(app)
put_mock.assert_called()
def test_upload_data_error(self):
with patch.object(App, 'db', new_callable=PropertyMock) as db_mock:
db_mock.return_value = MockClient(put_err=exception.AerospikeError)
app = App()
with self.assertRaises(exception.AerospikeError):
upload_data(app)
if __name__ == '__main__':
main()
hi i am building a desktop application in PyQt python and have a web browser loaded in that , now i want to add functionalities of http fox(Firefox plugin) to view the loaded URLs with the request passed and other headers associated with each URL same as in http fox.
I have written the code from showing the loaded URLs but not finding a way to show the other headers on click of each URL. i have heard about Cookie Jar in Qwebview but do not know how to show with each loaded URL.
My code is:-
class Manager(QNetworkAccessManager):
def __init__(self, table):
QNetworkAccessManager.__init__(self)
self.finished.connect(self._finished)
self.table = table
def _finished(self, reply):
headers = reply.rawHeaderPairs()
headers = {str(k):str(v) for k,v in headers}
content_type = headers.get("Content-Type")
url = reply.url().toString()
status = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
status, ok = status.toInt()
self.table.update([url, str(status), content_type])
i want somethinglike-
[![here on the upper part we have loaded URLs and below that we can see the header, i have written the code for loaded URLs but how to show the headers][1]][1]
Is this what you are looking for?
import logging
import sys
from PyQt4.QtNetwork import QNetworkAccessManager, QNetworkRequest
from PyQt4.QtCore import QUrl, QEventLoop
log = logging.getLogger(__name__)
class Manager(QNetworkAccessManager):
def __init__(self, table=list()):
QNetworkAccessManager.__init__(self)
self.finished.connect(self._finished)
self.table = table
def _finished(self, reply):
headers = reply.rawHeaderPairs()
headers = {str(k): str(v) for k, v in headers}
content_type = headers.get("Content-Type")
url = reply.url().toString()
status = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
status, ok = status.toInt()
self.table.append([str(url), str(status), content_type])
log.info(self.table)
request = reply.request()
log.info(request.rawHeader("User-Agent"))
method = reply.operation()
if method == QNetworkAccessManager.GetOperation:
log.info("get")
request.url().queryItems()
if method == QNetworkAccessManager.PostOperation:
log.info("post")
def test():
manager = Manager()
log.info("Sending request")
manager.get(QNetworkRequest(QUrl("http://www.google.com/")))
# just for testing purpose to wait for the request to finish
l = QEventLoop()
manager.finished.connect(l.quit)
l.exec_()
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
app = QApplication(sys.argv)
test()