I'm working on an async FastAPI project and I want to connect to the database during tests. Coming from Django, my instinct was to create pytest fixtures that take care of creating/dropping the test database. However, I couldn't find much documentation on how to do this. The most complete instructions I could find were in this tutorial, but they don't work for me because they are all synchronous. I'm somewhat new to async development so I'm having trouble adapting the code to work async. This is what I have so far:
import pytest
from sqlalchemy.ext.asyncio import create_async_engine, session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy_utils import database_exists, create_database
from fastapi.testclient import TestClient
from app.core.db import get_session
from app.main import app
Base = declarative_base()
#pytest.fixture(scope="session")
def db_engine():
default_db = (
"postgresql+asyncpg://postgres:postgres#postgres:5432/postgres"
)
test_db = "postgresql+asyncpg://postgres:postgres#postgres:5432/test"
engine = create_async_engine(default_db)
if not database_exists(test_db): # <- Getting error on this line
create_database(test_db)
Base.metadata.create_all(bind=engine)
yield engine
#pytest.fixture(scope="function")
def db(db_engine):
connection = db_engine.connect()
# begin a non-ORM transaction
connection.begin()
# bind an individual Session to the connection
Session = sessionmaker(bind=connection)
db = Session()
# db = Session(db_engine)
yield db
db.rollback()
connection.close()
#pytest.fixture(scope="function")
def client(db):
app.dependency_overrides[get_session] = lambda: db
PREFIX = "/api/v1/my-endpoint"
with TestClient(PREFIX, app) as c:
yield c
And this is the error I'm getting:
E sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: https://sqlalche.me/e/14/xd2s)
/usr/local/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py:67: MissingGreenlet
Any idea what I have to do to fix it?
You try to use sync engine with async session. Try to use:
from sqlalchemy.ext.asyncio import AsyncSession
Session = sessionmaker(bind= connection, class_=AsyncSession)
https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html
Related
how can get a list of table objects with an asynchronous session?
I tried many options but never found the right one.
This is how i get the session object itself
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy import MetaData
from sqlalchemy import inspect
from sqlalchemy.future import select
from utils import *
import asyncio
engine = create_async_engine(config.database_url).execution_options(autocommit=True)
async_session = sessionmaker(
engine, expire_on_commit=False, class_=AsyncSession
)
async def execute(*args):
return await (async_session()).execute(*args)
async def tables():
session= async_session()
#print(await session.run_sync( inspect(engine).get_table_names ))
asyncio.run(tables())
I am using FastAPI with asyncpg and sqlalchemy. The FastAPI app uses "long-lived" sqlalchemy sessions to facilitate the use of PostGres prepared statements, which speeds up request times.
Note that prepared statements only last for the duration of the current database session, cf. https://www.postgresql.org/docs/current/sql-prepare.html.
Using this setup, querying the database returns decimal.Decimal as dtype for certain columns, I require float.
I tried to fix this using the approach documented here:
https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html
Calls to the db still return columns in decimal.Decimal type, however.
What am I missing?
from fastapi import FastAPI
from sqlalchemy import event
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
app = FastAPI()
db_engine = create_async_engine(f"postgresql+asyncpg://user:pw#localhost:5432/my_db")
db_session = None
#app.on_event("startup")
async def startup_event():
# Generate db session
global db_session
db_session = AsyncSession(db_engine, autocommit=False, future=True)
# Prepare SQL statements for speeding up request times using https://www.postgresql.org/docs/current/sql-prepare.html
db_session = await prepare_statements(...)
# Register type codec
#event.listens_for(db_session.bind.sync_engine, "connect")
def register_custom_types(dbapi_connection):
dbapi_connection.run_async(
lambda connection: connection.set_type_codec(
'numeric', encoder=str, decoder=float,
format='text', schema = "pg_catalog"
)
)
My setup includes:
asyncpg 0.25.0
sqlalchemy 1.4.17
fastapi 0.65.1
python 3.9.4
Windows & Linux (works on neither)
Instead of:
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
# an Engine, which the Session will use for connection
# resources
engine = create_engine('sqlite:///...')
# create session and add objects
with Session(engine) as session:
session.add(some_object)
session.add(some_other_object)
session.commit()
I create a sessionmaker (according to example in documentation, see bellow):
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
# an Engine, which the Session will use for connection
# resources, typically in module scope
engine = create_engine('postgresql://scott:tiger#localhost/')
# a sessionmaker(), also in the same scope as the engine
Session = sessionmaker(engine)
# we can now construct a Session() without needing to pass the
# engine each time
with Session() as session:
session.add(some_object)
session.add(some_other_object)
session.commit()
Can I use the sessions from session maker in different threads (spawning multiple sessions at the same time)? In other words, is session maker thread safe object? If yes, can multiple sessions exists and read/write into same tables at the same time?
Furthermore, what is the advantage of of using 'scoped_session' - is it realated to problem of multiple sessions (one per thread)?:
# set up a scoped_session
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker
session_factory = sessionmaker(bind=some_engine)
Session = scoped_session(session_factory)
# now all calls to Session() will create a thread-local session
some_session = Session()
# you can now use some_session to run multiple queries, etc.
# remember to close it when you're finished!
Session.remove()
Session objects are not thread-safe, but are thread-local. What I recommend using is sessionmaker instead of Session. It will yield a Session object every time you need it, thus not idling the database connection. I'd use the approach below.
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
DB_ENGINE = create_engine('sqlite:///...')
DB_SES_MAKER = sessionmaker(bind=DB_ENGINE)
def get_db():
db = DB_SES_MAKER()
try:
yield db
finally:
db.close()
Then call get_db whenever needed:
db = next(get_db())
I'm using fastapi to create a basic API to do some statistics from a postgres database.
I have just started using sqlalchemy as I want to do connection pooling, and based off of my googling it seems the route to go down.
I've implemented this in my main.py file,
def get_db():
try:
db = SessionLocal()
yield db
finally:
db.close()
Then using depends from fastapi with my URL params,
async def get_data(xxx ,db: SessionLocal = Depends(get_db)):
conn = db()
With the sessions function being,
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from environment.config import settings
SQLALCHEMY_DATABASE_URL = settings.DATABASE_URL
engine = create_engine(SQLALCHEMY_DATABASE_URL)
SessionLocal = sessionmaker(autocommit=False,autoflush=False,bind=engine)
I'm receiving a type error of SessionLocal not being callable, wondering what I'm missing here?
The issue I was having was when testing the API against being called, multiple calls to the API were essentially recreating a connection to the database, which was super laggy just testing locally - so wanting to make that...well work :)
Imports for Main.py
from plistlib import UID
import string
from typing import Optional
from pandas import concat
from fastapi import FastAPI, HTTPException, Header,Depends
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from db.session import SessionLocal
from biometrics.TaskGet import GetSentiment, GetScheduled, GetAllActualSubstance, GetEmotions
import aggregate.biometric_aggregators as biometric_aggregators
import os
Based on answer below, I just let it use DB. Now I get this error weirdly though.
web_1 | AttributeError: 'Session' object has no attribute 'cursor'
Made sqlalchemy functions to do the same calls, now getting the same error as before.
I originally tried to just return the function without using pd.read_sql, but it didn't work - any idea on what I've done wrong here?
from sqlalchemy.orm import Session
import pandas as pd
from . import models
def sentimentDataframe(db: Session, user_id: str):
Sentiment = pd.read_sql((get_sentiment(db,user_id)),con=db)
Sentiment['created'] =pd.to_datetime(Sentiment['created'], unit='s')
return Sentiment.set_index('created')
def get_sentiment(db: Session, user_id: str, skip: int = 0, limit: int = 100):
return db.query(models.Sentiment).filter(models.Sentiment.user_id == user_id).order_by(models.Sentiment.created.desc()).offset(skip).limit(limit).all()
You should not be doing this
conn = db()
as db is already an instance of session
You can already use it like so
db.add(<SQLAlchemy Base Instance>)
db.commit()
Im writting a flask api using flaskrestful,sqlalchemy, Postgres, nginx,uwsgi. Im a newbie to python.These are my configuration
database.py
from cuewords import app
from flask.ext.sqlalchemy import SQLAlchemy
from sqlalchemy.pool import NullPool
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, String , Text , Boolean , DateTime, MetaData, Table ,Sequence
from sqlalchemy.dialects.postgresql import JSON
Base = declarative_base()
db_name="postgresql+psycopg2://admin:password#localhost:5434/database_name"
from sqlalchemy.orm import sessionmaker
engine = create_engine(db_name,poolclass=NullPool ,echo=True)
Session = sessionmaker(autocommit=False ,bind=engine)
connection = engine.connect()
metadata = MetaData()
api.py
class Webcontent(Resource):
def post(self):
session=Session()
...assign some params...
try:
insert_data=Websitecontent(site_url,publisher_id)
session.add(insert_data)
session.commit()
Websitecontent.update_url(insert_data.id,session)
except:
session.rollback()
raise
finally:
return "Data created "
session.close()
else:
return "some value"
Here im first saving the just the url then saving all the content of the site using boilerpipe later .Idea is to move to queue later
model.py
class Websitecontent(Base):
#classmethod
def update_url(self,id,session):
existing_record=session.query(Websitecontent).filter_by(id=int(id)).first()
data=Processing.processingContent(str(existing_record.url))
#boilerpipe processing the content here
#assigning some data to existing record in session
session.add(existing_record)
session.commit()
Websitecontent.processingWords(existing_record,session)
#classmethod
def processingWords(self,record,session)
...processing
Websitecontent.saveKeywordMapping(session,keyword_url,record)
#classmethod
def saveKeywordMapping(session,keyword_url,record)
session.commit()
session.close()
So this code works perfectly in locally but its doesnt work in production .So when i check pag_stat_activity it show the state "idle in transaction". The app hangs then i have to restart the servers. i dont get it why session.close() does not close the pool connection why its keeping psql transaction state busy . Guys any help would be really appreciated.
You are returning before closing the session:
return "Data created "
session.close()
I think returning inside finally might swallow the exception, as well.