We Keep Coding

FastAPI Sqlalchemy get lastest result

I have setup my first Python FastAPI but getting stuck. I have a function that query some results. The following function query the first entry in the database on a specific date. Now I want the last entry on a date or all results sort by highest id but how do i do this?
def get_workday(db: Session, workday_date: date):
return db.query(DBWorkday).where(DBWorkday.date == workday_date).first()
full code:
from datetime import date
from fastapi import FastAPI, Depends
from pydantic import BaseModel
from typing import Optional, List
from sqlalchemy import Date, create_engine
from sqlalchemy.orm import declarative_base, sessionmaker, Session
from sqlalchemy import Column, String, Integer
app = FastAPI()
# SqlAlchemy Setup
SQLALCHEMY_DATABASE_URL = 'sqlite:///../db/database.db'
engine = create_engine(SQLALCHEMY_DATABASE_URL, echo=True, future=True)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
# Dependency
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
# A SQLAlchemny ORM
class DBWorkday(Base):
__tablename__ = 'workdays'
id = Column(Integer, primary_key=True, index=True)
date = Column(Date)
start_time = Column(String(4))
end_time = Column(String(4))
type = Column(String, nullable=True)
location = Column(String, nullable=True)
Base.metadata.create_all(bind=engine)
# Workday Pydantic model
class Workday(BaseModel):
date: date
start_time: str
end_time: str
type: Optional[str] = None
location: Optional[str] = None
class Config:
orm_mode = True
# Methods for interacting with the database
def get_workday(db: Session, workday_date: date):
return db.query(DBWorkday).where(DBWorkday.date == workday_date).first()
#app.get('/workday/{date}')
def get_workday_view(date: date, db: Session = Depends(get_db)):
return get_workday(db, date)

return db.query(DBWorkday).where(DBWorkday.date == workday_date).order_by(DBWorkday.id.desc()).first()

SQLALchemy: Where does the attribute error after data import come from?

I wrote an application using SQLAlchemy's object relational mapper to store and access data from an SQLite3 database.
I can call add_userto add one or multiple users and call get_users to get them
I can import data from excel and get them with get_users
I can import data from excel and add a user with add_user
BUT I can't get the users with the get_users function afterwards, because I'm getting the following error for the entry created with add_user: AttributeError: 'NoneType' object has no attribute 'id'
What am I doing wrong?
Here's a simple version of the application:
orm_test.py
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
def orm_setup():
Base = declarative_base()
engine = create_engine('sqlite:///:main:', echo=True)
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
return Base, engine, session
orm_test_class.py
from sqlalchemy import Column, Integer, String
from orm_test import orm_setup
Base = orm_setup()[0]
engine = orm_setup()[1]
class User(Base):
__tablename__ = 'person'
id = Column('id', Integer, primary_key=True)
username = Column('username', String, unique=True)
Base.metadata.create_all(bind=engine)
orm_test_functions.py
from orm_test_class import User
from orm_test import orm_setup
session = orm_setup()[2]
def add_user(name):
u = User()
user_name = str(name)
u.username = user_name
session.add(u)
session.commit()
def get_users():
users = session.query(User).all()
for user in users:
print(user.id, user.username)
session.close()
main.py
import fire
from orm_test_functions import add_user, get_users
if __name__ == '__main__':
fire.Fire()
data_import.py
import fire
import pandas as pd
from orm_test import orm_setup
# import engine from orm
engine = orm_setup()[1]
def data_import():
file = 'Data.xlsx'
df_user = pd.read_excel(file, sheet_name = 'User')
df_user.to_sql('person', engine, if_exists='replace', index=False)
# Command line interface
if __name__ == '__main__':
fire.Fire()

The problem is that df_to_sql drops the original table, which has a primary key defined, and replaces it with a table that does not define a primary key.
From the dataframe_to_sql docs
replace: Drop the table before inserting new values.
You can get around this by setting if_exists='append' instead of if_exists='replace'.
df_user.to_sql('person', engine, if_exists='append', index=False)
If necessary you can emulate the "replace" behaviour by deleting any existing records from the table before importing the data.
This is the code I used to reproduce and resolve:
import io
import sqlalchemy as sa
from sqlalchemy import orm
import pandas as pd
Base = orm.declarative_base()
class User(Base):
__tablename__ = 'person'
id = sa.Column('id', sa.Integer, primary_key=True)
username = sa.Column('username', sa.String, unique=True)
engine = sa.create_engine('sqlite://', echo=True, future=False)
# Drop all is redundant for in-memory db
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
sessionmaker = orm.sessionmaker(engine)
def add_user(name):
session = sessionmaker()
u = User()
user_name = str(name)
u.username = user_name
session.add(u)
session.commit()
def get_users():
session = sessionmaker()
users = session.query(User).all()
for user in users:
print(user.id, user.username)
print()
session.close()
DATA = """\
id,username
1,Alice
2,Bob
"""
buf = io.StringIO(DATA)
df_user = pd.read_csv(buf)
df_user.to_sql('person', engine, if_exists='append', index=False)
users = get_users()
add_user('Carol')
users = get_users()
engine.dispose()

You should set the column id with AUTOINCREMENT keyword, see https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#using-the-autoincrement-keyword

How to check if a SQLAlchemy mapping matches the existing tables [duplicate]

To prevent human errors, I'd like to check that the current SQL database schema matches the SQLAlchemy models code and there aren't migrations need to run on the application startup. Is there a way to iterate all models on SQLAlchemy and then see if the database schema is that what the model expects it to be?
This is to prevent the errors popping up later (HTTP 500 due to missing table, field, etc.)

Based on #yoloseem hints above, here is a complete answer:
import logging
from sqlalchemy import inspect
from sqlalchemy.ext.declarative.clsregistry import _ModuleMarker
from sqlalchemy.orm import RelationshipProperty
logger = logging.getLogger(__name__)
def is_sane_database(Base, session):
"""Check whether the current database matches the models declared in model base.
Currently we check that all tables exist with all columns. What is not checked
* Column types are not verified
* Relationships are not verified at all (TODO)
:param Base: Declarative Base for SQLAlchemy models to check
:param session: SQLAlchemy session bound to an engine
:return: True if all declared models have corresponding tables and columns.
"""
engine = session.get_bind()
iengine = inspect(engine)
errors = False
tables = iengine.get_table_names()
# Go through all SQLAlchemy models
for name, klass in Base._decl_class_registry.items():
if isinstance(klass, _ModuleMarker):
# Not a model
continue
table = klass.__tablename__
if table in tables:
# Check all columns are found
# Looks like [{'default': "nextval('sanity_check_test_id_seq'::regclass)", 'autoincrement': True, 'nullable': False, 'type': INTEGER(), 'name': 'id'}]
columns = [c["name"] for c in iengine.get_columns(table)]
mapper = inspect(klass)
for column_prop in mapper.attrs:
if isinstance(column_prop, RelationshipProperty):
# TODO: Add sanity checks for relations
pass
else:
for column in column_prop.columns:
# Assume normal flat column
if not column.key in columns:
logger.error("Model %s declares column %s which does not exist in database %s", klass, column.key, engine)
errors = True
else:
logger.error("Model %s declares table %s which does not exist in database %s", klass, table, engine)
errors = True
return not errors
Below is the py.test testing code to exercise this:
"""Tests for checking database sanity checks functions correctly."""
from pyramid_web20.system.model.sanitycheck import is_sane_database
from sqlalchemy import engine_from_config, Column, Integer, String
import sqlalchemy
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import sessionmaker, relationship
from sqlalchemy import ForeignKey
def setup_module(self):
# Quiet log output for the tests
import logging
from pyramid_web20.system.model.sanitycheck import logger
#logger.setLevel(logging.FATAL)
def gen_test_model():
Base = declarative_base()
class SaneTestModel(Base):
"""A sample SQLAlchemy model to demostrate db conflicts. """
__tablename__ = "sanity_check_test"
#: Running counter used in foreign key references
id = Column(Integer, primary_key=True)
return Base, SaneTestModel
def gen_relation_models():
Base = declarative_base()
class RelationTestModel(Base):
__tablename__ = "sanity_check_test_2"
id = Column(Integer, primary_key=True)
class RelationTestModel2(Base):
__tablename__ = "sanity_check_test_3"
id = Column(Integer, primary_key=True)
test_relationship_id = Column(ForeignKey("sanity_check_test_2.id"))
test_relationship = relationship(RelationTestModel, primaryjoin=test_relationship_id == RelationTestModel.id)
return Base, RelationTestModel, RelationTestModel2
def gen_declarative():
Base = declarative_base()
class DeclarativeTestModel(Base):
__tablename__ = "sanity_check_test_4"
id = Column(Integer, primary_key=True)
#declared_attr
def _password(self):
return Column('password', String(256), nullable=False)
#hybrid_property
def password(self):
return self._password
return Base, DeclarativeTestModel
def test_sanity_pass(ini_settings, dbsession):
"""See database sanity check completes when tables and columns are created."""
engine = engine_from_config(ini_settings, 'sqlalchemy.')
conn = engine.connect()
trans = conn.begin()
Base, SaneTestModel = gen_test_model()
Session = sessionmaker(bind=engine)
session = Session()
try:
Base.metadata.drop_all(engine, tables=[SaneTestModel.__table__])
except sqlalchemy.exc.NoSuchTableError:
pass
Base.metadata.create_all(engine, tables=[SaneTestModel.__table__])
try:
assert is_sane_database(Base, session) is True
finally:
Base.metadata.drop_all(engine)
def test_sanity_table_missing(ini_settings, dbsession):
"""See check fails when there is a missing table"""
engine = engine_from_config(ini_settings, 'sqlalchemy.')
conn = engine.connect()
trans = conn.begin()
Base, SaneTestModel = gen_test_model()
Session = sessionmaker(bind=engine)
session = Session()
try:
Base.metadata.drop_all(engine, tables=[SaneTestModel.__table__])
except sqlalchemy.exc.NoSuchTableError:
pass
assert is_sane_database(Base, session) is False
def test_sanity_column_missing(ini_settings, dbsession):
"""See check fails when there is a missing table"""
engine = engine_from_config(ini_settings, 'sqlalchemy.')
conn = engine.connect()
trans = conn.begin()
Session = sessionmaker(bind=engine)
session = Session()
Base, SaneTestModel = gen_test_model()
try:
Base.metadata.drop_all(engine, tables=[SaneTestModel.__table__])
except sqlalchemy.exc.NoSuchTableError:
pass
Base.metadata.create_all(engine, tables=[SaneTestModel.__table__])
# Delete one of the columns
engine.execute("ALTER TABLE sanity_check_test DROP COLUMN id")
assert is_sane_database(Base, session) is False
def test_sanity_pass_relationship(ini_settings, dbsession):
"""See database sanity check understands about relationships and don't deem them as missing column."""
engine = engine_from_config(ini_settings, 'sqlalchemy.')
conn = engine.connect()
trans = conn.begin()
Session = sessionmaker(bind=engine)
session = Session()
Base, RelationTestModel, RelationTestModel2 = gen_relation_models()
try:
Base.metadata.drop_all(engine, tables=[RelationTestModel.__table__, RelationTestModel2.__table__])
except sqlalchemy.exc.NoSuchTableError:
pass
Base.metadata.create_all(engine, tables=[RelationTestModel.__table__, RelationTestModel2.__table__])
try:
assert is_sane_database(Base, session) is True
finally:
Base.metadata.drop_all(engine)
def test_sanity_pass_declarative(ini_settings, dbsession):
"""See database sanity check understands about relationships and don't deem them as missing column."""
engine = engine_from_config(ini_settings, 'sqlalchemy.')
conn = engine.connect()
trans = conn.begin()
Session = sessionmaker(bind=engine)
session = Session()
Base, DeclarativeTestModel = gen_declarative()
try:
Base.metadata.drop_all(engine, tables=[DeclarativeTestModel.__table__])
except sqlalchemy.exc.NoSuchTableError:
pass
Base.metadata.create_all(engine, tables=[DeclarativeTestModel.__table__])
try:
assert is_sane_database(Base, session) is True
finally:
Base.metadata.drop_all(engine)

Check out Runtime Inspection API.
You can pass Engine to inspect() as well. Once you have sqlalchemy.engine.reflection.Inspector object, now you can use get_table_names(), get_columns(tbl_name), and any other methods(e.g. for primary keys, constraints, indexes, ...) to examine the "real" schema that your database has.

How can I decouple objects from a SQLAlchemy session such that I can insert them into a different database?

I have two databases (src and dst) and each database uses the same SQLAlchemy model. I would like to query src and insert the results of that query into dst.
When I attempt to insert the values from src into dst, my code throws a InvalidRequestError exception because the queried objects are attached to the src session.
#!/usr/bin/env python
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
uid = Column(Integer, primary_key=True)
username = Column(String, unique=True, nullable=False)
gid = Column(Integer, nullable=False)
password = Column(String)
gecos = Column(String)
home_directory = Column(String)
shell = Column(String)
def __init__(self, passwd_string):
passwd_fields = passwd_string.split(':')
self.username = passwd_fields[0]
self.password = passwd_fields[1]
self.uid = passwd_fields[2]
self.gid = passwd_fields[3]
self.gecos = passwd_fields[4]
self.home_directory = passwd_fields[5]
self.shell = passwd_fields[6]
def get_session(db_uri):
engine = create_engine(db_uri)
Base.metadata.create_all(engine)
return sessionmaker(bind=engine)()
def add_users(session, user_list):
session.add_all(user_list)
session.commit()
def main():
users = [
User('root:x:0:0:Root User:/:/bin/bash'),
User('genuser:x:100:100:Generic User:/home/user:/bin/bash')
]
# insert users to src database
src = get_session('sqlite:///src.sqlite')
add_users(src, users)
# query src database for users
src_users = src.query(User).all()
# insert users found in src database into dst database
try:
dst = get_session('sqlite:///dst.sqlite')
add_users(dst, src_users)
except InvalidRequestError as error:
print "InvalidRequestError: {}".format(error)
if __name__ == "__main__":
main()
Running above code generates the following exception/error:
InvalidRequestError: Object '<User at 0x10bc10250>' is already attached to session '1' (this is '2')
In response to #Deepak's answer, I implemented a copy() method in User:
def copy(self):
return User(
'{}:{}:{}:{}:{}:{}:{}'.format(
self.username,
self.password,
self.uid,
self.gid,
self.gecos,
self.home_directory,
self.shell
)
)
Which means I can now do:
try:
dst = get_session('sqlite:///dst.sqlite')
dst.add(user.copy()) for user in src_users
except InvalidRequestError as error:
print "InvalidRequestError: {}".format(error)

You are trying to query via Session of Database 1 and inserting the attached user object of this session to another session.
A better way would be inserting the values in 2nd table in the same way you are inserting in 1st table.
So get the values from table 1:
src_users = src.query(User).all()
duplicate_records = []
for row in src.query(User).all():
duplicate_user.append(User(row.username +":"+ row.password +":"+str(row.uid)+":"+str(row.gid)+":"+row.gecos+":"+row.home_directory+":"+row.shell))
And then insert this list in your second table.

How can I select all rows with sqlalchemy?

I am trying to get all rows from a table.
In controller I have:
meta.Session.query(User).all()
The result is [, ], but I have 2 rows in this table.
I use this model for the table:
import hashlib
import sqlalchemy as sa
from sqlalchemy import orm
from allsun.model import meta
t_user = sa.Table("users",meta.metadata,autoload=True)
class Duplicat(Exception):
pass
class LoginExistsException(Exception):
pass
class EmailExistsException(Exception):
pass
And next, in the same file:
class User(object):
def loginExists(self):
try:
meta.Session
.query(User)
.filter(User.login==self.login)
.one()
except orm.exc.NoResultFound:
pass
else:
raise LoginExistsException()
def emailExists(self):
try:
meta
.Session
.query(User)
.filter(User.email==self.email)
.one()
except orm.exc.NoResultFound:
pass
else:
raise EmailExistsException()
def save(self):
meta.Session.begin()
meta.Session.save(self)
try:
meta.Session.commit()
except sa.exc.IntegrityError:
raise Duplicat()
orm.mapper(User, t_user)

You can easily import your model and run this:
from models import User
# User is the name of table that has a column name
users = User.query.all()
for user in users:
print user.name

I use the following snippet to view all the rows in a table. Use a query to find all the rows. The returned objects are the class instances. They can be used to view/edit the values as required:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine, Sequence
from sqlalchemy import String, Integer, Float, Boolean, Column
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class MyTable(Base):
__tablename__ = 'MyTable'
id = Column(Integer, Sequence('user_id_seq'), primary_key=True)
some_col = Column(String(500))
def __init__(self, some_col):
self.some_col = some_col
engine = create_engine('sqlite:///sqllight.db', echo=True)
Session = sessionmaker(bind=engine)
session = Session()
for class_instance in session.query(MyTable).all():
print(vars(class_instance))
session.close()