I'm trying to implement a basic resource access using SQL Alchemy 1.4 and a PostgreSQL database.
Python code
from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base
Base: DeclarativeMeta = declarative_base()
class User(Base):
__tablename__ = 'user'
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
email = Column(String(length=255), index=True, nullable=False)
class Resource(Base):
__tablename__ = "resource"
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
name = Column(String(length=255), index=True, nullable=False)
class UserResourceRole(Base):
__tablename__ = "user_resource_role"
user_id = Column(
UUID(as_uuid=True),
ForeignKey("user.id", ondelete="CASCADE"),
primary_key=True
)
resource_id = Column(
UUID(as_uuid=True),
ForeignKey("resource.id", ondelete="CASCADE"),
primary_key=True,
)
can_edit = Column(Boolean, default=False, nullable=False)
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.ext.asyncio.engine import AsyncEngine
from sqlalchemy.orm import sessionmaker
from the_other_file import User, Resource, UserResourceRole
async def select_all(user: User, db_session: AsyncSession):
results = await db_session.execute(
select(Resource, UserResourceRole)
.join(
UserResourceRole,
Resource.id == UserResourceRole.resource_id
)
.where(UserResourceRole.user_id == user.id)
)
return results.scalars().all()
engine: AsyncEngine = create_async_engine(POSTGRES_URL, future=True)
async_session = sessionmaker(
bind=engine, class_=AsyncSession, expire_on_commit=False, future=True
)
# ignore the fact that it's not executed in asyncio loop
a_db_session = await async_session()
resources = await select_all(user=a_real_user_is_here, db_session=a_db_session)
print(resources)
I can't retrieve anything from UserResourceRole in my results. It only contains data from Resource. If i swap the objects in the select call, then I can only retrieve data from UserResourceRole.
What I'm expecting
I'm expecting to have the same result of that SQL query :
SELECT *
FROM resource
INNER JOIN user_resource_role
ON resource.id = user_resource_role.resource_id
WHERE user_resource_role.user_id = :a_user_id
The query generated by SQL Alchemy is exactly the same (except the verbosity) :
SELECT resource.id, resource.name, user_resource_role.user_id, user_resource_role.resource_id, user_resource_role.can_edit
FROM resource
JOIN user_resource_role
ON resource.id = user_resource_role.resource_id
WHERE user_resource_role.user_id = :user_id_1
If you try
for entry in results:
print(entry)
it will show you a list of tuple of (Resource, UserResourceRole). Apparently the call to .scalars().all() only leave the first value.
My current solution is to turn results into a list and manually manipulate it.
Remove scalars(). So, it should be
return results.all()
This will return a list of tuples
[(resource_1, userresourcerole_1),(resource_2, userresourcerole_2),...]
Related
I would like to query a user based on their code and mobile number.
Of note, I am using an async session when connecting to the DB.
This is my models and query:
class model_user(Base):
__tablename__ = "users"
id = Column(UUID)
code = Column(Integer)
user_name = Column(String(11), unique=True)
first_name = Column(String(255), index=True)
last_name = Column(String(255), index=True)
contact = relationship("model_contact", back_populates="user")
class model_contact(Base):
__tablename__ = "contacts"
id = Column(Integer)
mobile = Column(String(320))
user_id = Column(
UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True
)
user = relationship("model_user", back_populates="contact")
# Here is my query:
query = await db.execute(
select(user_model)\
.filter_by(code==5)\
.options(
joinedload(model_user.contact)
)\
.filter(model_contact.mobile == mobile)
This really has nothing to do with whether or not you are using the async extension of SQLAlchemy. Queries are constructed the same way. Only the session setup and interaction is obviously different.
Side notes:
You should use PascalCase to name your classes and including the term "model" in the name is typically not good style, i.e. User and Contact.
Since you have a one-to-many relationship between your user model and your contact model (i.e. one user can have multiple sets of contact info), you should name the relationship attribute on the user model with plural, i.e. contacts.
The simplest way to do what you want that I can think of is using the Select.where method. You can then construct an SQL EXISTS subquery using the any method of the relationship. The statement would look like this:
statement = select(User).where(
User.code == 123,
User.contacts.any(Contact.mobile == "555")
)
Here is a full working example using aiosqlite just to demonstrate that this works with async tools:
from asyncio import run
from sqlalchemy import Column, ForeignKey, Integer, String, select
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
Base = declarative_base()
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
code = Column(Integer)
contacts = relationship("Contact", back_populates="user")
class Contact(Base):
__tablename__ = "contacts"
id = Column(Integer, primary_key=True)
mobile = Column(String(320))
user_id = Column(
Integer(), ForeignKey("users.id"), nullable=False, index=True
)
user = relationship(User, back_populates="contacts")
async def main():
engine = create_async_engine("sqlite+aiosqlite://", echo=True)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
async_session = sessionmaker(
engine, expire_on_commit=False, class_=AsyncSession
)
async with async_session() as session:
statement = select(User).where(
User.code == 123,
User.contacts.any(Contact.mobile == "555")
)
await session.execute(statement)
if __name__ == "__main__":
run(main())
Running this script produces the following SQL output for the select query:
SELECT users.id, users.code
FROM users
WHERE users.code = ? AND (EXISTS (SELECT 1
FROM contacts
WHERE users.id = contacts.user_id AND contacts.mobile = ?))
...
(123, '555')
This approach should give you the results you want.
Using sqlalchemy 1.4.x, I've set up the following classes:
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm import declarative_base, sessionmaker
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.orm import backref, relationship
from sqlalchemy import ForeignKey, Column, Integer, Unicode
from sqlalchemy import create_engine
engine = create_engine("sqlite:///:memory:", echo=True)
Session = sessionmaker(bind=engine)
session = Session()
Base = declarative_base()
class Infra(Base):
__tablename__ = "infra"
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(Unicode(200), index=True, unique=True)
protections = association_proxy(
"infra_protections",
"pscore",
creator=lambda k, v: Infra_Protection(protection=k, pscore=v),
)
class Protection(Base):
__tablename__ = "protection"
id = Column(Integer, primary_key=True, autoincrement=True)
ptype = Column(Unicode(200), index=True, unique=True)
def __init__(self, protection):
self.ptype = protection
class Infra_Protection(Base):
__tablename__ = "infraprotection"
infra_id = Column(
Integer, ForeignKey("infra.id", ondelete="CASCADE"), primary_key=True
)
protection_id = Column(
Integer, ForeignKey("protection.id", ondelete="CASCADE"), primary_key=True
)
prot = relationship("Protection")
protection = association_proxy("prot", "ptype")
infra = relationship(
Infra,
backref=backref(
"infra_protections",
collection_class=attribute_mapped_collection("protection"),
cascade="all, delete-orphan",
),
)
pscore = Column(Integer, nullable=False, unique=False, server_default="0")
Now I'd like to add some Infra objects, and associated Protections:
Base.metadata.create_all(engine)
i = Infra(name="Foo")
i.protections["test"] = 1
i.protections["test 2"] = 2
session.add(i)
session.commit()
# now, add another
j = Infra(name="Bar")
j.protections["test"] = 3
j.protections["test 2"] = 4
session.add(j)
session.commit() # UNIQUE constraint failed: protection.ptype
It's obvious why the unique constraint is violated, but I'm wondering how I can modify my association proxy setup to avoid this in a reasonably robust way. Some sort of get_or_create on the Protection __init__?
The way to deal with this is by implementing one of the UniqueObject recipes from here: https://github.com/sqlalchemy/sqlalchemy/wiki/UniqueObject
Here is my model:
user_map = Table(
"user_map",
Column('user_id', Integer, ForeignKey('user.id'), primary_key=True),
Column('map_id', Integer, ForeignKey('map.id'), primary_key=True),
PrimaryKeyConstraint('user_id', 'map_id', name='pk_user_map')
)
class Map(Base):
id = Column(Integer, primary_key=True)
name = Column(String)
owner_id = Column(Integer, ForeignKey('user.id'))
shared_maps = relationship(
'User',
secondary=user_map,
backref=backref('maps', lazy='dynamic')
)
class User(Base):
id = Column(Integer, primary_key=True)
name = Column(String)
email = Column(String, unique=True)
shared_maps = Map.query.filter(Map.shared_maps.any()).all()
I want to query the user_map table, using the join condition "Map.id == user_map.map_id", but SQLAlchemy is trying to join using "Map.id == user_map.map_id and Map.owner_id == user_map.user_id". How can I specify my join condition?
I tried to use primaryjoin attribute in the relationship and to specify the condition inside the .join() but without success. Thanks in advance!
Based on your code, I've rebuilt the setup; I guess your relationships were mixed up. Furthermore, I've hardly ever seen primary keys (or PrimaryKeyConstraints) in sqlalchemy's many-to-many association tables. It may make sense from a non-orm perspective, but as far as I know, it is unusual or even not required at all.
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
Base = declarative_base()
UsersXMaps = sa.Table(
'users_x_maps',
Base.metadata,
sa.Column('user', sa.Integer, sa.ForeignKey('users.id')),
sa.Column('map', sa.Integer, sa.ForeignKey('maps.id'))
)
class User(Base):
__tablename__ = 'users'
id = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.String)
mail = sa.Column(sa.String, unique=True)
own_maps = relationship('Map', back_populates='owner')
maps = relationship(
'Map',
secondary=UsersXMaps,
back_populates='users'
)
def __str__(self):
return '{} ({}) with {} maps'.format(
self.name, self.mail, len(self.own_maps))
class Map(Base):
__tablename__ = 'maps'
id = sa.Column(sa.Integer, primary_key=True)
name = sa.Column(sa.String)
owner_id = sa.Column(sa.Integer, sa.ForeignKey('users.id'))
owner = relationship('User', back_populates='own_maps')
users = relationship(
'User',
secondary=UsersXMaps,
back_populates='maps'
)
def __str__(self):
return '{} (by {})'.format(self.name, self.owner.name)
So far for the setup; I've extended it a bit for proper output when printing strings. Additionally, your Map.shared_maps relationship actually refers to Users, not Maps, so I also renamed that one.
When binding your association table to the two classes, you can refer to it from both sides (even though back_populates seems to overwrite/replace the original definition) - this simplifies joins from either side.
Executing the following works as expected:
if __name__ == '__main__':
engine = sa.create_engine('sqlite:///usermaps.db')
sfactory = sessionmaker(engine)
session = sfactory()
Base.metadata.create_all(bind=engine)
bilbo = User(id=1, name='Bilbo', mail='bilbo#shire.nz')
frodo = User(id=2, name='Frodo', mail='frodo#shire.nz')
mordor = Map(id=1, name='Mordor', owner=frodo, users=[bilbo, frodo])
gondor = Map(id=2, name='Gondor', owner=bilbo, users=[bilbo, frodo])
rohan = Map(id=3, name='Rohan', owner=bilbo, users=[bilbo, frodo])
session.add_all([frodo, bilbo, mordor, gondor, rohan])
session.commit()
print('Maps by owner:')
for owner in [bilbo, frodo]:
print(owner)
for item in session.query(Map).filter(Map.owner == owner).all():
print(' - ' + str(item))
print('Maps by users:')
for item in session.query(Map).filter(Map.users.any()).all():
print(' - ' + str(item))
The output is:
Maps by owner:
Bilbo (bilbo#shire.nz) with 2 maps
- Gondor (by Bilbo)
- Rohan (by Bilbo)
Frodo (frodo#shire.nz) with 1 maps
- Mordor (by Frodo)
Maps by users:
- Mordor (by Frodo)
- Gondor (by Bilbo)
- Rohan (by Bilbo)
I'm trying to figure out why I need to use a no_autoflush block when inserting data into an association proxy if the association proxy data has been accessed first. An example of this is bellow (using MySQL):
from sqlalchemy import create_engine, Integer, Column, String, ForeignKey, UniqueConstraint
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, scoped_session
Base = declarative_base()
engine = create_engine('{}://{}:{}#{}/{}'.format(...))
session_factory = sessionmaker(bind=engine)
Session = scoped_session(session_factory)
class DomainModel(Base):
__tablename__ = 'domains'
id = Column(Integer, primary_key=True)
name = Column(String(255), nullable=False, unique=True)
domains_to_servers = relationship("DomainServerModel", back_populates="domain")
servers = association_proxy('domains_to_servers', 'server',
creator=lambda s: DomainServerModel(server=s))
class ServerModel(Base):
__tablename__ = 'servers'
id = Column(Integer, primary_key=True)
name = Column(String(128), nullable=False, unique=True, index=True)
domains_to_servers = relationship("DomainServerModel", back_populates="server")
domains = association_proxy('domains_to_servers', 'domain',
creator=lambda d: DomainServerModel(domain=d))
class DomainServerModel(Base):
__tablename__ = 'domains_to_servers'
id = Column(Integer, primary_key=True)
domain_id = Column(Integer, ForeignKey('domains.id'), nullable=False)
server_id = Column(Integer, ForeignKey('servers.id'), nullable=False)
server = relationship('ServerModel', back_populates="domains_to_servers")
domain = relationship('DomainModel', back_populates="domains_to_servers")
def test():
session = Session()
with session.no_autoflush:
s = session.query(ServerModel).filter_by(name='test.com').one()
print(s.domains)
d = DomainModel(name='test1.com')
session.add(d)
session.commit()
s.domains.append(d)
session.commit()
if __name__ == '__main__':
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
session = Session()
session.add(ServerModel(name='test.com'))
session.commit()
test()
I'm trying to add a new domain_to_server mapping via the server/domain association proxy. If I don't access the association proxy first, ie remove the print statement in test(), then I can add the domain without needing the session.no_autoflush block. But with the print statement in there, it will fail without the session.no_autoflush block with an IntegrityError, saying that server_id cannot be null in the domains to servers table.
I'm trying to figure out why the no_autoflush block is needed here. I don't see any mention of it in the association_proxy docs. Is this simply the way it is, and all inserts into an association_proxy should to happen in a no_autoflush bock in case it has been accessed prior to the insert?
I have proxy association between Content and ContentRevision. Usage of sqlalchemy.ext.associationproxy.association_proxy produce an EXISTS condition:
from sqlalchemy import create_engine, Column, Integer, Text, ForeignKey, inspect, String, and_
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, foreign, remote
DeclarativeBase = declarative_base()
class ContentRevision(DeclarativeBase):
__tablename__ = 'content_revision'
revision_id = Column(Integer, primary_key=True)
content_id = Column(Integer, ForeignKey('content.id'))
description = Column(Text())
title = Column(String(32))
class Content(DeclarativeBase):
__tablename__ = 'content'
id = Column(Integer, primary_key=True)
revisions = relationship("ContentRevision",
foreign_keys=[ContentRevision.content_id])
revision = relationship(
"ContentRevision",
uselist=False,
primaryjoin=lambda: and_(
remote(Content.id) == foreign(ContentRevision.content_id),
ContentRevision.revision_id == ContentRevision.revision_id == session.query(ContentRevision.revision_id)
.filter(ContentRevision.content_id == Content.id)
.order_by(ContentRevision.revision_id.desc())
.limit(1)
.correlate(Content)
),
)
title = association_proxy('revision', 'title')
description = association_proxy('revision', 'description')
# Prepare database and session
engine = create_engine('sqlite://', echo=False)
DeclarativeBase.metadata.create_all(engine)
session_maker = sessionmaker(engine)
session = session_maker()
#
c1 = Content()
c1.revisions.append(ContentRevision(title='rev', description='rev1'))
session.add(c1)
session.flush()
c1.revisions.append(ContentRevision(title='rev', description='rev2'))
assert [('rev', 'rev1'), ('rev', 'rev2')] == session.query(ContentRevision.title, ContentRevision.description).all()
print(str(session.query(Content).filter(Content.title == 'foo')))
"""
SELECT content.id AS content_id
FROM content
WHERE EXISTS (SELECT 1
FROM content_revision
WHERE content.id = content_revision.content_id AND content_revision.revision_id = (SELECT content_revision.revision_id AS content_revision_revision_id
FROM content_revision
WHERE content_revision.content_id = content.id ORDER BY content_revision.revision_id DESC
LIMIT :param_1) AND content_revision.title = :title_1)
"""
How to make a query on associated column Content.title using the join declared in primaryjoin relationship of Content.revision ?