We Keep Coding

SQLAlchemy 1.4 warnings on overlapping relationships with a many-to-many relationship with association table

I have a model in SQLAlchemy which defines a many-to-many relationship using an association table (automap is being used here because I'm using an existing database):
from sqlalchemy import (Column, Table, MetaData, Integer, Text, LargeBinary,
ForeignKey, Float, Boolean, Index)
from sqlalchemy.ext.automap import automap_base, AutomapBase
from sqlalchemy.orm import Session, deferred, relationship
Base: AutomapBase = automap_base()
class VariantAssociation(Base):
__tablename__ = "sample_variant_association"
vid = Column(Integer, ForeignKey("variants.variant_id"),
primary_key=True)
sid = Column(Integer, ForeignKey("samples.sample_id"),
primary_key=True)
vdepth = Column(Integer)
valt_depth = Column(Integer)
gt = Column(Text)
gt_type = Column(Integer)
fraction = Column(Float)
variant = relationship("Variant", back_populates="samples")
sample = relationship("Sample", back_populates="variants")
__table_args__ = (Index('ix_sample_variant_association_valt_depth',
"valt_depth"),
Index('ix_sample_variant_association_vdepth',
"vdepth"),
Index('ix_sample_variant_association_vid', 'vid'),
Index('ix_sample_variant_association_sid', 'sid'),
Index('ix_sample_variant_association_fraction',
'fraction')
)
class Variant(Base):
__tablename__ = "variants"
variant_id = Column(Integer, primary_key=True)
info = deferred(Column(LargeBinary))
samples = relationship("VariantAssociation",
back_populates="variant")
class Sample(Base):
__tablename__ = "samples"
sample_id = Column(Integer, primary_key=True, index=True)
name = Column(Text, index=True)
variants = relationship("VariantAssociation",
back_populates="sample")
class SampleGenotypeCount(Base):
__tablename__ = 'sample_genotype_counts'
sample_id = Column(Integer, primary_key=True)
num_hom_ref = Column(Integer)
num_het = Column(Integer)
num_hom_alt = Column(Integer)
num_unknown = Column(Integer)
class DataMigration(Base):
__tablename__ = "datamigration"
done = Column(Boolean, primary_key=True)
On querying, this eventually generates this warning:
Query:
query = session.query(Variant).join(
Variant.samples).join(Sample)
Warning:
/usr/local/lib/python3.9/site-packages/sqlalchemy/orm/relationships.py:3441: SAWarning:
relationship 'Variant.variantassociation_collection' will copy column variants.variant_id to
column sample_variant_association.vid, which conflicts with relationship(s): 'Variant.samples'
(copies variants.variant_id to sample_variant_association.vid). If this is not the intention,
consider if these relationships should be linked with back_populates, or if viewonly=True
should be applied to one or more if they are read-only. For the less common case that foreign
key constraints are partially overlapping, the orm.foreign() annotation can be used to isolate
the columns that should be written towards. The 'overlaps' parameter may be used to remove
this warning. (Background on this error at: http://sqlalche.me/e/14/qzyx)
I've been looking through SO and the SQLAlchemy documentation but I was unable to find what could cause this issue since (in my view) the back_populates parameters are in the right places.
Where would the error in the model be? SQLAlchemy 1.3.23 did not generate one, FTR.

In order to set your own relationship names, you need to prevent Automap from generating relationships by iteself. You can achieve this by setting 'generate_relationship' to a function that returns None.
def generate_relationships(base, direction, return_fn, attrname, local_cls, referred_cls, **kw):
return None
Base.prepare(generate_relationship=generate_relationships)

Additional primary key constraints on association object when specifying relationship in sqlalchemy

I have a many to many relationship that has a specific set of characteristics. I thought I could implement this in sqlalchemy with an association table as below:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import Column, Integer, ForeignKey, Unicode, Enum
import enum
Base = declarative_base()
class Person(Base):
__tablename__ = 'person'
id = Column(Integer, primary_key=True)
name = Column(Unicode)
worksAt = relationship('Address', secondary='parelationship')
manages = relationship('Address', secondary='parelationship')
resides = relationship('Address', secondary='parelationship')
## How do I specify the additional constraint of
## parelationship.relation = Relationships.resident?
class Address(Base):
__tablename__ = 'address'
id = Column(Integer, primary_key=True)
name = Column(Unicode)
class Relationships(enum.Enum):
resident = 1
worker = 2
manager = 3
class PersonAddressRelationship(Base):
__tablename__ = 'parelationship'
personId = Column(Integer, ForeignKey('person.id'), primary_key=True)
adressID = Column(Integer, ForeignKey('address.id'), primary_key=True)
relation = Column(Enum(Relationships), primary_key=True)
Is there a neat way of specifying the worksAt, manages, resides relationships (and equally worksHere, isManagedBy etc in the Address table)?

Either define the primaryjoin or the secondaryjoin with the additional predicate, or use a derived table as secondary.
Using a derived table:
worksAt = relationship(
'Address',
secondary=lambda:
PersonAddressRelationship.__table__.select().
where(PersonAddressRelationship.relation == Relationships.worker).
alias(),
viewonly=True)
Using primaryjoin:
manages = relationship(
'Address', secondary='parelationship',
primaryjoin=lambda:
and_(Person.id == PersonAddressRelationship.personId,
PersonAddressRelationship.relation == Relationships.manager),
viewonly=True)
Using secondaryjoin:
resides = relationship(
'Address', secondary='parelationship',
secondaryjoin=lambda:
and_(Address.id == PersonAddressRelationship.adressID,
PersonAddressRelationship.relation == Relationships.manager),
viewonly=True)
Note that in all the examples the expression is passed as a callable (a lambda), so that it can be lazily evaluated during mapper configuration.

Self referential relationship including a relationship attribute

Situation
I have the Self-Referential Many-to-Many Relationship (almost identical to the sqlalchemy manual entry of the same heading). This relationship is governed by the table entity_weights. This code works!
Question
How do I include an attribute in class Entity representing the table column entity_weights.weight. Lets say the attribute would be called Entity.child_weights. It is important that the rank of Entity.child_entities and Entity.child_weights are identical.
entity_weights = Table('entity_weights', Base.metadata,
Column('id',Integer, primary_key=True),
Column('parent_entity_id',Integer, ForeignKey('entity.id')),
Column('entity_id',Integer, ForeignKey('entity.id')),
Column('weight',Float))
class Entity(Base):
__tablename__ = 'entity'
id = Column(Integer, primary_key=True)
name = Column(String)
domicile_id = Column(Integer, ForeignKey('domicile.id'))
entity_type = Column('type',Enum('asset','institution','model'))
source_table_id = Column(Integer)
child_entities = relationship('Entity',
secondary=entity_weights,
primaryjoin=id==entity_weights.c.parent_entity_id,
secondaryjoin=id==entity_weights.c.entity_id,
backref='parent_entity_id'
)

The cleanest solution I've found for this scenario is to break up the child_entities relationship by adding entity_weights as a one-to-many relationship on Entity and use an association proxy to proxy the weight value as well as the remote side of the many-to-many relationship:
class EntityWeight(Base):
__tablename__ = "entity_weights"
id = Column(Integer, primary_key=True)
parent_entity_id = Column(Integer, ForeignKey('entity.id'))
entity_id = Column(Integer, ForeignKey('entity.id'))
weight = Column(Float)
entity = relationship("Entity", primaryjoin=lambda: EntityWeight.entity_id == Entity.id)
class Entity(Base):
...
_child_weights = relationship(EntityWeight, primaryjoin=id == EntityWeight.parent_entity_id)
child_weights = association_proxy("_child_weights", "weight")
child_entities = association_proxy("_child_weights", "entity")

SQLAlchemy: order by a relationship field in a relationship

In a Pyramid application I'm working on, I have the following scenario:
class Widget(Base):
__tablename__ = 'widgets'
id = Column(Integer, primary_key=True)
name = Column(String(50))
sidebar = Column(mysql.TINYINT(2))
def __init__(self, name, sidebar):
self.name = name
self.sidebar = sidebar
class Dashboard(Base):
__tablename__ = 'dashboard'
user_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
widget_id = Column(Integer, ForeignKey('widgets.id'), primary_key=True)
delta = Column(mysql.TINYINT)
widget = relationship('Widget')
def __init__(self, user_id, widget_id, delta):
self.user_id = user_id
self.widget_id = widget_id
self.delta = delta
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
login = Column(Unicode(255), unique=True)
password = Column(Unicode(60))
fullname = Column(Unicode(100))
dashboard = relationship('Dashboard', order_by='Dashboard.widget.sidebar, Dashboard.delta')
def __init__(self, login, password, fullname):
self.login = login
self.password = crypt.encode(password)
self.fullname = fullname
So, I want the User 'dashboard' relationship to have the dashboard records for the user but ordered by 'sidebar' (which is a relationship property of Dashboard). Currently I am getting this error:
sqlalchemy.exc.InvalidRequestError: Property 'widget' is not an instance of ColumnProperty (i.e. does not correspond directly to a Column).
Is this ordering possible in a relationship declaration?
Thanks!

With this, try to think what SQL SQLAlchemy should emit when it tries to load User.dashboard. Like SELECT * FROM dashboard JOIN widget ... ORDER BY widget.sidebar ? Or SELECT * FROM dashboard ORDER BY (SELECT sidebar FROM widget... ? ordering the results by a different table is too open-ended of a job for relationship() to decide on it's own. The way this can be done is by providing a column expression in terms of Dashboard that can provide this ordering, when the ORM emits a simple SELECT against dashboard's table, as well as when it refers to it in a not-so-simple SELECT where it might be joining across User, Dashboard tables at once (e.g. eager loading).
We provide custom SQL expressions, particularly those that involve other tables, using column_property(), or alternatively with deferred() when we don't want that expression to be loaded by default (as is likely the case here). Example:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Widget(Base):
__tablename__ = 'widgets'
id = Column(Integer, primary_key=True)
name = Column(String(50))
sidebar = Column(Integer)
class Dashboard(Base):
__tablename__ = 'dashboard'
user_id = Column(Integer, ForeignKey('users.id'), primary_key=True)
widget_id = Column(Integer, ForeignKey('widgets.id'), primary_key=True)
delta = Column(Integer)
widget = relationship('Widget')
widget_sidebar = deferred(select([Widget.sidebar]).where(Widget.id == widget_id))
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
login = Column(Unicode(255), unique=True)
dashboard = relationship('Dashboard', order_by='Dashboard.widget_sidebar, Dashboard.delta')
e = create_engine("sqlite://", echo=True)
Base.metadata.create_all(e)
s = Session(e)
w1, w2 = Widget(name='w1', sidebar=1), Widget(name='w2', sidebar=2)
s.add_all([
User(login='u1', dashboard=[
Dashboard(
delta=1, widget=w1
),
Dashboard(
delta=2, widget=w2
)
]),
])
s.commit()
print s.query(User).first().dashboard
the final SQL emitted by the load of ".dashboard" is:
SELECT dashboard.user_id AS dashboard_user_id, dashboard.widget_id AS dashboard_widget_id, dashboard.delta AS dashboard_delta
FROM dashboard
WHERE ? = dashboard.user_id ORDER BY (SELECT widgets.sidebar
FROM widgets
WHERE widgets.id = dashboard.widget_id), dashboard.delta
Keep in mind that MySQL does a terrible job optimizing for subqueries like the one above. If you need high performance here, you might consider copying the value of "sidebar" into "dashboard", even though that makes consistency more difficult to maintain.

How to build many-to-many relations using SQLAlchemy: a good example

I have read the SQLAlchemy documentation and tutorial about building many-to-many relation but I could not figure out how to do it properly when the association table contains more than the 2 foreign keys.
I have a table of items and every item has many details. Details can be the same on many items, so there is a many-to-many relation between items and details
I have the following:
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
My association table is (It's defined before the other 2 in the code):
class ItemDetail(Base):
__tablename__ = 'ItemDetail'
id = Column(Integer, primary_key=True)
itemId = Column(Integer, ForeignKey('Item.id'))
detailId = Column(Integer, ForeignKey('Detail.id'))
endDate = Column(Date)
In the documentation, it's said that I need to use the "association object". I could not figure out how to use it properly, since it's mixed declarative with mapper forms and the examples seem not to be complete. I added the line:
details = relation(ItemDetail)
as a member of Item class and the line:
itemDetail = relation('Detail')
as a member of the association table, as described in the documentation.
when I do item = session.query(Item).first(), the item.details is not a list of Detail objects, but a list of ItemDetail objects.
How can I get details properly in Item objects, i.e., item.details should be a list of Detail objects?

From the comments I see you've found the answer. But the SQLAlchemy documentation is quite overwhelming for a 'new user' and I was struggling with the same question. So for future reference:
ItemDetail = Table('ItemDetail',
Column('id', Integer, primary_key=True),
Column('itemId', Integer, ForeignKey('Item.id')),
Column('detailId', Integer, ForeignKey('Detail.id')),
Column('endDate', Date))
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
details = relationship('Detail', secondary=ItemDetail, backref='Item')
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
items = relationship('Item', secondary=ItemDetail, backref='Detail')

Like Miguel, I'm also using a Declarative approach for my junction table. However, I kept running into errors like
sqlalchemy.exc.ArgumentError: secondary argument <class 'main.ProjectUser'> passed to to relationship() User.projects must be a Table object or other FROM clause; can't send a mapped class directly as rows in 'secondary' are persisted independently of a class that is mapped to that same table.
With some fiddling, I was able to come up with the following. (Note my classes are different than OP's but the concept is the same.)
Example
Here's a full working example
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import declarative_base, relationship, Session
# Make the engine
engine = create_engine("sqlite+pysqlite:///:memory:", future=True, echo=False)
# Make the DeclarativeMeta
Base = declarative_base()
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
name = Column(String)
projects = relationship('Project', secondary='project_users', back_populates='users')
class Project(Base):
__tablename__ = "projects"
id = Column(Integer, primary_key=True)
name = Column(String)
users = relationship('User', secondary='project_users', back_populates='projects')
class ProjectUser(Base):
__tablename__ = "project_users"
id = Column(Integer, primary_key=True)
notes = Column(String, nullable=True)
user_id = Column(Integer, ForeignKey('users.id'))
project_id = Column(Integer, ForeignKey('projects.id'))
# Create the tables in the database
Base.metadata.create_all(engine)
# Test it
with Session(bind=engine) as session:
# add users
usr1 = User(name="bob")
session.add(usr1)
usr2 = User(name="alice")
session.add(usr2)
session.commit()
# add projects
prj1 = Project(name="Project 1")
session.add(prj1)
prj2 = Project(name="Project 2")
session.add(prj2)
session.commit()
# map users to projects
prj1.users = [usr1, usr2]
prj2.users = [usr2]
session.commit()
with Session(bind=engine) as session:
print(session.query(User).where(User.id == 1).one().projects)
print(session.query(Project).where(Project.id == 1).one().users)
Notes
reference the table name in the secondary argument like secondary='project_users' as opposed to secondary=ProjectUser
use back_populates instead of backref
I made a detailed writeup about this here.

Previous Answer worked for me, but I used a Class base approach for the table ItemDetail. This is the Sample code:
class ItemDetail(Base):
__tablename__ = 'ItemDetail'
id = Column(Integer, primary_key=True, index=True)
itemId = Column(Integer, ForeignKey('Item.id'))
detailId = Column(Integer, ForeignKey('Detail.id'))
endDate = Column(Date)
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
details = relationship('Detail', secondary=ItemDetail.__table__, backref='Item')
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
items = relationship('Item', secondary=ItemDetail.__table__, backref='Detail')