We Keep Coding

How to establish a self-referencing many-to-many relationship [duplicate]

I've found examples for a self-referencing many-to-many relationship with an association table. How can I achieve the same using an association object?
The code below is based on: How can I achieve a self-referencing many-to-many relationship on the SQLAlchemy ORM back referencing to the same attribute?
from sqlalchemy import Table, Column, Integer, ForeignKey
from db.common import Base
from sqlalchemy.orm import relationship
M2M = Table('m2m',
Base.metadata,
Column('entity_parent_id',
Integer,
ForeignKey('entity.id'),
primary_key=True),
Column('entity_child_id',
Integer,
ForeignKey('entity.id'),
primary_key=True),
)
class Entity(Base):
__tablename__ = 'entity'
id = Column(Integer, primary_key=True)
entity_childs = relationship("Entity",
secondary=M2M,
primaryjoin="Enity.id==m2m.c.entity_parent_id",
secondaryjoin="Enity.id==m2m.c.entity_child_id",
)
entity_parents = relationship("Entity",
secondary=M2M,
primaryjoin="Enity.id==m2m.c.entity_child_id",
secondaryjoin="Enity.id==m2m.c.entity_parent_id",
)

The following approach uses an association object instead of an association table to get a self-referencing many-to-many relationship:
from sqlalchemy import Column, Integer, ForeignKey, create_engine, String
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class EntityAssociation(Base):
__tablename__ = 'entity_association'
entity_parent_id = Column(Integer, ForeignKey('entity.id'), primary_key=True)
entity_child_id = Column(Integer, ForeignKey('entity.id'), primary_key=True)
class Entity(Base):
__tablename__ = 'entity'
id = Column(Integer, primary_key=True)
name = Column(String)
entity_childs = relationship('Entity',
secondary='entity_association',
primaryjoin=id==EntityAssociation.entity_parent_id,
secondaryjoin=id==EntityAssociation.entity_child_id,
backref='childs')
entity_parents = relationship('Entity',
secondary='entity_association',
primaryjoin=id==EntityAssociation.entity_child_id,
secondaryjoin=id==EntityAssociation.entity_parent_id,
backref='parents')
def __repr__(self):
return f'<Entity(name={self.name})>'
if __name__ == '__main__':
engine = create_engine('sqlite://')
Base.metadata.create_all(engine)
Session = sessionmaker(engine)
db = Session()
parent1 = Entity(name='parent1')
parent2 = Entity(name='parent2')
child1 = Entity(name='child1')
child2 = Entity(name='child2')
parent1.entity_childs = [child1, child2]
parent2.entity_childs = [child2]
db.add(parent1)
db.add(parent2)
db.add(child1)
db.add(child2)
db.commit()
entities = db.query(Entity).all()
for entity in entities:
print(entity)
print(' Parent: ', entity.entity_parents)
print(' Childs: ', entity.entity_childs)
print()
This will have the following result:
<Entity(name=parent1)>
Parent: []
Childs: [<Entity(name=child1)>, <Entity(name=child2)>]
<Entity(name=child1)>
Parent: [<Entity(name=parent1)>]
Childs: []
<Entity(name=child2)>
Parent: [<Entity(name=parent1)>, <Entity(name=parent2)>]
Childs: []
<Entity(name=parent2)>
Parent: []
Childs: [<Entity(name=child2)>]

Defining relationship on association table with filtering

This example develops on the Parent/Child association table on the SQLAlchemy docs (https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object)
How can I define the 'son' or 'daughter' attribute on the Parent class to filter the association table to the appropriate child type table?
I understand how the 'children' relationship works, but I would like to add extra conditionals to the relationship. I suspect this involves the use of secondary in the definition of relationship, but this is at the edge of my understanding of relationships in SQLAlchemy.
from sqlalchemy import Table, Column, Integer, ForeignKey, String
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Association(Base):
__tablename__ = 'association'
left_id = Column(Integer, ForeignKey('left.id'), primary_key=True)
right_id = Column(Integer, ForeignKey('right.id'), primary_key=True)
child = relationship("Child", back_populates="parents")
parent = relationship("Parent", back_populates="children")
child_type = Column(Integer, ForeignKey('child_type'))
class Parent(Base):
__tablename__ = 'left'
id = Column(Integer, primary_key=True)
children = relationship("Association", back_populates="parent")
# I would also like to define 'sons' and 'daughters' here
class Child(Base):
__tablename__ = 'right'
id = Column(Integer, primary_key=True)
parents = relationship("Association", back_populates="child")
class ChildType(Base):
__tablename__ = 'child_type'
id = Column(Integer, primary_key=True)
name = Column(String(50))
son_type = ChildType(name='son')
daughter_type = ChildType(name='daughter')
dad = Parent()
son = Child()
dad_son = Association(child_type=son_type)
dad_son.child = son
dad.children.append(dad_son)
daughter = Child()
dad_daughter = Association(child_type=daughter_type)
dad_daughter.child = daughter
dad.children.append(dad_daughter)

Filtering the association table in the secondary to only include rows of the type that you are interested in for each relationship seems to work:
class Parent(Base):
__tablename__ = 'left'
id = Column(Integer, primary_key=True)
children = relationship("Association", back_populates="parent")
# I would also like to define 'sons' and 'daughters' here
sons = relationship(
'Child',
secondary="join(Association, ChildType, "
"and_(Association.child_type_id==ChildType.id, "
"ChildType.name=='son'))",
viewonly=True
)
daughters = relationship(
'Child',
secondary="join(Association, ChildType, "
"and_(Association.child_type_id==ChildType.id, "
"ChildType.name=='daughter'))",
viewonly=True
)
... but adding objects via those relationships won't work as they don't know how to construct the association object with the child_type, hence the viewonly. Also, I changed Association.child_type to child_type_id = Column(Integer, ForeignKey('child_type.id')) which is why that's different in the example.
The Child/ChildType thing stands out as a possible application of the 'Mapping Class Inheritance Hierarchies' pattern, if you aren't locked into this schema, that might be something to explore.

Automating sqlalchemy declarative model creation

Let’s say we have several sqlalchemy models for catalogues:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer
from sqlalchemy.orm import relationship
Base = declarative_base()
class Plane(Base):
__tablename__ = 'Plane'
plane_id = Column(Integer, primary_key=True)
class Car(Base):
__tablename__ = 'Car'
car_id = Column(Integer, primary_key=True)
Now for import/export purposes we want to relate these to external ids. So for Plane we would write:
class PlaneID(Base):
issuer = Column(String(32), primary_key=True)
external_id = Column(String(16), primary_key=True)
plane_id = Column(Integer, ForeignKey(Plane.plane_id))
plane = relationship(Plane, backref='external_ids')
A CarID model would be defined in exactly the same way.
What are possibilities to automate this process?
Maybe we could use a mixin, factory, decorator or meta class. How would we generate the dynamically named Columns then? It would be good to be able to add more Columns to the generated models as needed. For example:
class CarID(ExternalID):
valid_from = Column(Date)

You can subclass DeclarativeMeta - the metaclass used in declarative_base function:
from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.orm import relationship
class ExternalObject(DeclarativeMeta):
def __new__(mcs, name, bases, attributes):
if 'issuer' not in attributes:
attributes['issuer'] = Column(String(32), primary_key=True)
if 'external_id' not in attributes:
attributes['external_id'] = Column(String(16), primary_key=True)
if name[-2:] == 'ID':
ext_cls_name = name[:-2]
attr_rel = ext_cls_name.lower()
attr_id = '%s_id' % attr_rel
if attr_rel in attributes or attr_id in attributes:
# Some code here in case 'car' or 'car_id' attribute is defined in new class
pass
attributes[attr_id] = Column(Integer, ForeignKey('%s.%s' % (ext_cls_name, attr_id)))
attributes[attr_rel] = relationship(ext_cls_name, backref='external_ids')
new_cls = super().__new__(mcs, name, bases, attributes)
return new_cls
ExternalID = declarative_base(metaclass=ExternalObject)
After that you can create subclass from ExternalID and add another attributes like you did for CarID.

When do I need to use sqlalchemy back_populates?

When I try SQLAlchemy Relation Example following this guide: Basic Relationship Patterns
I have this code
#!/usr/bin/env python
# encoding: utf-8
from sqlalchemy import create_engine
from sqlalchemy import Table, Column, Integer, ForeignKey
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=engine)
session = Session()
Base = declarative_base(bind=engine)
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
children = relationship("Child")
class Child(Base):
__tablename__ = 'child'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('parent.id'))
parent = relationship("Parent")
Base.metadata.create_all()
p = Parent()
session.add(p)
session.commit()
c = Child(parent_id=p.id)
session.add(c)
session.commit()
print "children: {}".format(p.children[0].id)
print "parent: {}".format(c.parent.id)
It works well, but in the guide, it says the model should be:
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
**children = relationship("Child", back_populates="parent")**
class Child(Base):
__tablename__ = 'child'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('parent.id'))
**parent = relationship("Parent", back_populates="children")**
Why don't I need back_populates or backref in my example? When should I use one or the other?

If you use backref you don't need to declare the relationship on the second table.
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
children = relationship("Child", backref="parent")
class Child(Base):
__tablename__ = 'child'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('parent.id'))
If you're not using backref, and defining the relationship's separately, then if you don't use back_populates, sqlalchemy won't know to connect the relationships, so that modifying one also modifies the other.
So, in your example, where you've defined the relationship's separately, but didn't provide a back_populates argument, modifying one field wouldn't automatically update the other in your transaction.
>>> parent = Parent()
>>> child = Child()
>>> child.parent = parent
>>> print(parent.children)
[]
See how it didn't automatically fill out the children field?
Now, if you supply a back_populates argument, sqlalchemy will connect the fields.
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
children = relationship("Child", back_populates="parent")
class Child(Base):
__tablename__ = 'child'
id = Column(Integer, primary_key=True)
parent_id = Column(Integer, ForeignKey('parent.id'))
parent = relationship("Parent", back_populates="children")
So now we get
>>> parent = Parent()
>>> child = Child()
>>> child.parent = parent
>>> print(parent.children)
[Child(...)]
Sqlalchemy knows these two fields are related now, and will update each as the other is updated. It's worth noting that using backref will do this, too. Using back_populates is nice if you want to define the relationships on every class, so it's easy to see all the fields just be glancing at the model class, instead of having to look at other classes that define fields via backref.

Sqlalchemy: Querying on m2m relations for polymorphic classes

I've got two classes that are connected through a many-to-many relationship: Parent and Tag.
Base = declarative_base()
association_table = Table('associations', Base.metadata,
Column('parent_id', Integer, ForeignKey('parent.id')),
Column('tag_id', Integer, ForeignKey('tag.id')),
)
class Tag(Base):
__tablename__ = 'tags'
id = Column(Integer, Sequence('tag_id_seq'), primary_key=True)
name = Column(String)
class Parent(Base):
__tablename__ = 'parents'
id = Column(Integer, Sequence('parent_id_seq'), primary_key=True)
tags = relationship('Tag', secondary=association_table, backref='parents')
If I want to query for all the Tag objects that have one or more relationships to a Parent, I'd do:
session.query(Tag).filter(Tag.parents.any()).all()
However, this Parent class is parent to child classes, Alice and Bob:
class Alice(Parent):
__tablename__ = 'alices'
__mapper_args__ = {'polymorphic_identity': 'alice'}
alice_id = Column(Integer, ForeignKey('parents.id'), primary_key=True)
class Bob(Parent):
__tablename__ = 'bobs'
__mapper_args__ = {'polymorphic_identity': 'bob'}
bob_id = Column(Integer, ForeignKey('parents.id'), primary_key=True)
Now I'd like to be able to retrieve all the Tag objects that have one or more relations to an Alice object. The previous query, session.query(Tag).filter(Tag.parents.any()).all(), would not do as it doesn't discriminate between Alice or Bob objects - it doesn't even know about their existence.
I've messed around with Query's for a while with no success so I assume that if it can be done, it must have something to do with some extra lines of code in the Table classes like those shown above. While the documentation holds info about polymorphic classes and many-to-many relations and Mike Bayer himself offered someone this answer to a seemingly related question which looks interesting but which I'm far from understanding, I'm kind of stuck.
The code samples may disgust the Python interpreter, but does hopefully get my point across. Candy for those who can help me on my way!

While writing a little MWE I actually found a solution, which is actually almost the same as what I had tried already. budder gave me new hope for the approach though, thanks :)
from sqlalchemy import create_engine, ForeignKey, Column, String, Integer, Sequence, Table
from sqlalchemy.orm import sessionmaker, relationship, backref
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
association_table = Table('associations', Base.metadata,
Column('parent_id', Integer, ForeignKey('parents.id')),
Column('tag_id', Integer, ForeignKey('tags.id')),
)
class Tag(Base):
__tablename__ = 'tags'
id = Column(Integer, Sequence('tag_id_seq'), primary_key=True)
name = Column(String)
class Parent(Base):
__tablename__ = 'parents'
id = Column(Integer, Sequence('parent_id_seq'), primary_key=True)
tags = relationship('Tag', secondary=association_table, backref='parents')
class Alice(Parent):
__tablename__ = 'alices'
__mapper_args__ = {'polymorphic_identity': 'alice'}
alice_id = Column(Integer, ForeignKey('parents.id'), primary_key=True)
class Bob(Parent):
__tablename__ = 'bobs'
__mapper_args__ = {'polymorphic_identity': 'bob'}
bob_id = Column(Integer, ForeignKey('parents.id'), primary_key=True)
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
tag_a = Tag(name='a')
tag_b = Tag(name='b')
tag_c = Tag(name='c')
session.add(tag_a)
session.add(tag_b)
session.add(tag_c)
session.commit()
session.add(Alice(tags=[tag_a]))
session.add(Bob(tags=[tag_b]))
session.commit()
for tag in session.query(Tag).\
filter(Tag.parents.any(Parent.id == Alice.alice_id)).\
all():
print(tag.name)
If there's a good alternative approach, I'd still be interested. I can imagine sqlalchemy offering a more direct and elegant approach so that one could do, for example:
for tag in session.query(Tag).\
filter(Tag.alices.any()).\
all():
print(tag.name)

If you write your object classes a certain way you can use the blunt force method...Just search for them...Kind of like a crude query method:
all_tag_objects = session.query(Tag).all() ## All tag objects in your database
tags = []
for tag in all_tag_objects:
for parent in tag.parents:
if parent.alices != []: ## If there are alice objects in the tag parents alice reltionship instance variable...Then we append the tag because it meets our criteria.
flagged_tags.append(tag)
Sounds like you found a better way though, but I guess the ultimate test would be to actually do a speed test.