I'm trying to set up a secondary many-to-many relationship from one table to two others, via a third in the middle that links to all three. I have two files - one for ORM objects (model.py) and one for schema objects (schema.py) They look like this:
model.py
import schema
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import *
class AbstractBase(object):
def __repr__(self):
macros = ["%s=%s" % (key, getattr(self, key, None)) for key in self.__table__.columns.keys()]
rep = "<%s(%s)>" % (self.__class__.__name__, str.join(', ', macros))
return rep
Base = declarative_base(cls=AbstractBase)
class A(Base):
__table__ = schema.a_table
dees = relationship("D",
secondary=schema.b_table,
primaryjoin="A.a_id==b_table.c.a_id",
secondaryjoin="b_table.c.c_id==D.d_id")
cees = relationship("C",
secondary=schema.b_table,
primaryjoin="A.a_id==schema.b_table.c.a_id",
secondaryjoin="b_table.c.d_id==C.c_id",
backref="a_collection")
class C(Base):
__table__ = schema.c_table
class D(Base):
__table__ = schema.d_table
schema.py
from sqlalchemy import *
from sqlalchemy.dialects.mysql import *
metadata = MetaData()
a_table = Table(
'a',
metadata,
Column("a_id", INTEGER(), primary_key=True, nullable=False),
Column("date", DATETIME(timezone=True)),
)
b_table = Table(
'shipment_runs',
metadata,
Column("a_id", ForeignKey("a.a_id"), primary_key=True,),
Column("c_id", ForeignKey("c.c_id"), primary_key=True),
Column("d_id", ForeignKey("d.d_id")),
)
c_table = Table(
'c',
metadata,
Column('c_id', INTEGER(), primary_key=True, nullable=False),
Column('name', VARCHAR(64), unique=True),
)
d_table = Table(
'd',
metadata,
Column('d_id', INTEGER(), primary_key=True, nullable=False)
)
Unfortunately, instantiating this results in the following error:
sqlalchemy.exc.InvalidRequestError: When initializing mapper Mapper|A|a, expression 'A.a_id==b_table.c.a_id' failed to locate a name ("name 'b_table' is not defined"). If this is a class name, consider adding this relationship() to the class after both dependent classes have been defined.
Is there a way I can change my imports or make the mapper be aware of the objects in the schema module somehow?
Was able to get it by doing the following:
class B(Base):
__table__ = schema.b_table
class A(Base):
__table__ = schema.a_table
dees = relationship("D",
secondary=b.__table__,
primaryjoin="A.a_id==B.a_id",
secondaryjoin="B.c_id==D.d_id")
cees = relationship("C",
secondary=B.__table__,
primaryjoin="A.a_id==B.a_id",
secondaryjoin="B.d_id==C.c_id",
backref="a_collection")
All credit goes to this question:
SQLAlchemy Relationship Error: object has no attribute 'c'
In general, you can use the table name in a string, or drop the string and use your actual references.
primaryjoin="A.a_id==shipment_runs.c.a_id",
primaryjoin=schema.a_table.c.a_id==schema.b_table.c.a_id,
That being said, given that you have the ForeignKeys set up in your tables, SQLAlchemy is smart enough that you don't even need the joins for a simple relationship, just secondary.
c_list = relationship("C", secondary=schema.b_table, backref="a_list")
(I think the "C" and "D" are swapped in your example?)
Related
I am building a Python application that uses SQLAlchemy, and I am trying to implement a many-to-many relationship between a custom declarative base class and itself (self-referential). But I cannot get it to work. I am attaching the code down below, as well as the error traceback, in case anyone can help :) All the entities of the model already extended from this base class, and the application was working so far, in case that helps.
Thanks!!
Code (non-functional):
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from sqlalchemy import MetaData, Table
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy import String
permissions = Table(
'permissions', MetaData(),
Column('origin_id', String, ForeignKey('bases.id'), primary_key=True),
Column('target_id', String, ForeignKey('bases.id'), primary_key=True)
)
class Base:
__tablename__ = 'bases'
__table_args__ = {
'mysql_engine': 'InnoDB'
}
id = Column(String, primary_key=True)
targets = relationship(
'Base',
secondary='permissions',
primaryjoin='Base.id == permissions.c.origin_id',
secondaryjoin='Base.id == permissions.c.target_id',
backref='origins'
# Reference:
# https://docs.sqlalchemy.org/en/14/orm/join_conditions.html#self-referential-many-to-many
)
Base = declarative_base(cls=Base)
Traceback:
class ContactMethod(Base):
File "/usr/local/lib/python3.8/site-packages/SQLAlchemy-1.4.23-py3.8-linux-x86_64.egg/sqlalchemy/orm/decl_api.py", line 72, in __init__
_as_declarative(reg, cls, dict_)
File "/usr/local/lib/python3.8/site-packages/SQLAlchemy-1.4.23-py3.8-linux-x86_64.egg/sqlalchemy/orm/decl_base.py", line 126, in _as_declarative
return _MapperConfig.setup_mapping(registry, cls, dict_, None, {})
File "/usr/local/lib/python3.8/site-packages/SQLAlchemy-1.4.23-py3.8-linux-x86_64.egg/sqlalchemy/orm/decl_base.py", line 177, in setup_mapping
return cfg_cls(registry, cls_, dict_, table, mapper_kw)
File "/usr/local/lib/python3.8/site-packages/SQLAlchemy-1.4.23-py3.8-linux-x86_64.egg/sqlalchemy/orm/decl_base.py", line 299, in __init__
self._scan_attributes()
File "/usr/local/lib/python3.8/site-packages/SQLAlchemy-1.4.23-py3.8-linux-x86_64.egg/sqlalchemy/orm/decl_base.py", line 511, in _scan_attributes
raise exc.InvalidRequestError(
sqlalchemy.exc.InvalidRequestError: Mapper properties (i.e. deferred,column_property(), relationship(), etc.) must be declared as #declared_attr callables on declarative mixin classes. For dataclass field() objects, use a lambda:
See a working self-containing example:
## Imports
from sqlalchemy import Column, ForeignKey, Integer, String, Table, create_engine
from sqlalchemy.orm import Session, as_declarative, declared_attr, relationship
## Configuration
engine = create_engine("sqlite:///:memory:", echo=True)
#as_declarative()
class Base(object):
#declared_attr
def __tablename__(cls):
return cls.__name__.lower()
permissions = Table(
"permissions",
Base.metadata,
Column("origin_id", String, ForeignKey("model_base.id"), primary_key=True),
Column("target_id", String, ForeignKey("model_base.id"), primary_key=True),
)
## Model definitions
class ModelBase(Base):
__tablename__ = "model_base"
id = Column(Integer, primary_key=True)
type = Column(String, nullable=False)
__mapper_args__ = {
"polymorphic_on": type,
"polymorphic_identity": None,
}
targets = relationship(
"ModelBase",
secondary="permissions",
primaryjoin="model_base.c.id == permissions.c.origin_id",
secondaryjoin="model_base.c.id == permissions.c.target_id",
backref="origins",
# lazy="raise",
)
class ClassA(ModelBase):
__tablename__ = "class_a"
__mapper_args__ = {"polymorphic_identity": "class_a"}
id = Column(ForeignKey("model_base.id"), primary_key=True)
name = Column(String)
class ClassB(ModelBase):
__tablename__ = "class_b"
__mapper_args__ = {"polymorphic_identity": "class_b"}
id = Column(ForeignKey("model_base.id"), primary_key=True)
name = Column(String)
value = Column(Integer)
## Tests
def _main():
with Session(engine) as session:
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
print("=" * 80)
# data
a1, a2 = [ClassA(name="a1"), ClassA(name="a2")]
b1, b2 = [ClassB(name="b1"), ClassB(name="b2", value=3)]
session.add_all([a1, a2, b1, b2])
session.flush()
a1.targets.append(a2)
a1.targets.append(b1)
a1.targets.append(b2)
print(b2.targets)
print(b2.origins)
session.commit()
session.expunge_all()
print("=" * 80)
a1 = session.query(ClassA).first()
print(a1)
print(a1.origins)
print(a1.targets)
session.commit()
if __name__ == "__main__":
_main()
Here’s an example of what the error output recommends to remediate:
from sqlalchemy.ext.declarative import declared_attr
class Base:
...
#declared_attr
def targets(cls):
return relationship(
'Base',
secondary='permissions',
primaryjoin='Base.id == permissions.c.origin_id',
secondaryjoin='Base.id == permissions.c.target_id',
backref='origins'
)
...
Sidenote: you can use the as_declarative mixin on your base class as a shortcut.
References
Augmenting the base: https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/mixins.html#augmenting-the-base
declared_attr: https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/api.html#sqlalchemy.ext.declarative.declared_attr
as_declarative: https://docs.sqlalchemy.org/en/13/orm/extensions/declarative/api.html#sqlalchemy.ext.declarative.as_declarative
I have a table whose name must be created dynamically. Here is how the table is created:
def create_data_table(table_name):
meta = Base.metadata
my_table= Table(
table_name, meta,
Column('id', BigInteger, primary_key=True, nullable=False),
Column('file_name', String(250), nullable=False)
)
meta.create_all(engine, [my_table])
This works just fine. However, I also need to pass the table name as a variable when I insert rows, and that's where I am running into problems.
Here are the things I have tried:
def insert_data(table_name, data):
meta = Base.metadata
my_table = Table(
table_name, meta,
Column('id', BigInteger, primary_key=True),
Column('file_name', String(250))
)
conn = engine.connect()
conn.execute(my_table.insert(), data)
This results in an error, telling me that my_table is already defined in my metadata.
To avoid this, I have tried creating a class instead:
class my_table(Base):
__tablename__ = ''
id = Column('id', String(50), primary_key=True)
file_name = Column('file_name', String(250))
def __init__(self, table_name, id, file_name):
__tablename__ = table_name
self.id = id
self.file_name = file_name
Again, I was able to create the table without any problems, but SQLAlchemy did not allow me to pass the table name as a parameter when I tried to insert rows.
Ideally, I would like to use the first approach. Any help is appreciated!
The SQLAlchemy MetaData object has a mapping of all of the tables it holds through the MetaData.tables attribute. Which is:
A dictionary of Table objects keyed to their name or “table key”.
You do not want to recreate the Table either directly or through the declarative mapping, just access it by name from MetaData.tables. E.g.:
Base.metadata.tables[table_name]
Your insert_data() implementation can simply be:
def insert_data(table_name, data):
my_table = Base.metadata.tables[table_name]
conn = engine.connect()
conn.execute(my_table.insert(), data)
Could some one help me figure out how should i write primaryjoin/secondaryjoin
on secondary table that lacking one ForeignKey definition. I can't modify database
itself since it's used by different application.
from sqlalchemy import schema, types, func, orm
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class A(Base):
__tablename__ = 'atab'
id = schema.Column(types.SmallInteger, primary_key=True)
class B(Base):
__tablename__ = 'btab'
id = schema.Column(types.SmallInteger, primary_key=True)
a = orm.relationship(
'A', secondary='abtab', backref=orm.backref('b')
)
class AB(Base):
__tablename__ = 'abtab'
id = schema.Column(types.SmallInteger, primary_key=True)
a_id = schema.Column(types.SmallInteger, schema.ForeignKey('atab.id'))
b_id = schema.Column(types.SmallInteger)
I've tried specifing foreign on join condition:
a = orm.relationship(
'A', secondary='abtab', backref=orm.backref('b'),
primaryjoin=(id==orm.foreign(AB.b_id))
)
But received following error:
ArgumentError: Could not locate any simple equality expressions involving locally mapped foreign key columns for primary join condition '"atab".id = "abtab"."a_id"' on relationship Category.projects. Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or are annotated in the join condition with the foreign() annotation. To allow comparison operators other than '==', the relationship can be marked as viewonly=True.
You can add foreign_keys to your relationship configuration. They mention this in a mailing list post:
from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
logon = Column(String(10), primary_key=True)
group_id = Column(Integer)
class Group(Base):
__tablename__ = 'groups'
group_id = Column(Integer, primary_key=True)
users = relationship('User', backref='group',
primaryjoin='User.group_id==Group.group_id',
foreign_keys='User.group_id')
engine = create_engine('sqlite:///:memory:', echo=True)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
u1 = User(logon='foo')
u2 = User(logon='bar')
g = Group()
g.users = [u1, u2]
session.add(g)
session.commit()
g = session.query(Group).first()
print([user.logon for user in g.users])
output:
['foo', 'bar']
I have a question concerning the mapping of entities in SQLAlchemy.
I have a transient object, which already contains foreign keys to some persistent objects. I want that SQLAlchemy fetches the referenced objects and assigns them to their relationship-attributes. From the SQLAlchemy documentation, I thought that I have to use the merge-operation on the session to achieve this. But in my configuration, it doesn't work.
This is a minimum example demonstrating my problem:
# -*- coding: utf-8 -*-
from sqlalchemy import create_engine
from sqlalchemy.orm import mapper
from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey
from sqlalchemy.orm import relationship, sessionmaker
class User(object):
def __init__(self, id, name, fullname, password, best_friend_id=None):
self.id = id
self.name = name
self.fullname = fullname
self.password = password
self.best_friend_id = best_friend_id
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password)
class Dog(object):
def __init__(self, id, name):
self.id = id
self.name = name
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password)
engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=engine)
session = Session()
metadata = MetaData()
dogs_table = Table('dogs', metadata,
Column('id', Integer, primary_key=True),
Column('name', String),
)
users_table = Table('users', metadata,
Column('id', Integer, primary_key=True),
Column('name', String),
Column('fullname', String),
Column('password', String),
Column('best_friend_id', Integer, ForeignKey('dogs.id'))
)
metadata.create_all(engine)
mapper(User, users_table, properties={'best_friend': relationship(Dog, uselist=False)})
mapper(Dog, dogs_table)
dog = Dog(id=1, name='Hasso')
lordling = User(id=2, name='John', fullname='Miller', password='very_secret', best_friend_id=1)
session.add(dog)
session.commit()
merged_lordling = session.merge(lordling)
print str(merged_lordling.best_friend.name)
I expect that merged_lordling.best_friend contains the dog 'Hasso'. But it is still None.
I was bit by this same problem recently. Once you established a relationship, you should simply assign your Dog instance to User.best_friend directly, not explicitly using the foreign key. I don't know why exactly that happens, but while investigating a similar problem I realized that if you do that, SQLAlchemy doesn't populate the relationship property until you flushed all the related instances.
So, instead of:
dog = Dog(id=1, name='Hasso')
lordling = User(id=2, name='John', fullname='Miller', password='very_secret',
best_friend_id=1)
session.add(dog)
Simply do:
dog = Dog(id=1, name='Hasso')
lordling = User(id=2, name='John', fullname='Miller', password='very_secret',
best_friend=dog)
session.add(lordling)
Or even:
lordling = User(id=2, name='John', fullname='Miller', password='very_secret',
best_friend=Dog(id=1, name='Hasso'))
session.add(lordling)
As a general rule, avoid using the foreign key columns directly when you have a relationship established. Embrace the ORM, and only assign or query directly from foreign keys when you really have no other choice. I learned that the hard way.
I would like to have a 'relationship' in an inherited (mixin) class.
However, when I create the inherited object, the relationship object is None. I cannot append to it.
How do I resolve this?
Here is code based upon the documentation
from sqlalchemy import Column, Integer, String, DateTime, Boolean, BigInteger, Float
from sqlalchemy import ForeignKey
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Target(Base):
__tablename__ = "target"
id = Column(Integer, primary_key=True)
class RefTargetMixin(object):
#declared_attr
def target_id(cls):
return Column('target_id', ForeignKey('target.id'))
#declared_attr
def target(cls):
return relationship("Target",
primaryjoin="Target.id==%s.target_id" % cls.__name__
)
class Foo(RefTargetMixin, Base):
__tablename__ = 'foo'
id = Column(Integer, primary_key=True)
print repr(RefTargetMixin.target)
print repr(Foo.target)
print repr(Foo().target)
The output is:
<sqlalchemy.orm.properties.RelationshipProperty object at 0x24e7890>
<sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x24e7690>
None
In general, I should be able to append to the relationship object (target), but here I cannot because it is None. Why?
the reason the value is None is because you've defined this as a many-to-one relationship. Many-to-one, from parent-to-child, means there is a foreign key on the parent, which can only refer to one and only one child. If you'd like something of class RefTargetMixin to refer to a collection of items, then foreign keys must be on the remote side.
So then the goal here is to make any object that is a subclass of RefTargetMixin be a potential parent for a Target. This pattern is called the polymorphic association pattern. While it is common in many ORM toolkits to provide this by declaring a "polymorphic foreign key" on Target, this is not a good practice relationally, so the answer is to use multiple tables in some way. There are three scenarios for this provided in SQLAlchemy core in the examples/generic_association folder, including "single association table with discriminator", "table per association", and "table per related". Each pattern provides the identical declarative pattern for RefTargetMixin here but the structure of the tables changes.
For example, here is your model using "table per association", which in my view tends to scale the best provided you don't need to query multiple types of RefTargetMixin objects at once (note I literally used the example as is, just changed the names):
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy import create_engine, Integer, Column, \
String, ForeignKey, Table
from sqlalchemy.orm import Session, relationship
class Base(object):
"""Base class which provides automated table name
and surrogate primary key column.
"""
#declared_attr
def __tablename__(cls):
return cls.__name__.lower()
id = Column(Integer, primary_key=True)
Base = declarative_base(cls=Base)
class Target(Base):
pass
class RefTargetMixin(object):
#declared_attr
def targets(cls):
target_association = Table(
"%s_targets" % cls.__tablename__,
cls.metadata,
Column("target_id", ForeignKey("target.id"),
primary_key=True),
Column("%s_id" % cls.__tablename__,
ForeignKey("%s.id" % cls.__tablename__),
primary_key=True),
)
return relationship(Target, secondary=target_association)
class Customer(RefTargetMixin, Base):
name = Column(String)
class Supplier(RefTargetMixin, Base):
company_name = Column(String)
engine = create_engine('sqlite://', echo=True)
Base.metadata.create_all(engine)
session = Session(engine)
session.add_all([
Customer(
name='customer 1',
targets=[
Target(),
Target()
]
),
Supplier(
company_name="Ace Hammers",
targets=[
Target(),
]
),
])
session.commit()
for customer in session.query(Customer):
for target in customer.targets:
print target
This is the normal behaviour : Foo has one Target. When you create the Foo object, it has no Target yet, so the value of Foo().target is None.
If you want Foo to have multiple Targets, you should put a foo_id in Target, and not a target_id in Foo, and use a backref.
Also, in that case, it is not needed to specify the primary join.