We Keep Coding

Efficiently copy data between databases with sqlalchemy

I'm trying to mirror a postgresql + PostGIS database that I defined with sqlalchemy to a sqlite (spatialite) file database. The session.merge() method appears to work for adding the instances queried from the first session to the other session, but it does not scale for nearly a million rows. See the example below that copies data from an in-memory sqlite database to another memory database for the sake of easy reproducibility. I'm looking for an approach (potentially completely different from what I'm doing now) to efficiently move all the data from one database to another.
from sqlalchemy import create_engine
from sqlalchemy import Table, Column, Integer, ForeignKey, String
from sqlalchemy.orm import declarative_base, sessionmaker
from sqlalchemy.orm import relationship, joinedload
engine_0 = create_engine('sqlite:///:memory:', echo=True)
engine_1 = create_engine('sqlite:///:memory:', echo=True)
Base = declarative_base()
Session0 = sessionmaker(bind=engine_0)
Session1 = sessionmaker(bind=engine_1)
# Define ORM models
association_table = Table('association', Base.metadata,
Column('parent_id', ForeignKey('parent.id'), primary_key=True),
Column('child_id', ForeignKey('child.id'), primary_key=True)
)
class Parent(Base):
__tablename__ = 'parent'
id = Column(Integer, primary_key=True)
name = Column(String)
children = relationship(
"Child",
secondary=association_table,
back_populates="parents")
class Child(Base):
__tablename__ = 'child'
id = Column(Integer, primary_key=True)
name = Column(String)
parents = relationship(
"Parent",
secondary=association_table,
back_populates="children")
# Create schema
Base.metadata.create_all(engine_0)
Base.metadata.create_all(engine_1)
# Create some example instances
# Children
bart = Child(name='Bart')
lisa = Child(name='Lisa')
maggie = Child(name='Maggie')
milhouse = Child(name='Milhouse')
# Parents
homer = Parent(name='Homer',
children=[bart, lisa, maggie])
marge = Parent(name='Marge',
children=[bart, lisa, maggie])
flanders = Parent(name='Ned')
kirk = Parent(name='Kirk', children=[milhouse])
# Insert data into first database
session_0 = Session0()
session_0.add_all([homer, marge, flanders, kirk])
session_0.commit()
# Query the data and insert it into the second database
all_obj = session_0.query(Parent).options(joinedload('*')).all()
session_0.expunge_all()
session_1 = Session1()
for obj in all_obj:
session_1.merge(obj)
session_1.commit()
# MAke sure that 4 instance of child are present in the second database
print(session_1.query(Child).all())
One alternative approach I have tried (unsuccessfully) is to make the parent objects transient using the sqlalchemy.orm.make_transient() function and use session.add_all() instead of session.merge() to insert the objects into the second session. However, this does not propagate to the relationships and only Parent objects are made transient.

How do I change the schema for both a table and a foreign key?

I have the following simplified database access layer and two tables:
class DataAccessLayer():
def __init__(self):
conn_string = "mysql+mysqlconnector://root:root#localhost/"
self.engine = create_engine(conn_string)
Base.metadata.create_all(self.engine)
Session = sessionmaker()
Session.configure(bind=self.engine)
self.session = Session()
class MatchesATP(Base):
__tablename__ = "matches_atp"
__table_args__ = {"schema": "belgarath", "extend_existing": True}
ID_M = Column(Integer, primary_key=True)
ID_T_M = Column(Integer, ForeignKey("oncourt.tours_atp.ID_T"))
class TournamentsATP(Base):
__tablename__ = "tours_atp"
__table_args__ = {"schema": "oncourt", "extend_existing": True}
ID_T = Column(Integer, primary_key=True)
NAME_T = Column(String(255))
I want to be able to switch the schema names for the two tables to test databases as follows:
belgarath to belgarath_test
oncourt to oncourt_test
I've tried adding:
self.session.connection(execution_options={"schema_translate_map": {"belgarath": belgarath, "oncourt": oncourt}})
To the bottom of DataAccessLayer and then initialising the class with two variables as follows:
def __init__(self, belgarath, oncourt):
However, when I build the following query:
dal = DataAccessLayer("belgarath_test", "oncourt_test")
query = dal.session.query(MatchesATP)
print(query)
I get the following SQL:
SELECT belgarath.matches_atp.`ID_M` AS `belgarath_matches_atp_ID_M`, belgarath.matches_atp.`ID_T_M` AS `belgarath_matches_atp_ID_T_M`
FROM belgarath.matches_atp
This is still referencing the belgarath table.
I also can't figure out a way of changing the schema of the foreign key of oncourt.tours_atp.ID_T at the same time as the tables.
Are there individual solutions or a combined solution to my issues?

You might wanna decorate your subclassed Base declarative model with the #declared_attr decorator.
Try this--
In a base class for your models, say __init__.py...
from sqlalchemy.ext.declarative import declarative_base, declared_attr
SCHEMA_MAIN = 'belgarath' # figure out how you want to retrieve this
SCHEMA_TEST = 'belgarath_test'
class _Base(object):
#declared_attr
def __table_args__(cls):
return {'schema': SCHEMA_MAIN}
...
Base = declarative_base(cls=_Base)
Base.metadata.schema = SCHEMA_MAIN
Now that you have a Base that subclasses _Base with the main schema already defined, all your other models will subclass Base and do the following:
from . import Base, declared_attr, SCHEMA_TEST
class TestModel(Base):
#declared_attr
def __table_args__(cls):
return {'schema': SCHEMA_TEST}
Changing a schema for a foreign key could look like this:
class TournamentsATP(Base):
__tablename__ = "tours_atp"
__table_args__ = {"schema": "oncourt", "extend_existing": True}
ID_T = Column(Integer, primary_key=True)
NAME_T = Column(String(255))
match_id = Column('match_id', Integer, ForeignKey(f'{__table_args__.get("schema")}.matches_atp.id'))
Where match_id is a foreign key to matches_atp.id by using the __table_args[schema] element defined at the class level via #declared_attr.

It only took me 18 months to figure this out. Turns out I needed to add the schema_translate_map to an engine and then create the session with this engine:
from sqlalchemy import create_engine
engine = create_engine(conn_str, echo=False)
schema_engine = engine.execution_options(schema_translate_map={<old_schema_name>: <new_schema_name>})
NewSession = sessionmaker(bind=schema_engine)
session = NewSession()
All ready to roll...

Assuming your goal is to:
have dev/test/prod schemas on a single mysql host
allow your ORM classes to be flexible enough to be used in three different environments without modification
Then John has you most of the way to one type of solution. You could use #declared_attr to dynamically generate __table_args__ as he has suggested.
You could also consider using something like flask-sqlalchemy that comes with a built-in solution for this:
import os
DB_ENV = os.getenv(DB_ENV)
SQLALCHEMY_BINDS = {
'belgarath': 'mysql+mysqlconnector://root:root#localhost/belgarath{}'.format(DB_ENV),
'oncourt': 'mysql+mysqlconnector://root:root#localhost/oncourt{}'.format(DB_ENV)
}
class MatchesATP(Base):
__bind_key__ = "belgarath"
ID_M = Column(Integer, primary_key=True)
ID_T_M = Column(Integer, ForeignKey("oncourt.tours_atp.ID_T"))
class TournamentsATP(Base):
__bind_key__ = "oncourt"
ID_T = Column(Integer, primary_key=True)
NAME_T = Column(String(255))
Basically this method allows you to create a link to a schema (a bind key), and that schema is defined at run-time via the connection string. More information at the flask-sqlalchemy link.

How do I query resources in the nested collection in eve-sqlalchemy?

I am using Eve-SQLAlchemy==0.5.0
I would like to perform a nested query using Postman on my users such that I find all users that are within a specified organization.
Using SQL I would write my query such that:
select * from app_user
left join user_organization on user_organization.user_id = app_user.id
left join organization on organization.id = user_organization.organization_id
where organization.id = 2
I have a user model, an organization model, and a relational model linking the two user_organization.
from sqlalchemy import Column, DateTime, func, String, Integer
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class BaseModel(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
__abstract__ = True
_created = Column(DateTime, default=func.now())
_updated = Column(DateTime, default=func.now(), onupdate=func.now())
_etag = Column(String(40))
class User(BaseModel):
__tablename__ = 'app_user'
organizations = relationship("Organization", secondary=UserOrganization.__tablename__)
class Organization(BaseModel):
__tablename__ = 'organization'
name = Column(String)
class UserOrganization(BaseModel):
__tablename__ = 'user_organization'
user_id = Column(Integer,
ForeignKey('app_user.id', ondelete='CASCADE'))
organization_id = Column(Integer,
ForeignKey('organization.id', ondelete='CASCADE'))
In my settings.py I have the resources registered:
# Resource Registration
DOMAIN = DomainConfig({
'organization': ResourceConfig(Organization),
'user': ResourceConfig(User)
}).render()
I have a series of postman collections setup, and using a GET request I can easily query any attribute... GET localhost:5000/user?where={"id":1}
I have tried (amongst many other things):
GET user?where={"organizations": {"organization_id" :2 }}
GET user?where={"organizations": 2}

It seems it's not possible at the moment due to a bug. I will try to fix it within the next week.
The code in https://github.com/pyeve/eve-sqlalchemy/blob/master/eve_sqlalchemy/parser.py#L73 is causing a GET ?where={"organizations": 2} to result in a SQL expression like user_id = 42 AND organization_id = 42 is generated. Which rarely makes any sense.

SQLAlchemy many-to-many without foreign key

Could some one help me figure out how should i write primaryjoin/secondaryjoin
on secondary table that lacking one ForeignKey definition. I can't modify database
itself since it's used by different application.
from sqlalchemy import schema, types, func, orm
from sqlalchemy.dialects.postgresql import ARRAY
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class A(Base):
__tablename__ = 'atab'
id = schema.Column(types.SmallInteger, primary_key=True)
class B(Base):
__tablename__ = 'btab'
id = schema.Column(types.SmallInteger, primary_key=True)
a = orm.relationship(
'A', secondary='abtab', backref=orm.backref('b')
)
class AB(Base):
__tablename__ = 'abtab'
id = schema.Column(types.SmallInteger, primary_key=True)
a_id = schema.Column(types.SmallInteger, schema.ForeignKey('atab.id'))
b_id = schema.Column(types.SmallInteger)
I've tried specifing foreign on join condition:
a = orm.relationship(
'A', secondary='abtab', backref=orm.backref('b'),
primaryjoin=(id==orm.foreign(AB.b_id))
)
But received following error:
ArgumentError: Could not locate any simple equality expressions involving locally mapped foreign key columns for primary join condition '"atab".id = "abtab"."a_id"' on relationship Category.projects. Ensure that referencing columns are associated with a ForeignKey or ForeignKeyConstraint, or are annotated in the join condition with the foreign() annotation. To allow comparison operators other than '==', the relationship can be marked as viewonly=True.

You can add foreign_keys to your relationship configuration. They mention this in a mailing list post:
from sqlalchemy import create_engine
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
logon = Column(String(10), primary_key=True)
group_id = Column(Integer)
class Group(Base):
__tablename__ = 'groups'
group_id = Column(Integer, primary_key=True)
users = relationship('User', backref='group',
primaryjoin='User.group_id==Group.group_id',
foreign_keys='User.group_id')
engine = create_engine('sqlite:///:memory:', echo=True)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
u1 = User(logon='foo')
u2 = User(logon='bar')
g = Group()
g.users = [u1, u2]
session.add(g)
session.commit()
g = session.query(Group).first()
print([user.logon for user in g.users])
output:
['foo', 'bar']

SQLAlchemy declarative property from join (single attribute, not whole object)

I wish to create a mapped attribute of an object which is populated from another table.
Using the SQLAlchemy documentation example, I wish to make a user_name field exist on the Address class such that it can be both easily queried and easily accessed (without a second round trip to the database)
For example, I wish to be able to query and filter by user_name Address.query.filter(Address.user_name == 'wcdolphin').first()
And also access the user_name attribute of all Address objects, without performance penalty, and have it properly persist writes as would be expected of an attribute in the __tablename__
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(50))
addresses = relation("Address", backref="user")
class Address(Base):
__tablename__ = 'addresses'
id = Column(Integer, primary_key=True)
email = Column(String(50))
user_name = Column(Integer, ForeignKey('users.name'))#This line is wrong
How do I do this?
I found the documentation relatively difficult to understand, as it did not seem to conform to most examples, especially the Flask-SQLAlchemy examples.

You can do this with a join on the query object, no need to specify this attribute directly. So your model would look like:
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import sessionmaker, relation
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
engine = create_engine('sqlite:///')
Session = sessionmaker(bind=engine)
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(50))
addresses = relation("Address", backref="user")
class Address(Base):
__tablename__ = 'addresses'
id = Column(Integer, primary_key=True)
email = Column(String(50))
user_id = Column(Integer, ForeignKey("users.id"))
Base.metadata.create_all(engine)
A query after addresses with filtering the username looks like:
>>> session = Session()
>>> session.add(Address(user=User(name='test')))
>>> session.query(Address).join(User).filter(User.name == 'test').first()
<__main__.Address object at 0x02DB3730>
Edit: As you can directly access the user from an address object, there is no need for directly referencing an attribute to the Address class:
>>> a = session.query(Address).join(User).filter(User.name == 'test').first()
>>> a.user.name
'test'

If you truly want Address to have a SQL enabled version of "User.name" without the need to join explicitly, you need to use a correlated subquery. This will work in all cases but tends to be inefficient on the database side (particularly with MySQL), so there is possibly a performance penalty on the SQL side versus using a regular JOIN. Running some EXPLAIN tests may help to analyze how much of an effect there may be.
Another example of a correlated column_property() is at http://docs.sqlalchemy.org/en/latest/orm/mapped_sql_expr.html#using-column-property.
For the "set" event, a correlated subquery represents a read-only attribute, but an event can be used to intercept changes and apply them to the parent User row. Two approaches to this are presented below, one using regular identity map mechanics, which will incur a load of the User row if not already present, the other which emits a direct UPDATE to the row:
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.ext.declarative import declarative_base
Base= declarative_base()
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(50))
addresses = relation("Address", backref="user")
class Address(Base):
__tablename__ = 'addresses'
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey('users.id'))
email = Column(String(50))
Address.user_name = column_property(select([User.name]).where(User.id==Address.id))
from sqlalchemy import event
#event.listens_for(Address.user_name, "set")
def _set_address_user_name(target, value, oldvalue, initiator):
# use ORM identity map + flush
target.user.name = value
# use direct UPDATE
#object_session(target).query(User).with_parent(target).update({'name':value})
e = create_engine("sqlite://", echo=True)
Base.metadata.create_all(e)
s = Session(e)
s.add_all([
User(name='u1', addresses=[Address(email='e1'), Address(email='e2')])
])
s.commit()
a1 = s.query(Address).filter(Address.user_name=="u1").first()
assert a1.user_name == "u1"
a1.user_name = 'u2'
s.commit()
a1 = s.query(Address).filter(Address.user_name=="u2").first()
assert a1.user_name == "u2"