I am using Flask for my python wsgi server, and sqlalchemy for all my database access.
I think I would like to use the Flask-Sqlalchemy extension in my application, but I do not want to use the declarative base class (db.Model), instead, I want to use the base from sqlalchemy.ext.declarative.
Does this defeat the entire purpose of using the extension?
My use case:
I would like the extension to help me manage sessions/engines a little better, but I would like to handle all models separately.
I actually wouldn't mind using the extension, but I want to write strict models. I am porting code from a non-flask application, and I will be pushing changes back to that project as I go. If flask-sqlalchemy allows me to cheat on Table metadata for instance, that is going to cause problems when the code is pushed back out. There are also portions of my code that do lots of type checking (polymorphic identities), and I also remember reading that type checking on Table is not recommended when using the extension.
You can have Flask-SQLAlchemy expose your own base Model instead of it's built-in one. Just subclass SQLAlchemy and override make_declarative_base.
from flask.ext.sqlalchemy import SQLAlchemy
class CustomAlchemy(SQLAlchemy):
def make_declarative_base(self):
base = declarative_base(...)
...
return base
db = CustomAlchemy()
I'm actually using sqlalchemy in flask without using declarative base and I don't have any problems. You can always do that if you want to, there is no obligation to use object relational mapper, ORM is just one part of sqlalchemy. You can always just stay with alchemy sql expression language, define your tables in model objects, and define some methods there that will use expression language. I have a code like this (Model is the object i defined earlier), connect is a decorator which connects to db, it works fine for me.
def connect(func):
eng = create_engine(app.config["DATABASE"])
#wraps(func)
def wrapped(*args,**kwargs):
with closing(eng.connect()) as con:
result = con.execute(func(*args,**kwargs))
return result
return wrapped
class User_(Model):
def __init__(self):
Model.__init__(self)
self.metadata = MetaData()
self.structure = Table("users", self.metadata,
Column("id",Integer,primary_key=True),
Column("username",VARCHAR(64)),
Column("password",TEXT),
Column("email",VARCHAR(100)),
Column("about_me",TEXT),
Column("deadline",DATETIME),
Column("points",INTEGER)),
Column("date_created",DATETIME))
#connect
def get_hashed_pass(self,username):
""" """
t = self.structure
s = select([t.c.password]).where(t.c.username == str(username))
return s
#other methods follow
Flask's documentation concerning alchemy explicitly says that it is completely okay to do that:
If you just want to use the database system (and SQL) abstraction layer you basically only need the engine
P.S. Oh, and one more thing, they say in the docs that if you want to get started quickly you're better off using extension, but I'm personally not so sure about that, if you're like me and you feel more familiar with the sql queries rather then with ORM, it may be much easier for you to get started quickly without extension.
SQLAlchemy themselves actually recommend you use the Flask wrapper (db.Model) for Flask projects. That being said I have used the declarative_base model in several of my Flask projects where it made more sense.
It does defeat the whole purpose of the SQLAlchemy class from flask-sqlalchemy.
Here's some sample code:
from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
import datetime
#set up sqlalchemy
engine = create_engine('postgresql://<username>:<password>#localhost/flask_database')
Base = declarative_base()
metadata = Base.metadata
metadata.bind = engine
Session = sessionmaker(bind=engine, autoflush=True)
session = Session()
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
api_owner_id = Column(Integer, ForeignKey('api.id'))
email = Column(String(120), unique=True)
username = Column(String(120), unique=True)
first_name = Column(String(120))
last_name = Column(String(120))
business_name = Column(String(120))
account_type = Column(String(60))
mobile_phone = Column(String(120))
street = Column(String(120))
street2 = Column(String(120))
city = Column(String(120))
state = Column(String(120))
zip_code = Column(String(120))
country = Column(String(120))
creation_date = Column(DateTime, default=datetime.datetime.now())
password = Column(String(120))
#github stuffs
github_link = Column(Boolean, default=False)
github_usn = Column(String(120))
github_oauth_token = Column(String(160))
#balanced stuffs
balanced_account_uri = Column(String(120))
ach_verified = Column(Boolean, default=False)
active = Column(Boolean, default=True)
profile_updated = Column(Boolean, default=False)
account_balance = Column(Numeric(precision=10, scale=2), default=0.00)
admin = Column(Boolean, default=False)
devapp = relationship('DevApp', backref="user", lazy="dynamic")
projects = relationship('Project', backref="user", lazy="dynamic")
proposals = relationship('Proposal', backref="user", lazy="dynamic")
transactions = relationship('Monies', backref="user", lazy="dynamic")
def __repr__(self):
return self.email
Related
Let's say I have the following SQLAlchemy model.
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column
from sqlalchemy.types import String, Integer
Base = declarative_base()
class User(Base):
__table__ = "users"
id = Column(Integer, primary_key=True, autoincrement=True)
first_name = Column(String)
last_name = Column(String)
age = Column(Integer)
def as_dict(self):
return {key: getattr(self, key) for key in self.__mapper__.c.keys()}
And I make a query to my database like so:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from user import User
engine = create_engine("sqlite:///:memory:")
Session = sessionmaker(bind=engine)
session = Session()
# Add something to database
user = User(first_name="John", last_name="Doe", age=38)
session.add(user)
session.commit()
# query for the result now
result = session.query(User).filter_by(id=1).first()
session.expunge_all()
Now my question is, is it a bad practice to modify result after it has been expunged from the session if I just want to serialize and send it back to the client?
e.g.
# ..continuing from above
result.age = result.age + 4
print(json.dumps(result.as_dict()))
It's not bad practise if you have a legitimate reason for doing so, although it may be confusing if the data returned from your application differs from the data in your database.
Consider your example of a user with a first_name and last_name, it's perfectly legitimate to combine those and return it as name. Your business case may allow users to customise how they wish to be identified and greeted, some preferring first name whilst some users may prefer first and last. In this case it's good practise to perform this logic otherwise it would become the responsibility of the client application which may have to be repeated in several places.
Playing with Flask, Graphene and am running into a problem. Consider the following.
The Model project.model.site:
from project import db
from project.models import user
from datetime import datetime
class Site(db.Model):
__tablename__ = 'sites'
id = db.Column(db.Integer(), primary_key=True)
owner_id = db.Column(db.Integer, db.ForeignKey('users.id'))
name = db.Column(db.String(50))
desc = db.Column(db.Text())
location_lon = db.Column(db.String(50))
location_lat = db.Column(db.String(50))
creation_date = db.Column(db.DateTime(), default=datetime.utcnow())
users = db.relationship(
user,
backref=db.backref('users',
uselist=True,
cascade='delete,all'))
The model schema (project.schemas.site_schema)
from graphene_sqlalchemy import SQLAlchemyObjectType
from project.models import site as site_model
import graphene
class SiteAttributes:
owner_id = graphene.ID(description="Site owners user.id")
name = graphene.String(description="Site Name")
desc = graphene.String(description="Site description")
location_lon = graphene.String(description="Site Longitude")
location_lat = graphene.String(description="Site Latitude")
creation_date = graphene.DateTime(description="Site Creation Date")
class Site(SQLAlchemyObjectType, SiteAttributes):
"""Site node."""
class Meta:
model = site_model
interfaces = (graphene.relay.Node,)
and finally the main schema through which I plan to expose the GraphQL api (project.schemas.schema))
from graphene_sqlalchemy import SQLAlchemyConnectionField
import graphene
from project.schemas import site_schema, trade_schema, user_schema
class Query(graphene.ObjectType):
"""Query objects for GraphQL API."""
node = graphene.relay.Node.Field()
user = graphene.relay.Node.Field(user_schema.User)
userList = SQLAlchemyConnectionField(user_schema.User)
site = graphene.relay.Node.Field(site_schema.Site)
siteList = SQLAlchemyConnectionField(site_schema.Site)
trade = graphene.relay.Node.Field(trade_schema.Trade)
tradeList = SQLAlchemyConnectionField(trade_schema.Trade)
schema = graphene.Schema(query=Query)
If I load the model as such when starting up all is well. Migrations happen, the application runs perfectly fine. If I load the model through the schema though the application fails with the following message:
AssertionError: You need to pass a valid SQLAlchemy Model in Site.Meta, received "<module 'project.models.site' from '/vagrant/src/project/models/site.py'>".
I initialized SQLAlchemy with flask_sqlalchemy. Which makes me wonder is the model that is created not considered a valid SQLAlchemy Model ? Or am I doing a basic error here that I am just not seeing. I am assuming it's the latter.
Based on the error message, it seems that project.models.site (imported in the second snippet with from project.models import site as site_model) is a Python module rather than a subclass of db.Model or similar. Did you perhaps mean to import Site (uppercase) instead of site?
So fixing packages to classes finally go me in the right direction. It turns out that the issue was deeper then that. And the only way to get to it was by reading the hidden exceptions.
First I ensured that actual models where loaded rather than the modules. Thank you So much for that one #jwodder
In the end this https://github.com/graphql-python/graphene-sqlalchemy/issues/121 was ended up pointing me in the right direction. By checking the actual exception messages I found my way to a solution
I'm learning SQLAlchemy and I want to make sure that I've understood the backref parameter in relationship correctly.
For example
from app import db
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(64), unique=True)
posts = db.relationship('Post', backref='author', lazy=True)
class Post(db.Model):
id = db.Column(db.Integer, primary_key=True)
body = db.Column(db.String(140))
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
Say I have a User object j = models.User.query.get(1). My question is, is there any difference between the following things?
j.posts
Post.query.filter_by(author=j).all()
Post.query.with_parent(j).all()
Post.query.with_parent(j, property='posts').all()
Post.query.with_parent(j, property=User.posts).all()
The results returned are same, but I don't know whether the SQL statements executed are identical.
What I've tried
The SQLAlchemy docs says:
with_parent(instance, property=None, from_entity=None)
...the given property can be None, in which case a search is performed against this Query object’s target mapper.
So the last three statements seem same, but I don't really understand what does this Query object’s target mapper refer to. Is it Post in this case, for this query is performed on Post?
Even if the generated SQL statements are identical, the commands you enlisted may have a different impact on your application, e.g. j.posts will cache (memoize, do not confuse with Werkzeug caching) results you have got, while others will fetch them every single time.
If you remove .all() from your queries you can simply print them:
query = Post.query.filter_by(author=j)
print(query)
Which would result in:
SELECT post.id AS post_id, post.body AS post_body, post.user_id AS post_user_id
FROM post
WHERE ? = post.user_id
Using .all() is essentially like getting [m for m in query]).
The trick with query-printing will not work for j.posts which will return something like:
> print(j.posts)
> [Post(...), Post(..)]
Still, you can see all the silently emitted queries using built-in sqlalchemy loggers. See the following code:
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.engine import Engine
from sqlalchemy import event
import logging
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/tests.db'
db = SQLAlchemy(app)
logging.basicConfig()
logger = logging.getLogger('sqlalchemy.engine')
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(64), unique=True)
posts = db.relationship('Post', backref='author', lazy=True)
class Post(db.Model):
id = db.Column(db.Integer, primary_key=True)
body = db.Column(db.String(140))
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
db.drop_all()
db.create_all()
user = User(username='test', posts=[Post(body='some body')])
db.session.add(user)
db.session.commit()
# start logging
logger.setLevel(logging.DEBUG)
j = User.query.get(1)
queries = {
"j.posts",
"Post.query.filter_by(author=j)",
"Post.query.with_parent(j)",
"Post.query.with_parent(j, property='posts')",
"Post.query.with_parent(j, property=User.posts)",
}
def test_queries():
for name in queries:
print('\n=======')
print('Executing %s:' % name)
query = eval(name)
print(query)
test_queries() # you should see j.posts query here
print('Second test')
test_queries() # but not here
Getting back to your question: yes, the emitted SQL queries are identical.
In Query object’s target mapper, Query object's target refers to Post in your example. Decoupling this, when you declare Post class, inheriting from db.Model, for SQLAlchemy it is like creating an object Post and mapping the properties of this object to columns of specially created table.
Underneath there is an instance of Mapper class, which is responsible for the mapping for every single model that you create (learn more about mapping here: Types of Mappings). You can simply get this mapper calling class_mapper on your model or object_mapper on an instance of your model:
from sqlalchemy.orm import object_mapper, class_mapper,
from sqlalchemy.orm.mapper import Mapper
assert object_mapper(j) is class_mapper(User)
assert type(class_mapper(User)) is Mapper
The Mapper has all the necessary information about the columns and relations you have in your model. When calling Post.query.with_parent(j) this information is used to find a property (i.e. relationship) relating Post and User objects, so in your case to populate 'property' with User.posts.
To see the queries you can run your python script with -i and then run each query individually and it will print out the SQL code it runs.
Example:
main.py:
import sqlalchemy
from sqlalchemy import create_engine, Column, Integer, String, Sequence
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import os
engine = create_engine('sqlite:///:memory:', echo=True)
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = Column(Integer, Sequence('user_id_seq'), primary_key=True)
name = Column(String(50))
fullname = Column(String(50))
password = Column(String(12))
def __repr__(self):
return "< User(name={}, fullname={}, password={} )>".format(self.name, self.fullname, self.password)
Base.metadata.create_all(engine)
ed_user= User(name='ed', fullname='Ed Jones', password='edpassword')
Session = sessionmaker(bind=engine, autoflush=False)
session = Session()
session.add(ed_user)
session.add_all([
User(name='wendy', fullname='Wendy Williams', password='foobar'),
User(name='mary', fullname='Mary Contraty', password='xxg527'),
User(name='fred', fullname='Fred Flinstone', password='blah')
])
session.commit()
os.system('clear')
Now you run it with python -i main.py, type: session.query(User).filter_by(name='ed').first() and you will see the SQL generated. After running all of your tests I concluded that they are all identical. With this method you can test any query and see if there is any difference.
p.s. I added the os.system('clear') to remove all the unnecessary output from creating the database and some other stuff.
I have two separate SQLAlchemy interfaces to a Postgres database. The first interface, in the context of a Flask App, contains this model:
app = create_app() # sets the SQLAlchemy Database URI, etc.
db = SQLAlchemy(app)
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow)
updated_at = db.Column(db.DateTime, onupdate=datetime.datetime.utcnow)
name = db.Column(db.String, nullable=False)
The second interface is not through Flask -- rather, it's a script that listens for a particular event, in which case it is meant to perform some computations and update a row in the database. To accomplish this, I have SQLAlchemy reflect the existent database:
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker
from sqlalchemy.ext.automap import automap_base
from os import environ
dbPath = "postgresql://" + ...
engine = create_engine(dbPath)
Base = automap_base()
Base.prepare(engine, reflect=True)
metadata = MetaData(engine)
class User(object):
pass
users = Table('user', metadata, autoload=True, autoload_with=engine)
mapper(User, users)
Session = sessionmaker(bind=engine)
session = Session()
The issue I'm now running into is this: when I'm using the first interface to create a new entry or update one, things work as expected, and the created_at and updated_at fields are updated appropriately.
However, when I'm using the second interface -- importing the code and using session.query(User) to get an entry and to update it, the updated_at field doesn't change. Moreover, when I'm using this interface to create a new User, while it creates the new row as expected, it populates neither the created_at nor updated_at fields.
My questions:
Why is this happening? Why does the reflection seemingly break the default/onupdate methods?
How can I fix this?
default and onupdate are handled entirely client side in Python and so cannot be reflected from the DB. See "Limitations of Reflection". In case of default you could use server_default:
class User(db.Model):
...
created_at = db.Column(db.DateTime,
server_default=text("now() at time zone 'UTC'"))
and for onupdate you'd have to write a DB trigger and use server_onupdate=FetchedValue().
On the other hand you could avoid all that and just separate your models from your application code to a module, used by both your Flask application and your script. This would of course be a bit more involved as you'd have to use vanilla SQLAlchemy declarative instead of the customized db.Model base of Flask-SQLAlchemy. Or, you could use custom commands with Flask to implement your scripts, which would allow using the Flask-SQLAlchemy extensions.
I have a fairly simple flask app connected to a postgresql database. I am mainly using the flask app with flask-admin so that I can add records to the database and perhaps build it out into a dashboard later. It's an internal use catalog, basically.
What I am trying to do is also write a script that connects to a third party API to add/update records in the database, so it does not got through the flask app. I am using SQLAlchemy to do this because it's consistent with the app and I just need something to work without fussing over SQL statements.
The flask app's data model is defined as such:
app.py
from flask import Flask, render_template, request
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.dialects import postgresql
from flask_admin import Admin
# ... APPLICATION CONFIGURATION ...
# db Models
## Table for many to many
keywords = db.Table('keywords',
db.Column('keyword_id', db.Integer, db.ForeignKey('keyword.id')),
db.Column('dataset_id', db.String(24), db.ForeignKey('dataset.dataset_id')),
)
## Model classes
class Dataset(db.Model):
title = db.Column(db.String(120))
description = db.Column(db.Text())
dataset_id = db.Column(db.String(24), primary_key=True, unique=True)
#relationships
dataset_documentation = db.relationship('DataDocument', backref='dataset', lazy='dynamic')
keywords = db.relationship('Keyword', secondary=keywords, backref='dataset', lazy='dynamic')
def __str__(self):
return self.title
class Keyword(db.Model):
id = db.Column(db.Integer, primary_key=True)
keyword = db.Column(db.String(80))
def __str__(self):
return self.keyword
class DataDocument(db.Model):
id = db.Column(db.Integer, primary_key=True)
document = db.Column(db.String(120))
dataset_id = db.Column(db.String(24), db.ForeignKey('dataset.dataset_id'))
def __str__(self):
return self.document
# ... APPLICATION VIEWS ...
So we have datasets with some basic metadata and they have a one to many relationship with a filepath to a document and a many to many relationship to any number of keywords.
The separate script is connecting directly to the database, and mapping existing tables to objects that I can use to create a session and modify the database.
script.py
import config #local config only
from sqlalchemy import create_engine, MetaData, Table
from sqlalchemy.orm import mapper, sessionmaker
# Connecting to postgres database and creating a session with database objects, intantiate empty classes to populate
class Dataset(object):
pass
class DataDocument(object):
pass
class Keyword(object):
pass
## How to instantiate the MTM association table?
db_uri = config.SQLALCHEMY_DATABASE_URI
engine = create_engine(db_uri)
meta = MetaData(engine)
dataset_table = Table('dataset', meta, autoload=True) #correct
datadocument_table = Table('dataset', meta, autoload=True) #incorrect?
keyword_table = Table('keyword', meta, autoload=True) #incorrect?
mapper(Dataset, dataset_table) #correct
mapper(DataDocument, datadocument_table, meta, autoload=True) #??
mapper(Keyword, keyword_table, meta, autoload=True) #??
Session = sessionmaker(bind=engine)
session = Session()
# sample update
data_upsert = Dataset()
data_upsert.title = "Some title"
data_upsert.dataset_id = "Uniq_ID-123"
data_upsert.description = "lorem ipsum foo bar foo"
session.merge(data_upsert)
#attempt to add related properties
key1 = Keyword('test1')
key2 = Keyword('test2')
datadoc = DataDocument('path/to/document.txt')
# FAIL.
data_upsert.append(key1)
data_upsert.append(key2)
data_upsert.append(datadoc)
session.flush()
I am a newbie with sqlalchemy and I can just barely wrap my head around creating the Dataset object in the script from the database engine. But I was thinking in loading the Keyword and Datadocument tables as well that it would already understand the relationships based on what it is loading from the database, but this is where my understanding is running thin.
Is there a straightforward way to complete the picture here? I am assuming it doesn't make sense to define my models again explicitly in script.py, but in reviewing documentation and some tutorials, I am not seeing the missing pieces of loading these relationships into the session so that I can ingest all of the data into the database.
Update your model definitions to add constructor functions. In that case, it allows you to pass the parameters to the object upon instantiation.
models.py
## Model classes
class Dataset(db.Model):
title = db.Column(db.String(120))
description = db.Column(db.Text())
dataset_id = db.Column(db.String(24), primary_key=True, unique=True)
#relationships
dataset_documentation = db.relationship('DataDocument', backref='dataset', lazy='dynamic')
keywords = db.relationship('Keyword', secondary=keywords, backref='dataset', lazy='dynamic')
def __init__(self, title=None, desc=None, dataset_id=None):
self.title = title
self.description = desc
self.dataset_id = dataset_id
def __str__(self):
return self.title
class Keyword(db.Model):
id = db.Column(db.Integer, primary_key=True)
keyword = db.Column(db.String(80))
def __init__(self, keyword=None):
self.keyword = keyword
def __str__(self):
return self.keyword
class DataDocument(db.Model):
id = db.Column(db.Integer, primary_key=True)
document = db.Column(db.String(120))
dataset_id = db.Column(db.String(24), db.ForeignKey('dataset.dataset_id'))
def __init__(self, document, dataset_id):
self.document = document
self.dataset_id = dataset_id
def __str__(self):
return self.document
No need to define the model classes again in the script.py. You can simply import the classes you want to use from the models.py. Then you can insert the data object with its related objects altogether into the database in this way:
script.py
import config #local config only
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from models import Dataset, DataDocument, Keyword
def loadSession(engine):
""""""
Session = sessionmaker(bind=engine)
session = Session()
return session
engine = create_engine(config.SQLALCHEMY_DATABASE_URI, echo=False)
Base = declarative_base(engine)
# load session
session = loadSession(engine)
data_upsert = Dataset(title="Some title", dataset_id="Uniq_ID-125", desc="lorem ipsum foo bar foo")
# add related properties here
key1 = Keyword('test1')
key2 = Keyword('test2')
datadoc = DataDocument('path/to/document.txt', dataset_id="Uniq_ID-125")
# append the properties to the object
data_upsert.dataset_documentation.append(datadoc)
data_upsert.keywords.append(key1)
data_upsert.keywords.append(key2)
session.add(data_upsert)
session.commit()
I've tested the code locally and hope it works for you.