I am trying to create a pyramid framework authentication addon/Plugin. The Plugin needs to have a database that stores user logins and other data, so if a user uses my addon his database must contain certain tables/models from the addon. e.g this Users table
class User(Base):
__tablename__ = 'User'
id = Column(Integer, primary_key=True)
username = Column(Text())
user_firstname = Column(Text())
user_lastname = Column(Text())
user_email = Column(Text())
user_password = Column(Text())
user_registrationdate = Column(DateTime())
user_email_key = Column(Text())
user_email_key_date_created = Column(DateTime())
user_email_approved = Column(Boolean())
user_email_sent = Column(Boolean())
user_active_account = Column(Boolean())
user_banned = Column(Boolean())
user_banned_reason = Column(Text())
I need this model to be included in the users main app, i am guessing i have to include something in my includeme shown below
def includeme(config):
config.include('pyramid_mako')
config.add_route('pyramid.admin', '/pyramid/admin')
#static views
config.add_static_view('assets', 'pyramidadmin:static/assets/')
config.scan('pyramidadmin.views')
but i have no Idea what to do. Is there a way i can merge the Base and DBSession from my addon to the base in the main application so that if a user runs ../bin/initialize_myapp_db the tables from myaddon and from the main app are all created?
I've had that same problem myself. I'm still working with trying to my stuff more pluggable, but with mixed results. The way I tackled this problem was to bind each declarative base class with the same engine. In my generic scaffolding I do this:
# create db engine
engine = engine_from_config(settings, 'sqlalchemy.')
# setup db.sessionmaker
settings['db.sessionmaker'] = DBSession
# bind session to engine
DBSession.configure(bind=engine)
# bind objects to engine
Base.metadata.bind = engine
from trumpet.models.base import Base as TrumpetBase
TrumpetBase.metadata.bind = engine
if settings.get('db.populate', 'False') == 'True':
from mslemon.models.main import make_test_data
import mslemon.models.misslemon
Base.metadata.create_all(engine)
TrumpetBase.metadata.create_all(engine)
#initialize_sql(engine)
You can use the following method:
import sqlalchemy
import sqlalchemy.orm as orm
from zope.sqlalchemy import ZopeTransactionExtension
from sqlalchemy.ext.declarative import declarative_base
DBSession = None
def get_sa_base(engine):
sabase = sqlalchemy.ext.declarative.declarative_base()
sabase.metadata.reflect(engine)
return sabase
def includeme(config):
global DBSession
engine = sqlalchemy.engine_from_config(config.registry.settings)
if DBSession is None:
DBSession = orm.scoped_session(
orm.sessionmaker(extension=ZopeTransactionExtension()))
DBSession.remove()
DBSession.configure(bind=engine)
Base = get_sa_base(engine)
# example:
Base.metadata.tables.values()
An example can be found here.
Related
I need help with Enum field type as it is not accepted by Swagger and I am getting error message **TypeError: Object or Type eGameLevel is not JSON serializable**. Below is the complete set of code for table. Complete set of code with DB table and sqlalchemy settings is provided. I already tried it with Marshmallow-Enum Flask package and it didn't worked. Looking for kind help with some explanation about the solution so I can learn it well. :-)
I am using MySQL with the Flask. In Postgres its pretty easy to manage all the choice fields. All I need is a working example or a link to repository where MySQL choice fields are showing up in swagger drop down.
My Model:
import enum
from app import db
from typing import List
class eGameLevel(enum.Enum):
BEGINNER = 'Beginner'
ADVANCED = 'Advanced'
class Game(Base):
__tablename__ = 'game_stage'
id = db.Column(db.Integer(), primary_key=True)
game_level= db.Column(db.Enum(eGameLevel),
default=eGameLevel.BEGINNER, nullable=False)
user_id = db.Column(db.Integer(), db.ForeignKey('users.id', ondelete='CASCADE'), nullable=False)
user = db.relationship('User', backref='game__level_submissions', lazy=True)
def __init__(self, game_level, user_id):
self.game_level = game_level
self.user_id = user_id
def __repr__(self):
return 'Game(game_level%s, ' \
'user_id%s'%(self.game_level,
self.user_id)
def json(self):
return {'game_level':self.game_level,
'user_id':self.user_id}
#classmethod
def by_game_id(cls, _id):
return cls.query.filter_by(id=_id)
#classmethod
def find_by_game_level(cls, game_level):
return cls.query.filter_by(game_level=game_level)
#classmethod
def by_user_id(cls, _user_id):
return cls.query.filter_by(user_id=_user_id)
#classmethod
def find_all(cls) -> List["Game"]:
return cls.query.all()
def save_to_db(self) -> None:
db.session.add(self)
db.session.commit()
def delete_from_db(self) -> None:
db.session.delete(self)
db.session.commit()
My Schema
from app import ma
from app.models import Gode
class GameSchema(ma.SQLAlchemyAutoSchema):
game = ma.Nested('GameSchema', many=True)
class Meta:
model = Game
load_instance = True
include_fk= True
My Resources:
from flask_restx import Resource, fields, Namespace
from app.models import Game
from app import db
from app.schemas import GameSchema
GAME_REQUEST_NOT_FOUND = "Game request not found."
GAME_REQUEST_ALREADY_EXSISTS = "Game request '{}' Already exists."
game_ns = Namespace('Game', description='Available Game Requests')
games_ns = Namespace('Game Requests', description='All Games Requests')
game_schema = GameSchema()
games_list_schema = GameSchema(many=True)
gamerequest = game_ns.model('Game', {
'game_level': fields.String('Game Level: Must be one of: BEGINNER, ADVANCED.'),
'user_id': fields.Integer,
})
class GameRequestsListAPI(Resource):
#games_ns.doc('Get all Game requests.')
def get(self):
return games_list_schema.dump(Game.find_all()), 200
#games_ns.expect(gamerequest)
#games_ns.doc("Create a Game request.")
def post(self):
game_json = request.get_json()
game_data = game_schema.load(game_json)
game_data.save_to_db()
return game_schema.dump(game_data), 201
Instead of trying to manage Enum fields for MySQL schema I suggest to use another table with backref to your eGameLevel. You can get rid of this whole fuss and also in future if you needed to add another option in your choice field you won't have to hardcode it.
Simply create a main table as Game and sub table as eGameLevel (with only one string field). You will be able to access choices from your Game table.
Whenever I get stuck I go to basics as mentioned in here.
I made a small example just to test the serialization of an Enum
from enum import Enum
import sqlalchemy as sa
from flask import Flask
from flask_restx import Api, Namespace, Resource
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
class eGameLevel(str, Enum):
BEGINNER = "Beginner"
ADVANCED = "Advanced"
engine = sa.create_engine("sqlite:///:memory:")
session = scoped_session(sessionmaker(bind=engine))
Base = declarative_base()
class Game(Base):
__tablename__ = "game"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True)
level = sa.Column(sa.Enum(eGameLevel), default=eGameLevel.BEGINNER, nullable=False)
def __repr__(self):
return f"Game(id={self.id}, level={self.level})"
def json(self):
data = {"id": self.id, "level": self.level}
return data
Base.metadata.create_all(engine)
g1 = Game(level=eGameLevel.BEGINNER)
g2 = Game(level=eGameLevel.ADVANCED)
session.add_all([g1, g2])
session.commit()
query_content = session.query(Game).all()
games_ns = Namespace("Game Requests", description="All Games Requests")
app = Flask(__name__)
api = Api(app)
#api.route("/game")
class GameRequestsListAPI(Resource):
#games_ns.doc("Get all Game requests.")
def get(self):
data = [x.json() for x in query_content]
return data, 200
app.run(debug=True)
This example works and I think the serialization is possible due to the str in the Enum declaration: class eGameLevel(str, Enum).
Instead of using Enum:
class eGameLevel(enum.Enum):
BEGINNER = 'Beginner'
ADVANCED = 'Advanced'
You can make use of dictionary:
eGameLevel = {"BEGINNER": 1, "ADVANCED": 2}
Then you can replace enum type for sql data model to String type as:
game_level= db.Column(db.Integer(),
default=eGameLevel["BEGINNER"], nullable=False)
And make appropriate checks using the defined dictionary throughout application. This will also solve issues with alembic as well for making db migrations.
You would also require modifying some of your python files. I would rather do it here directly, and then you can look up to modify them:
#Import at Resources
from flask import request
from app.models import Game, eGameLevel
Post Part:
# For post part
payload = request.json
game_obj = Game(game_level=eGameLevel[payload["game_level"]], user_id=payload["user_id"])
db.session.add(game_obj)
db.session.commit()
Furthermore, I did not understand what the from app.models import Gode meant.
I have a flask restful app connected to mySQL database and I am using SQLAlchemy. We can connect to the mySQL server using the following -
app.config['SQLALCHEMY_DATABASE_URI'] = f"mysql+pymysql://root:password#127.0.0.1:3306"
I am working on a use case where the database name will be provided on real-time basis through a GET request. Based on the database name provided, the app will connect to the respective database and perform the operations. For this purpose, I would like to have a way where I can tell the flask app to talk to the provided database (Flask app is already connected to the mySQL server). Currently, I am creating the connection again in the API class.
API: Calculate.py
from flask_restful import Resource, reqparse
from app import app
class Calculate(Resource):
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('schema', type=str, required=True, help='Provide schema name.')
args = parser.parse_args()
session['schema_name'] = args.get('schema')
app.config['SQLALCHEMY_DATABASE_URI'] = f"mysql+pymysql://root:password#127.0.0.1:3306/{session['schema_name']}"
from db_models.User import User
...
DB Model: User.py
from flask import session
from flask_sqlalchemy import SQLAlchemy
from app import app
db = SQLAlchemy(app)
class User(db.Model):
__tablename__ = 'user'
__table_args__ = {"schema": session['schema_name']}
User_ID = db.Column(db.Integer, primary_key=True)
Name = db.Column(db.String(50))
db.create_all()
The above thing works for me. But I would want to understand if there is an alternative to this or a better way of doing this.
Edit: The above code does not work. It references the first schema name that was provided even if I provide a new schema name in the same running instance of the app.
you can write the SQLALCHEMY path like this:
SQLALCHEMY_DATABASE_URI='mysql+pymysql://root:password#localhost:3306/database name'
According to the docs not all values can be updated (first parragraph), in your use case you should use SQLALCHEMY_BINDS variable in your use case this is a dict and create a Model for each schema. Example:
Db Model
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
SQLALCHEMY_DATABASE_URI = f"mysql+pymysql://root:password#127.0.0.1:3306/schema_name1"
SQLALCHEMY_BINDS = {
'db1': SQLALCHEMY_DATABASE_URI, # default
'db2': f"mysql+pymysql://root:password#127.0.0.1:3306/schema_name2"
}
app = Flask(__name__)
db = SQLALchemy(app)
then create a model for each schema
class UserModeldb1(db.Model):
__tablename__ = 'user'
__bind_key__ = 'db1' #this parameter is set according to the database
id = db.Column(db.Integer, primary_key=True)
...
class UserModeldb2(db.Model):
__tablename__ = 'user'
__bind_key__ = 'db2'
id = db.Column(db.Integer, primary_key=True)
...
finally in your get method add some logic to capture the schema and execute your model accorddingly. you should look this question is really helpful Configuring Flask-SQLAlchemy to use multiple databases with Flask-Restless
I have an flask app, using flask-slqalchemy to query a mysql database
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:password#localhost/abc'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)
there is a table "users" in "abc" database and it is already populated with several hundred rows.
Now i need to import this existing table, rather than first defining it with db.Model
how do i call the table?
if i do this
from sqlalchemy import Table
USERS = Table('users', db.metadata,autoload=True, autoload_with=db.engine)
then i am not able to make a query like
USERS.query.filter_by(done=1).with_entities(USERS.name,USERS.country).paginate(page, 15, False)
it generates an error
AttributeError: 'Table' object has no attribute 'query'
because this is sqlchemy command, not flask-sqlchemy, i dont fully understand this.
I have to first define the table USERS like i am creating it for the first time :
class USERS(db.Model):
__tablename__ = 'users'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
name = db.Column(db.VARCHAR(500))
country = db.Column(db.VARCHAR(50))
def __init__(self, id, name, country):
self.id = id
self.name = name
self.country = country
def __repr__(self):
return self.id
only then i am able to use USERS to query the database through flask-sqlalchemy
How do i access the an existing table users using flask-sqlchemy in an flask app?
In sqlalchemy you should query table(s) with session if you want to query Table(). Because 'Table' object has no attribute 'query'. And you do not need to create table if it has existed, just use it. sqlalchemy existing database query
from sqlalchemy import Table, Column, String, create_engine, MetaData
from sqlalchemy.orm import sessionmaker
engine = create_engine()
metadata = MetaData()
test_ = Table('test', metadata,
Column('msg', String, primary_key=True),
Column('msg_', String)
)
Session = sessionmaker(bind=engine)
session = Session()
print(session.query(test_).filter_by(msg_ = "test").with_entities("msg","msg_").one())
# ('t', 'test')
In flask_sqlalchemy, it almost same as sqlalchemy did.
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = ""
db = SQLAlchemy(app)
class test(db.Model):
msg = db.Column(db.String, primary_key=True)
msg_ = db.Column(db.String)
def __init__(self, msg, msg_):
self.msg = msg
self.msg_ = msg_
def __repr__(self):
return "msg: {} msg_: {}".format(self.msg,self.msg_)
result = test.query.filter_by(msg_="test").one()
print(result)
print(result.msg,result.msg_)
'''
msg: t msg_: test
t test
'''
I have a Flask app using Flask-SQLAlchemy with a MySQL database where the db is defined as the following:
db.py:
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
main.py:
from db import db
app = Flask(__name__)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_DATABASE_URI'] = "mysql+pymysql://" + \
DB_USERNAME + ":" + DB_PASSWORD + "#" + DB_HOST + "/" + DB_DATABASE
db.init_app(app)
#app.teardown_appcontext
def teardown_db(error):
db.session.close()
db.engine.dispose()
user.py:
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(80), unique=True, nullable=False)
email = db.Column(db.String(120), unique=True, nullable=False)
I query my database using models using either db.engine.execute() to write raw SQL queries where required or use the integrated Flask-SQLAlchemy APIs for reading data such as User.query.filter_by().all().
I write new data into the db using the following:
new_user_entry = User(username = "abc", email = "abc#example.com")
db.session.add(new_user_entry)
db.session.commit()
I am monitoring my MySQL server using show processlist and I notice that the database connections keep increasing by 2 for every single request that comes my way. The database connections seem to reset only when I stop the Flask process. With time, the MySQL server throws the below error:
`sqlalchemy.exc.TimeoutError: QueuePool limit of size 10 overflow 10 reached, connection timed out, timeout 30 (Background on this error at: http://sqlalche.me/e/3o7r)`
I am serving the app using gunicorn and gevent/eventlet with 2 worker processes. I use python3.
Am I missing something here? I tried ending the db session and disposing the engine, but this does not seem to work.
I finally found a fix to the above problem.
I used the declarative model defined in here instead of following the quickstart documentation for Flask-SQLAlchemy given here.
The changed files are as follows:
db.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine(DB_URI, convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
def init_db():
import user
Base.metadata.create_all(bind=engine)
main.py:
from db import init_db, db_session
init_db()
#app.teardown_appcontext
def shutdown_session(exception=None):
db_session.remove()
user.py:
from sqlalchemy import Column, Integer, String
from data_models.db import Base
class User(Base):
id = db.Column(Integer, primary_key=True)
username = db.Column(String(80), unique=True, nullable=False)
email = db.Column(String(120), unique=True, nullable=False)
To query for records we could either use User.query.filter_by().all() or db_engine.execute().
To write new data into the database, we can use the following:
new_user_entry = User(username = "abc", email = "abc#example.com")
db_session.add(new_user_entry)
db_session.commit()
In case we need to close session before creating a new child process (what is recommended), this is what we should use:
db.session.remove()
db.engine.dispose()
Like
from multiprocessing import Process
from app import db
#app.route('/process/start/', methods = ["GET"])
def process_start():
db.session.remove()
db.engine.dispose()
p = Process(target = long_task)
p.start()
return 'started the long task'
def long_task():
'''
do long task
'''
Use with statement, there is a test:
def test():
with db.session() as dbss:
qss = models.WhateverModel.query.session
assert dbss == qss
I have a project where I want to isolate DB initialization (SQLAlchemy) from the other module so I've create a module
db_initializer.py:
engine = create_engine('sqlite:///db') # TODO from config file
Session = sessionmaker(bind=engine)
Base = declarative_base(bind=engine)
def create_tables():
Base.metadata.create_all(engine)
First of all I need to put create_all in a function because my model is in another package.
model/foo.py:
from core.db_initializer import Base
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
name = Column(String)
def __init__(self, name: str = None):
self.name = name
def __repr__(self):
return "<User(id=%d, name=%s)>" % (int(self.id), str(self.name))
And my main call create_tables.
Is there any other to do that? And now I need to create the engine with custom config (IP,User, ...) and only the main script know the config it's possible?
Something like
main.py:
import db_initializer as db
import model.foo
db.init(config) # which create the engine and create table
When I do something like that I got problem with the Base object which have not bind to the engine which is not created yet. Any solutions?
You don't need to create engine or session before declaring your models. You can declare models in model/foo.py:
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class User(Base):
__tablename__ = 'user'
Let's assume you have some application in myapp.py. Declare it so it can be initialized with engine:
from sqlalchemy.orm import Session
import model.foo
class MyApp:
def __init__(self, engine):
self.engine = engine
def get_users(self):
session = Session(self.engine)
users = session.query(model.foo.User).all()
session.close()
return users
Create engine in main.py and use it to initialize models.foo.Base.metadata and other applications where you need it:
from sqlalchemy import create_engine
import model.foo
import myapp
engine = create_engine('sqlite:///db')
model.foo.Base.metadata.bind = engine
model.foo.Base.metadata.create_all()
app = myapp.MyApp(engine)
UPD: For scoped_session approach myapp.py can be look like this:
import model.foo
class MyApp:
def __init__(self, session):
self.session = session
def get_users(self):
return self.session.query(model.foo.User).all()
And main.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
import model.foo
import myapp
engine = create_engine('sqlite:///db')
session = scoped_session(sessionmaker(engine))
model.foo.Base.metadata.bind = engine
model.foo.Base.metadata.create_all()
app = myapp.MyApp(session)