I have a db models like this:
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
ticket_status = db.Column(db.Enum(TicketStatus, name='ticket_status', default=TicketStatus.UNUSED))
departure_time = db.Column(db.Date)
I want to change the value from all ticket_status after datetime.utcnow() passed the date value from departure_time.
I tried to code like this:
class TicketStatus(enum.Enum):
UNUSED = 'UNUSED'
USED = 'USED'
EXPIRED = 'EXPIRED'
def __repr__(self):
return str(self.value)
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
ticket_status = db.Column(db.Enum(TicketStatus, name='ticket_status', default=TicketStatus.UNUSED))
departure_time = db.Column(db.Date)
# TODO | set ticket expirations time
def __init__(self):
if datetime.utcnow() > self.departure_time:
self.ticket_status = TicketStatus.EXPIRED.value
try:
db.session.add(self)
db.session.commit()
except Exception as e:
db.session.rollback()
I also tried like this:
def ticket_expiration(self, payment_id):
now = datetime.utcnow().strftime('%Y-%m-%d')
payment = Payment.query.filter_by(id=payment_id).first()
if payment.ticket_status.value == TicketStatus.USED.value:
pass
elif payment and str(payment.departure_time) < now:
payment.ticket_status = TicketStatus.EXPIRED.value
elif payment and str(payment.departure_time) >= now:
payment.ticket_status = TicketStatus.UNUSED.value
try:
db.session.commit()
except Exception as e:
db.session.rollback()
return str('ok')
But it seems no effect when the datetime.utcnow() passed the date value from departure_time.
So the point of my questions is, how to change the value from a row automatically after a set of times..?
Finally I figure out this by using flask_apscheduler, and here is the snippet of my code that solved this questions:
Install flask_apscheduler:
pip3 install flask_apscheduler
create new module tasks.py
from datetime import datetime
from flask_apscheduler import APScheduler
from app import db
from app.models import Payment, TicketStatus
scheduler = APScheduler()
def ticket_expiration():
utc_now = datetime.utcnow().strftime('%Y-%m-%d')
app = scheduler.app
with app.app_context():
payment = Payment.query.all()
for data in payment:
try:
if data.ticket_status.value == TicketStatus.USED.value:
pass
elif str(data.departure_time) < utc_now:
data.ticket_status = TicketStatus.EXPIRED.value
elif str(data.departure_time) >= utc_now:
data.ticket_status = TicketStatus.UNUSED.value
except Exception as e:
print(str(e))
try:
db.session.commit()
except Exception as e:
db.session.rollback()
return str('ok')
and then register the package with the flask app in the __init__.py
def create_app(config_class=Config):
app = Flask(__name__)
app.config.from_object(Config)
# The other packages...
# The other packages...
scheduler.init_app(app)
scheduler.start()
return app
# import from other_module...
# To avoid SQLAlchemy circular import, do the import at the bottom.
from app.tasks import scheduler
And here is for the config.py:
class Config(object):
# The others config...
# The others config...
# Flask-apscheduler
JOBS = [
{
'id': 'ticket_expiration',
'func': 'app.tasks:ticket_expiration',
'trigger': 'interval',
'hours': 1, # call the task function every 1 hours
'replace_existing': True
}
]
SCHEDULER_JOBSTORES = {
'default': SQLAlchemyJobStore(url='sqlite:///flask_context.db')
}
SCHEDULER_API_ENABLED = True
In the config above, we can call the function to update db every 1 hours, seconds or others time according to our case, for more informations to set the interval time we can see it here.
I hope this answer helps someone who facing this in the future.
You may replace your status column with just "used" column which will contain Boolean value and make a hybrid attribute for state. https://docs.sqlalchemy.org/en/13/orm/extensions/hybrid.html
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
used = db.Column(db.Boolean(), default=False)
departure_time = db.Column(db.Date)
#hybrid_property
def status(self):
if datetime.utcnow() > self.departure_time:
return "EXPIRED"
elif self.used:
return "USED"
return "UNUSED"
Related
I have made one POST API named as "/get_report" which takes some input from user and return data according to inputs by doing some search queries on database. If I keep hitting the same API multiple times like for 7-8 times, on 9th hit it throws error "sqlalchemy.exc.TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)".
Here is my main.py:
from flask_sqlalchemy import SQLAlchemy
from flask import Flask, jsonify, request
from helper import *
from models.tx import *
app = Flask(__name__)
db = SQLAlchemy()
DB_URL = 'postgresql://postgres:postgres#localhost/test_db'
engine = create_engine(DB_URL)
Session = sessionmaker(engine)
app.config['SQLALCHEMY_DATABASE_URI'] = DB_URL
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db.init_app(app)
#contextmanager
def session_manager():
s = Session()
try:
yield s
except:
s.rollback()
raise
finally:
s.close()
#app.route('/get_report', methods=['POST'])
def get_report():
try:
vendor = request.form['vendor']
circle = request.form['circle']
c_name = request.form['c_name']
c_port = request.form['c_port']
all_ssr = request.form['all_ssr']
data_list = []
table_name = get_table_by_vendor(vendor, circle)
if table_name != None:
with session_manager() as s:
if all_ssr == 'SSR':
result = s.query(table_name).distinct(table_name.label, table_name.c_node, \
table_name.c_port, table_name.z_port, table_name.z_node) \
.filter(and_((or_( and_(table_name.c_node == c_name, table_name.c_port == c_port), \
and_(table_name.z_node == c_name, table_name.z_port == c_port ))), \
(table_name.__table__.c[field_name].like('SSR%')))).all()
elif all_ssr == 'ALL':
# Get all the records
result = s.query(table_name).distinct(table_name.label, table_name.c_node, \
table_name.c_port, table_name.z_port, table_name.z_node) \
.filter(or_( and_(table_name.c_node ==c_name, table_name.c_port == c_port), \
and_(table_name.z_node == c_name, table_name.z_port == c_port ))).all()
else:
result = []
# Preparing JSON data to send
for item in result:
port = c_port if c_port != '' else (item.c_port if item.c_node == c_name else item.z_port)
data_dict = {'user': item.user, 'port': item.port, 'rate':item.rate, 'down': item.down}
data_list.append(data_dict)
response = {"status": "Success", "message": "Successfully Fetched", "data": data_list}
return jsonify(response)
if __name__ == '__main__':
app.run(host='0.0.0.0', port = 5000,debug = True)
Here is my models/tx.py:
class C_Segment(db.Model):
__tablename__ = 'c_segment'
id = db.Column(db.Integer, primary_key=True)
c_node = db.Column(db.String(350), nullable=False)
c_port = db.Column(db.String(200), nullable=False)
label = db.Column(db.String(350), nullable=False)
z_port = db.Column(db.String(200), nullable=False)
z_node = db.Column(db.String(200), nullable=False)
user = db.Column(db.String(200), nullable=False)
down = db.Column(db.String(200), nullable=False)
port = db.Column(db.String(200), nullable=False)
rate = db.Column(db.String(200), nullable=FaI
return '<id {}>'.format(self.id)
I have searched a lot and found so many related content on google but none of them worked for me. I have tried to increase pool size and overflow size also but nothing happened.
I am not able to understand where is the exact issue. I have been stuck into this from last two days and has gone through many stack overflow contents and flask sqlalchemy session documents.
You have a set of connections in the pool, e.g. 15 (5 in the pool and 10 from possible overflow). If a request processing is longer than connection checkout timeout (how long you will wait on available connection, default 30s) you will get this error (all 15 connections are busy and your request must wait for available connection - and waits but only for 30s, after that you get an error).
Have you thought about a query optimization? How many records do you have in the table? How long a request last? You can profile your SQL query using:
EXPLAIN ANALYZE <query>
You can of course increase the pool timeout by setting e.g 300s (5min):
app.config['SQLALCHEMY_POOL_TIMEOUT'] = 300
or make the pool size bigger but it would not solve your problem genuinely. Such a long response time in a request is really bad UX and limit concurrent access to you application from clients.
So I recommend you to make your query faster.
I'm just join the stackoverflow community right now because I trying to complete my first Python-flask application and I got some problem to fetch and compare a single date for single user with sqlalchemy & sqlite.
Here my imports:
from flask import Flask, request, jsonify, render_template, url_for, redirect, flash
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy import create_engine, MetaData, func
import datetime
Here my table database model:
class BloodDonation(db.Model):
id = db.Column(db.Integer, primary_key = True)
name = db.Column(db.String, nullable=False)
age = db.Column(db.Integer, nullable=False)
blood_groups = db.Column(db.String, nullable=False)
city = db.Column(db.String, nullable=False)
phone_no = db.Column(db.String, nullable=False)
latest_donation = db.Column(db.Date, nullable=False)
next_donation = db.Column(db.Date)
donation_counter = db.Column(db.Integer, default=1)
This is a easy CRUD application where I should store blood donations inside a database with 3 months threshold donation per user.
To do that I use the mobile number as user identifier and I use datetime module, so when an user try to do more than an donation before 90 days the application have to show the message donation forbidden.
Here my code:
#app.route('/blood_donation', methods=['GET','POST'])
def blood_donation():
if request.method == 'GET':
blood_groups = ['A+', 'A-', 'B+', 'B-', 'AB+', 'AB-', 'O+', 'O-']
return render_template('donate.html', blood_groups = blood_groups)
else:
donation_day = datetime.date.today()
next_donation_date = threshold_time(donation_day) # this func return donation date + 90 days
data_fields = ['name','age','blood_groups','city','phone_no']
data_dict = {}
data_dict['latest_donation'] = donation_day
data_dict['next_donation'] = next_donation_date
for field in data_fields:
data_dict[field] = request.form.get(field).lower()
for value in data_dict.values():
if value == "":
return "Please enter all the details."
counter = db.session.query(BloodDonation).filter(BloodDonation.phone_no == data_dict['phone_no']).count()
next_bd_date = db.session.query(BloodDonation).filter(func.DATE(BloodDonation.latest_donation) >= 0)
if counter >= 0 and next_bd_date and\
db.session.query(BloodDonation).filter(func.DATE(BloodDonation.latest_donation)\
<= data_dict['latest_donation']):
counter += 1
data_dict['donation_counter'] = counter
blood_donation = BloodDonation(**data_dict)
db.session.add(blood_donation)
db.session.commit()
return redirect(url_for('home'))
return "Donation forbidden!!!"
I think the problem is in the if statement to compare dates and mobile number that I use as donation counter and user identifier since the application continue update the database without return donation forbidden.
Here the wrong code:
if counter >= 0 and next_bd_date and \
db.session.query(BloodDonation).filter(func.DATE(BloodDonation.latest_donation) <=
data_dict['latest_donation']):
I hope somebody can help me since I am not a pro but a beginner
Thanks in advance!
It looks like you might have several spacing issues. Python doesn't like it if you don't have proper spacing. Try this. Also I wasn't sure where you wanted that final return statement so I'm not sure if I spaced it correctly.
#app.route('/blood_donation', methods=['GET','POST'])
def blood_donation():
if request.method == 'GET':
blood_groups = ['A+', 'A-', 'B+', 'B-', 'AB+', 'AB-', 'O+', 'O-']
return render_template('donate.html', blood_groups = blood_groups)
else:
donation_day = datetime.date.today()
next_donation_date = threshold_time(donation_day)
data_fields = ['name','age','blood_groups','city','phone_no']
data_dict = {}
data_dict['latest_donation'] = donation_day
data_dict['next_donation'] = next_donation_date
for field in data_fields:
data_dict[field] = request.form.get(field).lower()
for value in data_dict.values():
if value == "":
return "Please enter all the details."
counter = db.session.query(BloodDonation).filter(BloodDonation.phone_no == data_dict['phone_no']).count()
next_bd = db.session.query(BloodDonation).filter(func.DATE(BloodDonation.latest_donation) >= 0)
if counter >= 0 and next_bd and \
db.session.query(BloodDonation).filter(func.DATE(BloodDonation.latest_donation) <= data_dict['latest_donation']):
counter += 1
data_dict['donation_counter'] = counter
blood_donation = BloodDonation(**data_dict)
db.session.add(blood_donation)
db.session.commit()
return redirect(url_for('home'))
return "Donation forbidden!!!"
I try to save some columns (eg: tags, models) with JSON encoded string.
And I hope to always keep then decoded in use.
I have read some refers to add configs to disable autocommit and autoflush , but it doesn't work.
While the instance was added into db.session and then changed value , orm still try to commit an UPDATE OPERATION and then raise TypeError.
Here is my code.
```python
import json
from sqlalchemy import orm
from flask_sqlalchemy import SQLAlchemy
session_options = dict(
bind=None,
autoflush=False,
autocommit=False,
expire_on_commit=False,
)
db = SQLAlchemy(session_options=session_options)
class Sample(db.Model):
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
# tags, models : string of json.dumps(array)
tags = db.Column(db.String(128), default='')
models = db.Column(db.String(128), default='')
def __init__(self, **kwargs):
cls = self.__class__
super(cls, self).__init__(**kwargs)
self.formatting()
#orm.reconstructor
def init_on_load(self):
self.formatting()
def formatting(self):
self.tags = json.loads(self.tags)
self.models = json.loads(self.models)
def save(self):
self.tags = json.dumps(self.tags)
self.models = json.dumps(self.models)
db.session.add(self)
db.session.commit()
self.formatting()
## fixme !!!
## formatting after saved will cause auto-commit and raise TypeError
```
Thank you :)
ps: Flask-SQLAlchemy==2.3.2
This error was raised by lacking called db.session.close() after db.session.commit()
I was told that db.session.close() is automatically called in db.session.commit(). And the real has denied my cognition.
And I try to refer the source code of sqlalchmey, and then I find the db.session is an instance of sqlalchemy.orm.scoping.scoped_session, NOT sqlalchemy.orm.SessionTransaction.
Here is the source code in sqlalchemy.orm.SessionTransaction
def commit(self):
self._assert_active(prepared_ok=True)
if self._state is not PREPARED:
self._prepare_impl()
if self._parent is None or self.nested:
for t in set(self._connections.values()):
t[1].commit()
self._state = COMMITTED
self.session.dispatch.after_commit(self.session)
if self.session._enable_transaction_accounting:
self._remove_snapshot()
self.close()
return self._parent
It’s really confusing.
If you want to repeat this Error, here is Test code:
"""
# snippet for testing <class:Sample>
"""
from flask import Flask
app = Flask(__name__)
app.config.from_mapping(
SQLALCHEMY_ECHO=True,
SQLALCHEMY_TRACK_MODIFICATIONS=False,
SQLALCHEMY_DATABASE_URI='sqlite:///test_orm.sqlite.db',
)
db.init_app(app=app)
db.app = app
db.create_all()
d1 = dict(
tags='["python2","flask"]',
models='["m1"]'
)
m1 = Sample(**d1)
print(1111, type(m1.tags), m1.tags)
m1.save()
print(1112, type(m1.tags), m1.tags)
dm1 = Sample.query.filter(Sample.id == m1.id).all()[0]
print(1113, dm1, type(dm1.tags), dm1.tags)
## fixme[Q1] !!!
## if not continue with $d2, it won't raise error of UPDATE $d1
d2 = dict(
tags='["python3","flask"]',
models='["m2", "m3"]'
)
m2 = Sample(**d2)
print(2221, type(m2.tags), m2.tags)
## fixme[Q1] !!!
# db.session.close()
## If session was not closed, error raise here.
m2.save()
print(2222, type(m2.tags), m2.tags)
dm2 = Sample.query.filter(Sample.id == m2.id).all()[0]
print(2223, dm2, type(dm2.tags), dm2.tags)
Thank you for your read ，wish to solve your same confusion.
I use sqlalchemy to perform some queries and updates, but now I have trouble with UPDATE operation , when I change the model attribute pet_time(ins.role_time = plus_days(ins.role_time,30)) and use session to commit , sqlalchemy didn't perform the update operation.I don't know what's going on ,can anybody help?
Base = declarative_base()
class User(Base):
__tablename__ = 'user'
id = Column(Integer,primary_key=True)
nick_name = Column(String)
create_time = Column(DateTime, default=datetime.datetime.now)
role_time = Column(DateTime)
# the db connection info is correct
def getDbSession():
return Session()
def queryUserById(id):
sess = getDbSession()
instance = sess.query(User)\
.filter(User.id == id)\
.limit(1).first()
return instance
def increaseRoleTime(id,inc=1):
ins = queryUserById(id)
sess = getDbSession()
if(ins.role_time is None):
ins.role_time = datetime.datetime.now()
inc = inc * 30
# ins.role_time = func.ADDDATE(ins.role_time,inc)
ins.role_time = plus_days(ins.role_time,inc)
sess.commit()
return 1
# import timedelta from datetime to plus days
def plus_days(mydatetime,days):
return mydatetime + timedelta(days=days)
in the function increaseRoleTime,I change attribute role_time and commit,but still not working.
Flask
Python 2.7
Postgres 9
Ubuntu 14.04
I'm using Flask and SQLAlchemy, after 15 consecutive HTTP requests I get:
QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30
Then server stops responding:
Very similar as:
Flask-SQLAlchemy TimeoutError
I don't have session engine, per my understanding Flask-SQLAlchemy should take care of it.
What do I need to configure to support more sessions and to clean existing ones periodically.
app.py
import models
api_app = Flask(__name__)
api_app.config.from_pyfile(settings.celery_config)
db = SQLAlchemy(api_app)
#api_app.teardown_appcontext
def shutdown_session(exception=None):
try:
db.session.close()
except AttributeError,e:
print str(e)
except Exception,e:
print api_app.name
print str(e)
#api_app.route('/api/1.0/job/<string:ref>/')
def get_job_by_id(ref):
"""
:param id:
:return:
"""
try:
if request.method == 'GET':
job = models.Job.query.filter(models.Job.reference == ref)
if job:
job_ = job.all()
if len(job_) == 1:
return jsonify(job_[0].serialize())
resp = Response(status=404, mimetype='application/json')
return resp
else:
resp = Response(status=405, mimetype='application/json')
return resp
except Exception,e:
print str(e)
resp = Response(status=500, mimetype='application/json')
return resp
models.py
from api_app import db
class Job(db.Model, AutoSerialize, Serializer):
__tablename__ = 'job'
__public__ = ('status','description','reference')
id = Column(Integer, primary_key=True, server_default=text("nextval('job_id_seq'::regclass)"))
status = Column(String(40), nullable=False)
description = Column(String(200))
reference = Column(String(50))
def serialize(self):
d = Serializer.serialize(self)
del d['id']
return d
based on #spicyramen comment:
increase SQLALCHEMY_POOL_SIZE The size of the database pool.