I use sqlalchemy to perform some queries and updates, but now I have trouble with UPDATE operation , when I change the model attribute pet_time(ins.role_time = plus_days(ins.role_time,30)) and use session to commit , sqlalchemy didn't perform the update operation.I don't know what's going on ,can anybody help?
Base = declarative_base()
class User(Base):
__tablename__ = 'user'
id = Column(Integer,primary_key=True)
nick_name = Column(String)
create_time = Column(DateTime, default=datetime.datetime.now)
role_time = Column(DateTime)
# the db connection info is correct
def getDbSession():
return Session()
def queryUserById(id):
sess = getDbSession()
instance = sess.query(User)\
.filter(User.id == id)\
.limit(1).first()
return instance
def increaseRoleTime(id,inc=1):
ins = queryUserById(id)
sess = getDbSession()
if(ins.role_time is None):
ins.role_time = datetime.datetime.now()
inc = inc * 30
# ins.role_time = func.ADDDATE(ins.role_time,inc)
ins.role_time = plus_days(ins.role_time,inc)
sess.commit()
return 1
# import timedelta from datetime to plus days
def plus_days(mydatetime,days):
return mydatetime + timedelta(days=days)
in the function increaseRoleTime,I change attribute role_time and commit,but still not working.
Related
When querying a table, I'd like to eager load a set of columns. Prior to 1.3.x, I could do it with the below code, but it now I'm getting:
sqlalchemy.exc.ArgumentError: Attribute "AliasedClass_Blueprint.engineer" does not link from element "mapped class
Blueprint->blueprints". Did you mean to use Building.blueprint.of_type(AliasedClass_Blueprint)?
The query in question is setup as follows:
def doQuery():
building = aliased(Building)
blueprint = aliased(Blueprint, name="blueprint")
engineer = aliased(Engineer, name="engineer")
with sessionScope() as session:
return session.query(building)\
.join(blueprint, blueprint.id==building.blueprintId)\
.outerjoin(engineer, blueprint.engineerId==engineer.id)\
.options(contains_eager(building.blueprint, alias=blueprint))\
.options(contains_eager(building.blueprint, alias=blueprint)\
# The error shows up here
.contains_eager(blueprint.engineer, alias=engineer))
The error suggests using building.blueprint.of_type(blueprint), which seems to work properly, but it looks to be accomplishing something similar to what alias=blueprint does. Does of_type(x) replace the alias=x parameter?
Below is a functioning example with both a working function and one that reproduces the error:
from sqlalchemy import create_engine, inspect, Column, Integer, String, DateTime, ForeignKey
from sqlalchemy.orm import sessionmaker, relationship, aliased, contains_eager
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Engineer(Base):
__tablename__ = "engineers"
id=Column(Integer, primary_key=True)
name=Column(String)
def __repr__(self):
return self.name
class Blueprint(Base):
__tablename__ = "blueprints"
id=Column(Integer, primary_key=True)
name=Column(String)
engineerId=Column(Integer, ForeignKey('engineers.id'))
engineer=relationship(Engineer, foreign_keys=[engineerId], backref="outputBlueprints")
def __repr__(self):
return self.name
class Building(Base):
__tablename__ = "buildings"
id=Column(Integer, primary_key=True)
name=Column(String)
blueprintId=Column(Integer, ForeignKey('blueprints.id'))
blueprint = relationship(Blueprint, foreign_keys=[blueprintId], remote_side=[Blueprint.id], backref='buildings')
def __repr__(self):
return self.name
engine = create_engine('sqlite:///:memory:', echo=True)
Base.metadata.create_all(engine)
sessionFactory = sessionmaker(bind=engine, expire_on_commit=False)
#contextmanager
def sessionScope():
try:
session = sessionFactory()
yield session
session.commit()
except Exception as e:
session.rollback()
raise
finally:
session.close()
return
def doQueryWorking():
building = aliased(Building)
blueprint = aliased(Blueprint, name="blueprint")
engineer = aliased(Engineer, name="engineer")
with sessionScope() as session:
return session.query(building)\
.join(blueprint, blueprint.id==building.blueprintId)\
.outerjoin(engineer, blueprint.engineerId==engineer.id)\
.options(contains_eager(building.blueprint.of_type(blueprint)))\
.options(contains_eager(building.blueprint.of_type(blueprint))\
.contains_eager(blueprint.engineer, alias=engineer))
def doQueryError():
building = aliased(Building)
blueprint = aliased(Blueprint, name="blueprint")
engineer = aliased(Engineer, name="engineer")
with sessionScope() as session:
return session.query(building)\
.join(blueprint, blueprint.id==building.blueprintId)\
.outerjoin(engineer, blueprint.engineerId==engineer.id)\
.options(contains_eager(building.blueprint, alias=blueprint))\
.options(contains_eager(building.blueprint, alias=blueprint)\
.contains_eager(blueprint.engineer, alias=engineer))
buildings = doQueryError()
I'm getting this error when I try to post and create a new registry using fastapi and sqlalchemy:
Class 'endpoints.resultados.ResultadoPruebaSerializer.InsertTResultadoRegla' is not mapped
Here is my code, thanks for your help.
Sqlalchemy Models
#as_declarative()
class Base:
def as_dict(self) -> dict:
return {c.key: getattr(self, c.key) for c in inspect(self).mapper.column_attrs}
class TResultadoRegla(Base):
__tablename__ = 'T_resultadoReglas'
idresultadoReglas = Column(Integer, primary_key=True)
idtareas = Column(ForeignKey('T_tareas.idtareas', ondelete='RESTRICT', onupdate='RESTRICT'), nullable=False, index=True)
idreglas = Column(ForeignKey('M_reglas.idreglas', ondelete='RESTRICT', onupdate='RESTRICT'), index=True)
fecCreacion = Column(Date)
resultado = Column(String(30))
M_regla = relationship('MRegla', backref="resultados")
T_tarea = relationship('TTarea')
Pydantic crud and schemas
class InsertTResultadoRegla(BaseModel):
idtareas: int
idreglas: int
fecCreacion: date
resultado: str
class Config:
orm_mode = True
def get_regla(db: Session, skip: int = 0, limit: int = 100):
return db.query(Regla).offset(skip).limit(limit).all()
def check_resultados(db: Session, user_resultados: Dict):
db_item = None
for resultado in user_resultados:
regla = get_regla_per_id(db, resultado["idreglas"])
regla = regla.as_dict()
if resultado["resultado"] >= int(regla["minimo"]) and resultado["resultado"] <= int(regla["maximo"]):
db_item = InsertTResultadoRegla(idtareas=1, idreglas=regla["idreglas"], fecCreacion=datetime.date.today(), resultado="CUMPLE")
db.add(db_item)
db.commit()
db.refresh(db_item)
else:
db_item = InsertTResultadoRegla(idtareas=1, idreglas=regla["idreglas"], fecCreacion=datetime.date.today(), resultado="NO CUMPLE")
db.add(db_item)
db.commit()
db.refresh(db_item)
return db_item
The issue is when you are trying to add it to the db, precisely,
db.add(db_item)
You have to add it like shown below:
db_item = InsertTResultadoRegla(idtareas=1, idreglas=regla["idreglas"], fecCreacion=datetime.date.today(), resultado="CUMPLE")
actual_db_item = models.TableName(** db_item.dict())
db.add(actual_db_item)
db.commit()
Before adding the data to the db you have to convert it into a proper "db_item". I hope this solves you issue.
I am trying to scrape parts of JIRA information into lists. This works correctly. But after I try to add this information into my SQLalchemy it won't add or commit anything. Where does it come from? While debugging I see the correct values in the variables fields.
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Date, Boolean, PrimaryKeyConstraint, UniqueConstraint
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
from sqlalchemy.sql import select
from Dataextractie import testplanImport, testExecutionImport, projectversionImport
from jira_ext import JiraExt
import json
from datetime import datetime
#from Databasemodel import Requirements, TestSet, Test, ProjectVersion, TestPlan, Testexecutie, TestRun
jiraServer = JiraExt(snip)
Base = declarative_base()
#class Requirements(Base):
# __tablename__ = "requirements"
#
# ReqID = Column('reqid', Integer, primary_key=True)
# Tags = Column('tags', String)
# Desc = Column('desc', String)
#class TestSet(Base):
# __tablename__ = "testset"
# TestSetID = Column('testsetid', Integer, primary_key=True)
# TestSetName = Column('testsetname', String)
# TestSetDesc = Column('testsetdesc', String)
# Testid = Column('testid', String, ForeignKey('test.testid'))
class ProjectVersion(Base):
__tablename__ = "projectversion"
#__table_args__ = (PrimaryKeyConstraint('productrelease.releaseid', 'productrelease.productid'),)
Projectid = Column('projectid', String, primary_key=True)
Desc = Column('desc', String)
Tags = Column('tags', String)
#ForeignKeyConstraint(['productid'], ['products.productid'])
class TestPlan(Base):
__tablename__ = "testplan"
#__table_args__ = (PrimaryKeyConstraint('testplan.runid', 'testplan.reqid'),)
PlanID = Column('planid', String, primary_key=True)
ProjectID = Column('projectid', String, ForeignKey('projectversion.projectid'))
Naam = Column('naam', String)
#ForeignKeyConstraint(['reqid', 'runid'], ['requirements.reqid', 'testrun.runid'])
class Testexecutie(Base):
__tablename__ = "testexecutie"
#__table_args__ = (PrimaryKeyConstraint('testexecutie.runid', 'testexecutie.testcadeid'),)
Planid = Column('planid', String, ForeignKey("testplan.planid"), primary_key=True)
Testdesc = Column('testdesc', String)
ExecutieID = Column('executieid', String, primary_key=True)
Status = Column('status', String)
__table_args__ = (UniqueConstraint('executieid'),)
class Test(Base):
__tablename__ = "test"
#__table_args__ = (PrimaryKeyConstraint('testcase.testcaseid'),)
TestID = Column('testid', String, primary_key=True)
Casename = Column('casename', String)
Executieid = Column('ExecutieID', String, ForeignKey('testexecutie.executieid'), primary_key=True)
Result = Column('result', String)
#class TestRun(Base):
# __tablename__ = "testrun"
# #__table_args__ = (PrimaryKeyConstraint('runid', 'releaseid'),)
#
# RunID = Column('runid', String, primary_key=True)
# ExecutieID = Column('executieid', String, ForeignKey('testexecutie.executieid'))
# TestID = Column('testid', String, ForeignKey('test.testid'))
# Tags = Column('tags', String)
# Tsttimestart = Column('testtimestart', Date, nullable=False)
# Tsttimeend = Column('testtimeend', Date, nullable=False)
# Testresult = Column('testresult', String, nullable=False)
# Testoutput = Column('testoutput', String)
#ForeignKeyConstraint(['runid', 'releaseid'], ['testexecutie.runid', 'productrelease.releaseid'])
engine = create_engine('snip', echo=True)
Base.metadata.create_all(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
############################################## Session Open, Klaar voor invoer
Dataextraction = testExecutionImport("10100")
counter = 0
for identifier in Dataextraction[5]:
projectversion = ProjectVersion()
testplan = TestPlan()
projectversion.Projectid = "10100"
testplan.ProjectID = "10100"
testplan.PlanID = identifier
#testplan.Naam = identifier[0]["summary"]
session.add(projectversion, testplan)
counter +=1
session.commit
for identifier in Dataextraction[6]:
counter = 0
testexecutie = Testexecutie()
testexecutie.ExecutieID = identifier
testexecutie.Planid = Dataextraction[5][counter]
session.add(testexecutie)
counter +=1
session.commit
for identifier in Dataextraction[4]:
counter = 0
test = Test()
test.TestID = identifier
test.Executieid = Dataextraction[2][counter]
test.Result = Dataextraction[3][counter]
session.add(test)
counter +=1
session.commit
session.close()
In case it's my scraper this is the code:
from jira_ext import JiraExt
import json
from datetime import datetime
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, Date, Boolean, PrimaryKeyConstraint, UniqueConstraint
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
from collections import namedtuple
jiraServer = JiraExt(url="Snip")
def testExecutionImport(projectplan):
testplanid = jiraServer.projectGetTestPlans(projectplan)
listtestplanid = []
listtestid = []
liststatus = []
listexecutieid = []
listindvexecutieid = []
listtestplannaam = []
listprojectid = []
for issue in testplanid["issues"]:
print(issue["key"])
listtestplannaam.append(issue["fields"])
listprojectid.append(projectplan)
#listexecutieid.append(issue["id"])
executionsintestplan = jiraServer.testPlanGetAllTestExecutions(issue["key"])
for key in executionsintestplan:
#for i in range(executionsintestplan):
importapi = jiraServer.testExecutionGetTests(key["key"])
currentexecutieid = key["key"]
listindvexecutieid = key["key"]
listtestplanid.append(issue["key"])
for key in importapi:
#testid = jiraServer.testExecutionGetTests(executionsintestplan)
#executieid = jiraServer.testExecutionGetTests(executionsintestplan)
#status = jiraServer.testExecutionGetTests(executionsintestplan)
listtestid.append(key["key"])
listexecutieid.append(currentexecutieid)
liststatus.append(key["status"])
#values["projectplanid"]=projectplan
# values["testid"] = testid
# values["executieid"] = executieid
# values["status"] = status
# values["planid"] = testplanid
return listtestplannaam, listprojectid, listexecutieid, liststatus, listtestid, listtestplanid, listindvexecutieid
You're not actually calling session.commit - you're just referencing the method:
session.commit
.. should be:
session.commit()
I have a db models like this:
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
ticket_status = db.Column(db.Enum(TicketStatus, name='ticket_status', default=TicketStatus.UNUSED))
departure_time = db.Column(db.Date)
I want to change the value from all ticket_status after datetime.utcnow() passed the date value from departure_time.
I tried to code like this:
class TicketStatus(enum.Enum):
UNUSED = 'UNUSED'
USED = 'USED'
EXPIRED = 'EXPIRED'
def __repr__(self):
return str(self.value)
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
ticket_status = db.Column(db.Enum(TicketStatus, name='ticket_status', default=TicketStatus.UNUSED))
departure_time = db.Column(db.Date)
# TODO | set ticket expirations time
def __init__(self):
if datetime.utcnow() > self.departure_time:
self.ticket_status = TicketStatus.EXPIRED.value
try:
db.session.add(self)
db.session.commit()
except Exception as e:
db.session.rollback()
I also tried like this:
def ticket_expiration(self, payment_id):
now = datetime.utcnow().strftime('%Y-%m-%d')
payment = Payment.query.filter_by(id=payment_id).first()
if payment.ticket_status.value == TicketStatus.USED.value:
pass
elif payment and str(payment.departure_time) < now:
payment.ticket_status = TicketStatus.EXPIRED.value
elif payment and str(payment.departure_time) >= now:
payment.ticket_status = TicketStatus.UNUSED.value
try:
db.session.commit()
except Exception as e:
db.session.rollback()
return str('ok')
But it seems no effect when the datetime.utcnow() passed the date value from departure_time.
So the point of my questions is, how to change the value from a row automatically after a set of times..?
Finally I figure out this by using flask_apscheduler, and here is the snippet of my code that solved this questions:
Install flask_apscheduler:
pip3 install flask_apscheduler
create new module tasks.py
from datetime import datetime
from flask_apscheduler import APScheduler
from app import db
from app.models import Payment, TicketStatus
scheduler = APScheduler()
def ticket_expiration():
utc_now = datetime.utcnow().strftime('%Y-%m-%d')
app = scheduler.app
with app.app_context():
payment = Payment.query.all()
for data in payment:
try:
if data.ticket_status.value == TicketStatus.USED.value:
pass
elif str(data.departure_time) < utc_now:
data.ticket_status = TicketStatus.EXPIRED.value
elif str(data.departure_time) >= utc_now:
data.ticket_status = TicketStatus.UNUSED.value
except Exception as e:
print(str(e))
try:
db.session.commit()
except Exception as e:
db.session.rollback()
return str('ok')
and then register the package with the flask app in the __init__.py
def create_app(config_class=Config):
app = Flask(__name__)
app.config.from_object(Config)
# The other packages...
# The other packages...
scheduler.init_app(app)
scheduler.start()
return app
# import from other_module...
# To avoid SQLAlchemy circular import, do the import at the bottom.
from app.tasks import scheduler
And here is for the config.py:
class Config(object):
# The others config...
# The others config...
# Flask-apscheduler
JOBS = [
{
'id': 'ticket_expiration',
'func': 'app.tasks:ticket_expiration',
'trigger': 'interval',
'hours': 1, # call the task function every 1 hours
'replace_existing': True
}
]
SCHEDULER_JOBSTORES = {
'default': SQLAlchemyJobStore(url='sqlite:///flask_context.db')
}
SCHEDULER_API_ENABLED = True
In the config above, we can call the function to update db every 1 hours, seconds or others time according to our case, for more informations to set the interval time we can see it here.
I hope this answer helps someone who facing this in the future.
You may replace your status column with just "used" column which will contain Boolean value and make a hybrid attribute for state. https://docs.sqlalchemy.org/en/13/orm/extensions/hybrid.html
class Payment(db.Model):
id = db.Column(db.Integer(), primary_key=True)
user_id = db.Column(db.Integer(), db.ForeignKey('user.id'))
used = db.Column(db.Boolean(), default=False)
departure_time = db.Column(db.Date)
#hybrid_property
def status(self):
if datetime.utcnow() > self.departure_time:
return "EXPIRED"
elif self.used:
return "USED"
return "UNUSED"
I am using AirFlow to schedule jobs, however it becoming slower than before, especially for task_stat method in views.py, I have more than 400 dags and there are 3 millions rows in table task_instance.
I have to wait for more than 40 seconds to get the response of task_stat now, is there any ways to optimize this method?
The union_all() of RunningTI and LastTI is the slowest one, and if I remove RunningTI and keep LastTI only when union the result, I could get the response in 5 seconds, but RunningTI is necessary for front end to show detail info.
Does it possible to optimize this query? The database is MySQL.
the task_stat method:
#expose('/task_stats')
#login_required
#provide_session
def task_stats(self, session=None):
TI = models.TaskInstance
DagRun = models.DagRun
Dag = models.DagModel
LastDagRun = (
session.query(DagRun.dag_id, sqla.func.max(DagRun.execution_date).label('execution_date'))
.join(Dag, Dag.dag_id == DagRun.dag_id)
.filter(DagRun.state != State.RUNNING)
.filter(Dag.is_active == True) # noqa: E712
.filter(Dag.is_subdag == False) # noqa: E712
.group_by(DagRun.dag_id)
.subquery('last_dag_run')
)
RunningDagRun = (
session.query(DagRun.dag_id, DagRun.execution_date)
.join(Dag, Dag.dag_id == DagRun.dag_id)
.filter(DagRun.state == State.RUNNING)
.filter(Dag.is_active == True) # noqa: E712
.filter(Dag.is_subdag == False) # noqa: E712
.subquery('running_dag_run')
)
# Select all task_instances from active dag_runs.
# If no dag_run is active, return task instances from most recent dag_run.
LastTI = (
session.query(TI.dag_id.label('dag_id'), TI.state.label('state'))
.join(LastDagRun, and_(
LastDagRun.c.dag_id == TI.dag_id,
LastDagRun.c.execution_date == TI.execution_date))
)
RunningTI = (
session.query(TI.dag_id.label('dag_id'), TI.state.label('state'))
.join(RunningDagRun, and_(
RunningDagRun.c.dag_id == TI.dag_id,
RunningDagRun.c.execution_date == TI.execution_date))
)
UnionTI = union_all(LastTI, RunningTI).alias('union_ti')
# if I remove RunningTi in union_all(), and change line below to
# UnionTI = union_all(LastTI).alias('union_ti'), it could save a lot of time
qry = (
session.query(UnionTI.c.dag_id, UnionTI.c.state, sqla.func.count())
.group_by(UnionTI.c.dag_id, UnionTI.c.state)
)
data = {}
for dag_id, state, count in qry:
if dag_id not in data:
data[dag_id] = {}
data[dag_id][state] = count
session.commit()
The related models:
class DagRun(Base, LoggingMixin):
"""
DagRun describes an instance of a Dag. It can be created
by the scheduler (for regular runs) or by an external trigger
"""
__tablename__ = "dag_run"
ID_PREFIX = 'scheduled__'
ID_FORMAT_PREFIX = ID_PREFIX + '{0}'
id = Column(Integer, primary_key=True)
dag_id = Column(String(ID_LEN))
execution_date = Column(UtcDateTime, default=timezone.utcnow)
start_date = Column(UtcDateTime, default=timezone.utcnow)
end_date = Column(UtcDateTime)
_state = Column('state', String(50), default=State.RUNNING)
run_id = Column(String(ID_LEN))
external_trigger = Column(Boolean, default=True)
conf = Column(PickleType)
dag = None
__table_args__ = (
Index('dag_id_state', dag_id, _state),
UniqueConstraint('dag_id', 'execution_date'),
UniqueConstraint('dag_id', 'run_id'),
)
class TaskInstance(Base, LoggingMixin):
__tablename__ = "task_instance"
task_id = Column(String(ID_LEN), primary_key=True)
dag_id = Column(String(ID_LEN), primary_key=True)
execution_date = Column(UtcDateTime, primary_key=True)
start_date = Column(UtcDateTime)
end_date = Column(UtcDateTime)
duration = Column(Float)
state = Column(String(20))
_try_number = Column('try_number', Integer, default=0)
max_tries = Column(Integer)
hostname = Column(String(1000))
unixname = Column(String(1000))
job_id = Column(Integer)
pool = Column(String(50))
queue = Column(String(50))
priority_weight = Column(Integer)
operator = Column(String(1000))
queued_dttm = Column(UtcDateTime)
pid = Column(Integer)
executor_config = Column(PickleType(pickler=dill))
__table_args__ = (
Index('ti_dag_state', dag_id, state),
Index('ti_dag_date', dag_id, execution_date),
Index('ti_state', state),
Index('ti_state_lkp', dag_id, task_id, execution_date, state),
Index('ti_pool', pool, state, priority_weight),
Index('ti_job_id', job_id),
)
class DagModel(Base):
__tablename__ = "dag"
"""
These items are stored in the database for state related information
"""
dag_id = Column(String(ID_LEN), primary_key=True)
# A DAG can be paused from the UI / DB
# Set this default value of is_paused based on a configuration value!
is_paused_at_creation = configuration.conf\
.getboolean('core',
'dags_are_paused_at_creation')
is_paused = Column(Boolean, default=is_paused_at_creation)
# Whether the DAG is a subdag
is_subdag = Column(Boolean, default=False)
# Whether that DAG was seen on the last DagBag load
is_active = Column(Boolean, default=False)
# Last time the scheduler started
last_scheduler_run = Column(UtcDateTime)
# Last time this DAG was pickled
last_pickled = Column(UtcDateTime)
# Time when the DAG last received a refresh signal
# (e.g. the DAG's "refresh" button was clicked in the web UI)
last_expired = Column(UtcDateTime)
# Whether (one of) the scheduler is scheduling this DAG at the moment
scheduler_lock = Column(Boolean)
# Foreign key to the latest pickle_id
pickle_id = Column(Integer)
# The location of the file containing the DAG object
fileloc = Column(String(2000))
# String representing the owners
owners = Column(String(2000))
# Description of the dag
description = Column(Text)
# Default view of the inside the webserver
default_view = Column(String(25))
# Schedule interval
schedule_interval = Column(Interval)
The link to github: https://github.com/apache/airflow/blob/master/airflow/www/views.py