From my understanding of SQLAlchemy, in order to add a model to a session, I need to call session.add(obj). However, for some reason, in my code, SQLAlchemy seems to do this automatically.
Why is it doing this, and how can I stop it? Am I approaching session in the correct way?
example
>>> from database import Session as db
>>> import clients
>>> from instances import Instance
>>> from uuid import uuid4
>>> len(db.query(Instance).all())
>>> 0 # Note, no instances in database/session
>>> i = Instance(str(uuid4()), clients.get_by_code('AAA001'), [str(uuid4())])
>>> len(db.query(Instance).all())
>>> 1 # Why?? I never called db.add(i)!
database.py
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_base
import config
Base = declarative_base()
class Database():
def __init__(self):
db_url = 'postgresql://{:s}:{:s}#{:s}:{}/{:s}'.format(
config.database['user'],
config.database['password'],
config.database['host'],
config.database['port'],
config.database['dbname']
)
self.engine = create_engine(db_url)
session_factory = sessionmaker(bind=self.engine)
self.session = scoped_session(session_factory)
Database = Database()
Session = Database.session
instance.py
from sqlalchemy import Column, Text, ForeignKey
from sqlalchemy.orm import relationship
from sqlalchemy.dialects.postgresql import UUID, ARRAY
import database
Base = database.Base
class Instance(Base):
__tablename__ = 'instances'
uuid = Column(UUID, primary_key=True)
client_code = Column(
Text, ForeignKey('clients.code', ondelete='CASCADE'), nullable=False)
mac_addresses = Column(ARRAY(Text, as_tuple=True),
primary_key=True)
client = relationship("Client", back_populates="instances")
def __init__(self, uuid, client, mac_addresses):
self.uuid = uuid
self.client = client
self.mac_addresses = tuple(mac_addresses)
client.py
from sqlalchemy import Column, Text
from sqlalchemy.orm import relationship
import database
from database import Session as db
Base = database.Base
class Client(Base):
__tablename__ = 'clients'
code = Column(Text, primary_key=True)
name = Column(Text)
instances = relationship("Instance", back_populates='client')
def __init__(self, code, name=None):
self.code = code
self.name = name
def get_by_code(code):
client = db.query(Client).filter(Client.code == code).first()
return client
When you create a SQLAlchemy object and link it directly to another SQLAlchemy object, both objects end up in the session.
The reason is that SQLAlchemy needs to make sure you can query these objects.
Take, for example, a user with addresses.
If you create a user in code, with an address, both the user and the address end up in the session, because the address is linked to the user and SQLAlchemy needs to make sure you can query all addresses of a user using user.addresses.all().
In that case all (possibly) existing addresses need to be fetched, as well as the new address you just added. For that purpose the newly added address needs to be saved in the database.
To prevent this from happening (for example if you only need objects to just calculate with), you can link the objects with their IDs/Foreign Keys:
address.user_id = user.user_id
However, if you do this, you won't be able to access the SQLAlchemy properties anymore. So user.addresses or address.user will no longer yield results.
The reverse is also true; I asked a question myself a while back why linking two objects by ID will not result in SQLAlchemy linking these objects in the ORM:
relevant stackoverflow question
another description of this behavior
Related
I am working on an ORM style version of the pq library (PostgreSQL powered python queue system) where users can have their own queue model. It also has added features such as bulk insert/get, asynchronous support and more (if all goes well I'll be able to publish it).
I am having difficulties creating a trigger (I use a PostgreSQL notification system) automatically after table creation (I want to make the usage as simple as possible so that would be much better than adding an additional classmethod for creating the trigger).
This is similar to the answer in this post however I cannot use this solution because I need to pass a connection (for escaping SQL identifiers by checking the dialect of the connection and for checking if objects exist beforehand).
Here is my attempt at it based on the post I mentionned earlier. I apologize for the long code but I figured I had to include a bit of context.
Base model
from sqlalchemy import (BIGINT, Column, func, Index, nullslast,
nullsfirst, SMALLINT, TIMESTAMP)
from sqlalchemy.orm import declared_attr, declarative_mixin
from sqlalchemy.event import listens_for
# this is the function that returns the base model
def postgres_queue_base(schema:str='public', tz_aware:bool=True, use_trigger:bool=True) -> 'PostgresQueueBase':
#declarative_mixin # this is only for MyPy, it does not modify anything
class PostgresQueueBase:
__tablename__ = 'queue'
#declared_attr
def __table_args__(cls):
return (Index(nullsfirst(cls.schedule_at), nullslast(cls.dequeued_at), postgresql_where=(cls.dequeued_at == None)),
{'schema':schema})
id = Column('id', BIGINT, primary_key=True)
internal_mapping = Column('internal_mapping', BIGINT, nullable=False)
enqueued_at = Column('enqueued_at', TIMESTAMP(timezone=tz_aware), nullable=False, server_default=func.now())
dequeued_at = Column('dequeued_at', TIMESTAMP(timezone=tz_aware))
expected_at = Column(TIMESTAMP(timezone=tz_aware))
schedule_at = Column(TIMESTAMP(timezone=tz_aware))
status = Column(SMALLINT, index=True)
#listens_for(PostgresQueueBase, "instrument_class", propagate=True)
def instrument_class(mapper, class_):
print('EVENT INSTRUMENT CLASS')
if use_trigger and mapper.local_table is not None:
trigger_for_table(table=mapper.local_table)
def trigger_for_table(table):
print('Registering after_create event')
#listens_for(table, "after_create")
def create_trigger(table, connection):
print('AFTER CREATE EVENT')
# code that creates triggers and logs that (here I'll just print something and put pseudo code in a comment)
# trig = PostgresQueueTrigger(schema=get_schema_from_model(table), table_name=table.name, connection=connection)
# trig.add_trigger()
print('Creating notify function public.notify_job')
# unique trigger name using hash of schema.table_name (avoids problems with long names and special chars)
print('Creating trigger trigger_job_5d69fc3870b446d0a1f56a793b799ae3')
return PostgresQueueBase
When I try the base model
from sqlalchemy import Column, create_engine, INTEGER, TEXT
from sqlalchemy.orm import declarative_base
# IMPORTANT: inherit both a declarative base AND the postgres queue base
Base = declarative_base()
PostgresQueueBase = postgres_queue_base(schema='public')
# create custom queue model
class MyQueue(Base, PostgresQueueBase):
# optional custom table name (by default it is "queue")
__tablename__ = 'demo_queue'
# custom columns
operation = Column(TEXT)
project_id = Column(INTEGER)
# create table in database
# change connection string accordingly!
engine = create_engine('postgresql://username:password#localhost:5432/postgres')
Base.metadata.create_all(bind=engine)
EVENT INSTRUMENT CLASS
Registering after_create event
I cannot see "AFTER CREATE EVENT" printed out 😟. How do I get the "after_create" event to be fired?
Thanks in advance for your help 👍!
Sorry, I finally figured it out... The table already existed so the events were never firing. Also the code above has some errors in the events (I could not test them since they were not being executed) and the composite index in table_args somehow gets the name """ NULLS FIRST"". I used a hash to have a better name and avoid problems with character limitation or escaping.
import hashlib
from sqlalchemy import (BIGINT, Column, func, Index, nullslast,
nullsfirst, SMALLINT, TIMESTAMP)
from sqlalchemy.orm import declared_attr, declarative_mixin
from sqlalchemy.event import listens_for
# this is the function that returns the base model
def postgres_queue_base(schema:str='public', tz_aware:bool=True, use_trigger:bool=True) -> 'PostgresQueueBase':
#declarative_mixin # this is only for MyPy, it does not modify anything
class PostgresQueueBase:
__tablename__ = 'queue'
#declared_attr
def __table_args__(cls):
# to prevent any problems such as escaping, SQL injection or limit of characters I'll just md5 the table name for the index
md5 = hashlib.md5(cls.__tablename__.encode('utf-8')).hexdigest()
return (Index(f'queue_prio_ix_{md5}', nullsfirst(cls.schedule_at), nullslast(cls.dequeued_at),
postgresql_where=(cls.dequeued_at == None)),
{'schema':schema})
id = Column('id', BIGINT, primary_key=True)
internal_mapping = Column('internal_mapping', BIGINT, nullable=False)
enqueued_at = Column('enqueued_at', TIMESTAMP(timezone=tz_aware), nullable=False, server_default=func.now())
dequeued_at = Column('dequeued_at', TIMESTAMP(timezone=tz_aware))
expected_at = Column(TIMESTAMP(timezone=tz_aware))
schedule_at = Column(TIMESTAMP(timezone=tz_aware))
status = Column(SMALLINT, index=True)
if use_trigger:
#listens_for(PostgresQueueBase, "instrument_class", propagate=True)
def class_instrument(mapper, class_):
if mapper.local_table is not None:
create_trigger_event(table=mapper.local_table)
def create_trigger_event(table):
#listens_for(table, "after_create")
def create_trigger(target, connection, **kw):
print('Create trigger')
return PostgresQueueBase
Working with SQL Alchemy for the first time and I keep getting an error when trying to commit.
Python3
SQLAlchemy 1.0.12
database.py
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine("sqlite:///db.test")
db_session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
def init_db():
# import all modules here that might define models so that
# they will be registered properly on the metadata. Otherwise
# you will have to import them first before calling init_db()
import src.models
Base.metadata.create_all(bind=engine)
models.py
from datetime import datetime
from typing import List
from sqlalchemy import Column, Table
from sqlalchemy import Integer, String, DateTime
from sqlalchemy import ForeignKey
from sqlalchemy.orm import relationship
from src.database import Base
class Answer(Base):
"""
The answer for a question.
"""
__tablename__ = "answer"
answer_id = Column(Integer(), primary_key=True)
answer = Column(String(), nullable=False, index=True)
question_id = Column(Integer(), ForeignKey("question.question_id"))
question = relationship("Question", back_populates="answer")
def __init__(self, answer: str, question_id: int = None):
self.answer = answer
self.question_id = question_id
class Question(Base):
__tablename__ = "question"
question_id = Column(Integer(), primary_key=True)
question = Column(String(), nullable=False, unique=True, index=True)
answer = relationship("Answer", uselist=False, back_populates="question")
def __init__(self, question: str, answer: Answer, options: List[Option] = None):
self.question = question
self.answer = answer
if options:
self.options = options
Create database
>>> from src.database import init_db
>>> init_db()
This creates the database as expected
Add items to the database
>>> from src.models import Question, Answer
>>> a = Answer("Yes")
>>> q = Question("Doctor Who?", a)
>>> from sqlalchemy.orm import Session
>>> s = Session()
>>> s.add(a)
>>> s.add(q)
Until now I did not get an error
>>> s.commit()
Here I get the error:
sqlalchemy.exc.UnboundExecutionError: Could not locate a bind configured on mapper Mapper|Question|question or this Session
What can I do to get this working?
From "Using the Session":
sessionmaker class is normally used to create a top level Session configuration
You shouldn't try and create a session using the Session class directly. Use the scoped_session wrapped sessionmaker defined in your database.py, as it has the correct binding configured etc.:
>>> from src.models import Question, Answer
>>> from src.database import db_session
>>> a = Answer("Yes")
>>> q = Question("Doctor Who?", a)
>>> s = db_session()
>>> s.add(a)
>>> s.add(q)
>>> s.commit()
You could also use a scoped_session as a proxy.
As the title says.
Here are the codes.
from sqlalchemy import Column, ForeignKey, Integer, String, DateTime, func, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy import create_engine
import sqlalchemy.exc
from sqlalchemy import event
from settings import DB_HOST
def return_a_scoped_session():
engine = create_engine(DB_HOST)
session_factory = sessionmaker(bind=engine)
db_session = scoped_session(session_factory)
return db_session()
Base = declarative_base()
class MyClass(Base):
"""Doc string for MyClass"""
__tablename__ = 'my_table'
file_name = Column(String(512), nullable=True)
class Aria2Jobs(Base):
__tablename__ = 'nh_downloading_jobs'
id = Column(Integer, primary_key = True)
file_name = Column(String(512), nullable=True)
is_verified = Column(Boolean, default=False, nullable=True)
def check_if_verified(self):
if self.is_verified:
# create an instance
a_job= MyClass(file_name=self.file_name)
_session = return_a_scoped_session()
_session.add(a_job)
_session.commit()
_session.close()
# event
#event.listens_for(Aria2Jobs.is_verified, 'set')
def send_to_jsonpyes_jobs(target, value, oldvalue, initiator):
target.check_if_verified()
# error is when I set a property of an object (this property will trigger an event 'set' and the event will try to commit a session.
session = return_a_scoped_session()
row = session.query(Aria2Jobs).first()
row.is_verified = True
session.add(row)
# the error came out
#
# sqlalchemy.exc.invalidrequesterror object is already attached to session
session.commit()
# How to commit another object in a property of an object?
How to commit another object in a property of an object in sqlalchemy?
As you can see, when I tried session.add(row),
error:
sqlalchemy.exc.invalidrequesterror object is already attached to session
I don't know which session the row is attached to.
I want to run the function check_if_verified
-- latest error --
sqlalchemy.exc.InvalidRequestError: Object '<Aria2Jobs at 0x7fad3a635050>' is already attached to session '1' (this is
Just remove session.add(row). Becouse you used session.query the row object is already in your session.
To save the data is enough to run session.commit function
You're using scoped_session incorrectly. What's happening here is each time you call return_a_scoped_session() it's returning a new session, with a completely new engine. row is being added to a different session somewhere else. (In fact, the code you posted doesn't even show the other places it's being added; I can't reproduce your error with the code you posted.) The solution is to fix your scoped_session:
engine = create_engine(DB_HOST)
Session = scoped_session(sessionmaker(bind=engine))
def return_a_scoped_session():
return Session()
I have a table and populating it with object list then I need to use their IDs, but I am getting an
Instance <location at 0x457f3b0> is not bound to a Session; attribute refresh operation cannot proceed
error.
I am populating a list with objects and send it to a function to insert all at once. Then I try to use IDs.
Here is my insert all function:
def insertlocations(locationlist):
session.add_all(locationlist)
session.commit()
session.close()
then I try to get IDs:
insertlocations(neighbourhoodlist)
session.flush(neighbourhoodlist)
for neighbourhood in neighbourhoodlist:
print neighbourhood.locationid
Session is global by the way. Any further info needed?
The data are inserted, as I look in the MySQL table.
Most likely your problem is that you already close() the session in your insertlocations() function.
When you then access neighbourhood.locationid, the session is closed and thatneighbourhood object isn't bound to a session any more.
For example, this should work:
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import sessionmaker
engine = create_engine('sqlite:///example.db')
engine.echo = True
Base = declarative_base()
class Location(Base):
__tablename__ = 'locations'
locationid = Column(Integer, primary_key=True)
name = Column(String)
address = Column(String)
def __init__(self, name, address):
self.name = name
self.address = address
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
def insertlocations(locationlist):
session.add_all(locationlist)
session.commit()
loc1 = Location('loc1', 'Foostreet 42')
loc2 = Location('loc2', 'Barstreet 27')
neighbourhoodlist = [loc1, loc2]
insertlocations(neighbourhoodlist)
for neighbourhood in neighbourhoodlist:
print neighbourhood.locationid
session.close()
Move session.close() out of your function and do it after you're done using that session.
Ditch the session.flush(), it's not needed since you already commit the session when you add the objects.
I have problem with setting up database connection. I want to set connection, where I can see this connection in all my controllers.
Now I use something like this in my controller:
db = create_engine('mysql://root:password#localhost/python')
metadata = MetaData(db)
email_list = Table('email',metadata,autoload=True)
In development.ini I have:
sqlalchemy.url = mysql://root#password#localhost/python
sqlalchemy.pool_recycle = 3600
How do I set _____init_____.py?
I hope you got pylons working; for anyone else that may later read question I'll present some pointers in the right direction.
First of all, you are only creating a engine and a metadata object. While you can use the engine to create connections directly you would almost always use a Session to manage querying and updating your database.
Pylons automatically setups this for you by creating a engine from your configuration file, then passing it to yourproject.model.__init__.py:init_model() which binds it to a scoped_session object.
This scoped_session object is available from yourproject.model.meta and is the object you would use to query your database. For example:
record = meta.Session.query(model.MyTable).filter(id=42)
Because it is a scoped_session it automatically creates a Session object and associates it with the current thread if it doesn't already exists. Scoped_session passes all action (.query(), .add(), .delete()) down into the real Session object and thus allows you a simple way to interact the database with having to manage the non-thread-safe Session object explicitly.
The scoped_session, Session, object from yourproject.model.meta is automatically associated with a metadata object created as either yourproject.model.meta:metadata (in pylons 0.9.7 and below) or yourproject.model.meta:Base.metadata (in pylons 1.0). Use this metadata object to define your tables. As you can see in newer versions of pylons a metadata is associated with a declarative_base() object named Base, which allows you to use SqlAlchemy's declarative style.
Using this from the controller
from yourproject import model
from yourproject.model import Session
class MyController(..):
def resource(self):
result = Session.query(model.email_list).\
filter(model.email_list.c.id=42).one()
return str(result)
Use real connections
If you really want to get a connection object simply use
from yourproject.model import Session
connection = Session.connection()
result = connection.execute("select 3+4;")
// more connection executions
Session.commit()
However this is all good, but what you should be doing is...
This leaves out that you are not really using SqlAlchemy much. The power of SqlAlchemy really shines when you start mapping your database tables to python classes. So anyone looking into using pylons with a database should take a serious look at what you can do with SqlAlchemy. If SqlAlchemy starts out intimidating simply start out with using its declarative approach, which should be enough for almost all pylons apps.
In your model instead of defining Table constructs, do this:
from sqlalchemy import Column, Integer, Unicode, ForeignKey
from sqlalchemy.orm import relation
from yourproject.model.meta import Base
class User(Base):
__tablename__ = 'users'
# primary_key implies nullable=False
id = Column(Integer, primary_key=True, index=True)
# nullable defaults to True
name = Column(Unicode, nullable=False)
notes = relation("UserNote", backref="user")
query = Session.query_property()
class UserNote(Base):
__tablename__ = 'usernotess'
# primary_key implies nullable=False
id = Column(Integer, primary_key=True, index=True)
userid = Column(Integer, index=True, ForeignKey("User.id"))
# nullable defaults to True
text = Column(Unicode, nullable=False)
query = Session.query_property()
Note the query objects. These are smart object that live on the class and associates your classes with the scoped_session(), Session. This allows you to event more easily extract data from your database.
from sqlalchemy.orm import eagerload
def resource(self):
user = User.query.filter(User.id==42).options(eagerload("notes")).one()
return "\n".join([ x.text for x in user.notes ])
1.0 version of Pylons use declarative syntax. More about this, you can see here .
In mode/init.py you can write somthing like this:
from your_programm.model.meta import Session, Base
from sqlalchemy import *
from sqlalchemy.types import *
def init_model(engine):
Session.configure(bind=engine)
class Foo(Base) :
__tablename__ = "foo"
id = Column(Integer, primary_key=True)
name = Column(String)
...
What you want to do is modify the Globals class in your app_globals.py file to include a .engine (or whatever) attribute. Then, in your controllers, you use from pylons import app_globals and app_globals.engine to access the engine (or metadata, session, scoped_session, etc...).