We Keep Coding

Django import-export library ForeignKey error (IntegrityError: FOREIGN KEY constraint failed in django)

First of all please sorry for my English. ))
So I have a problem with Django import-export library when I try to import data from csv/xls/xlsx files to the Django application DB.
How it looks like.
Here is my models.py:
class Department(models.Model):
department_name = models.CharField(max_length = 50, default = '')
def __str__(self):
return f'{self.department_name}'
class ITHardware(models.Model):
it_hardware_model = models.CharField(max_length = 100)
it_hardware_serial_number = models.CharField(max_length = 100,
blank = True, default = '')
it_hardware_department = models.ForeignKey(Department,
related_name = 'department', on_delete = models.SET_NULL, default = '',
null = True, blank = True, db_constraint=False)
admin.py:
#admin.register(Department)
class DepartmentAdmin(admin.ModelAdmin):
list_display = ('department_name', )
actions = [duplicate_object]
#admin.register(ITHardwareManufacturer)
class ITHardwareManufacturerAdmin(admin.ModelAdmin):
list_display = ('manufacturer', )
actions = [duplicate_object]
class ITHardwareImportExportAdmin(ImportExportModelAdmin):
resource_class = ITHardwareResource
list_display = ['id', 'it_hardware_manufacturer',
'it_hardware_model', 'it_hardware_serial_number',
'it_hardware_department']
actions = [duplicate_object]
resource.py:
class ITHardwareResource(resources.ModelResource):
it_hardware_department = fields.Field(
column_name = 'it_hardware_department',
attribute = 'ITHardware.it_hardware_department',
widget = widgets.ForeignKeyWidget(Department, field = 'department_name'))
class Meta():
model = ITHardware
fields = (
'id',
'it_hardware_model',
'it_hardware_serial_number',
'it_hardware_department',
)
export_order = (
'id',
'it_hardware_model',
'it_hardware_serial_number',
'it_hardware_department',
)
import file:
If I try to import data from file, I get such error:
String number: 1 - ITHardwareManufacturer matching query does not exist.
None, Canon, BX500CI, 5B1837T00976, Office_1, IT_1
And so on.
OK. When I fill Department table manually via adminpanel (pointed there IT_1, 2, 3 and 4 departments) I get this preview:
and then I get this error:
Please explain me what am I doing wrong and how to fix it.
Update
If I understood the concept correctly, then "column_name" is the name of the column from which/to which data is imported/exported, and "attribute" is the name of the database field for the same operations.
The export works fine for me.
However, for import, as I understand it, I incorrectly describe the mechanism for ForeignKey. That is, I'm trying to tell Django that when importing data from the "it_hardware_department" column of the file, this data should be written to the "it_hardware_department" field of the Department model using ForeignKeyWidget.
Perhaps some intermediate action is required to determine the mechanism for writing data to the Department model, where to specify something like "Department__it_hardware_department"?
Update: traceback
Traceback (most recent call last):
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\backends\base\base.py", line 242, in _commit
return self.connection.commit()
The above exception (FOREIGN KEY constraint failed) was the direct cause of the following exception:
File "C:\users\iv\mip\mip_env\lib\site-packages\django\core\handlers\exception.py", line 47, in inner
response = get_response(request)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\core\handlers\base.py", line 181, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\utils\decorators.py", line 130, in _wrapped_view
response = view_func(request, *args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\views\decorators\cache.py", line 44, in _wrapped_view_func
response = view_func(request, *args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\contrib\admin\sites.py", line 232, in inner
return view(request, *args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\utils\decorators.py", line 43, in _wrapper
return bound_method(*args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\views\decorators\http.py", line 40, in inner
return func(request, *args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\import_export\admin.py", line 113, in process_import
result = self.process_dataset(dataset, confirm_form, request, *args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\import_export\admin.py", line 125, in process_dataset
return resource.import_data(dataset,
File "C:\users\iv\mip\mip_env\lib\site-packages\import_export\resources.py", line 771, in import_data
return self.import_data_inner(
File "C:\users\iv\mip\mip_env\lib\site-packages\import_export\utils.py", line 25, in __exit__
self.context_manager.__exit__(*args)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\transaction.py", line 246, in __exit__
connection.commit()
File "C:\users\iv\mip\mip_env\lib\site-packages\django\utils\asyncio.py", line 33, in inner
return func(*args, **kwargs)
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\backends\base\base.py", line 266, in commit
self._commit()
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\backends\base\base.py", line 242, in _commit
return self.connection.commit()
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\users\iv\mip\mip_env\lib\site-packages\django\db\backends\base\base.py", line 242, in _commit
return self.connection.commit()
Exception Type: IntegrityError at /admin/mip_apps/ithardware/process_import/
Exception Value: FOREIGN KEY constraint failed

Based on your code sample, the import process will look for a field called it_hardware_department in the import file (and we can see that it is there). During import, it will attempt to find the FK relation in Department by doing a simple get(name=<value in column>. It will store this value in ITHardware.it_hardware_department - however this won't work because this isn't a valid model field. You should rename this to it_hardware_department (which you said you did). The fact that shows up in the preview screen tells me that this worked ok.
The 'FK contraint' failed error says that you have tried to write to the table with an invalid FK relationship. I suggest run the debugger or add a print to inspect the object before it is saved. It is likely that one of your required Foreign Keys will be empty here. I don't think it is it_hardware_department because this has a value in the preview. Are there other FKs in your model which are not listed?

db.drop_all() gives no such table error when I try to clear my database using command line arguments [closed]

Closed. This question is not reproducible or was caused by typos. It is not currently accepting answers.
This question was caused by a typo or a problem that can no longer be reproduced. While similar questions may be on-topic here, this one was resolved in a way less likely to help future readers.
Closed 1 year ago.
Improve this question
I am using a command line argument of -clear to empty the database of my application before the program starts up. My problem is that calling db.drop_all() from the run.py file throws a "no such table" error. I am wondering what the best approach to clearing my database before running the application would be. Here is my relevant code.
run.py
# Imports
from wms import desktopApp, db
from wms.functions import load_data
import argparse
# Parsing command line arguments
description = """IMPORTANT: When loading information to the database, please insure that all files are stored in the
resources folder and that the file format matches that described in the application documentation."""
parser = argparse.ArgumentParser(description=description)
# Arguments
parser.add_argument("-warehouses",
help="Loads the supplied warehouses.csv file into the database when the program is run.",
action="store_true")
parser.add_argument("-itemTemplates",
help="Loads the supplied item_templates.csv file into the database when the program is run.",
action="store_true")
parser.add_argument("-items",
help="Loads the supplied items.csv file into the database when the program is run.",
action="store_true")
parser.add_argument("-clear",
help="Clears the existing database.",
action="store_true")
args = parser.parse_args()
successful = load_data(args.warehouses, args.itemTemplates, args.items)
if not successful:
quit()
if __name__ == '__main__':
# Clear database
if args.clear:
while True:
confirmation = input("Are you sure you want to clear the database? (y/n): ")
if confirmation.lower() in ["yes", "y"]:
# Drop db logic
db.drop_all()
print("Database cleared.")
break
elif confirmation.lower() in ["no", "n"]:
print("Good thing we double checked.")
break
else:
pass
desktopApp.run()
desktopApp.keep_server_running()
init.py
# Imports
from flask import Flask
from flaskwebgui import FlaskUI
from flask_sqlalchemy import SQLAlchemy
# App config
app = Flask(__name__)
app.config["DEBUG"] = True
app.config["TESTING"] = True
app.config["TEMPLATES_AUTO_RELOAD"] = True
app.config["SECRET_KEY"] = '8e3416a9c67b328517b7b758875aaea0'
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///data.db"
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False # Removes warning message in console
# Database config
db = SQLAlchemy(app)
from wms.models import Warehouse, Item, ItemTemplate
db.create_all()
# Desktop UI config
desktopApp = FlaskUI(app,
width=1200,
height=675,
browser_path='C:\Program Files (x86)\Google\Chrome\Application\chrome.exe')
models.py
# Imports
from wms import db
from sqlalchemy.ext.hybrid import hybrid_property
# Constants
MAXIMUM_NAME_LEN = 14
LARGE_SCALE_SIZE = 150
# Template for creating items
class ItemTemplate(db.Model):
"""This template will simply store the information related to an item type.
Individual items that will be associated with the warehouse they're stored in
will inherit this information from the item templates."""
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(MAXIMUM_NAME_LEN), unique=True, nullable=False)
price = db.Column(db.Float, nullable=False)
cost = db.Column(db.Float, nullable=False)
size = db.Column(db.Integer, nullable=False)
low_threshold = db.Column(db.Integer, nullable=False)
instances = db.relationship('Item', backref='item_template', lazy=True) # Instances using this item template
#hybrid_property
def total_stock(self):
total_stock = 0
for instance in self.instances:
total_stock += instance.quantity
return total_stock
#hybrid_property
def possible_revenue(self):
return self.total_stock * (self.price - self.cost)
#hybrid_property
def warehouses(self):
return [instance.warehouse for instance in self.instances]
#hybrid_property
def stock(self):
stock = 0
for instance in self.instances:
stock += instance.quantity
return stock
# Actual items
class Item(db.Model):
"""This template will be used to represent the actual items that are associated with a warehouse."""
id = db.Column(db.Integer, primary_key=True)
quantity = db.Column(db.Integer, nullable=False)
warehouse = db.Column(db.String(MAXIMUM_NAME_LEN), db.ForeignKey('warehouse.name'), nullable=False)
item_template_id = db.Column(db.Integer, db.ForeignKey('item_template.id'), nullable=False)
# Warehouse
class Warehouse(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(MAXIMUM_NAME_LEN), unique=True, nullable=False)
capacity = db.Column(db.Integer, nullable=False)
items = db.relationship("Item", backref="home_warehouse", lazy=True)
#hybrid_property
def remaining_capacity(self):
capacityUsed = 0
for item in self.items:
capacityUsed += item.quantity * ItemTemplate.query.filter_by(id=item.item_template_id).first().size
return self.capacity - capacityUsed
#hybrid_property
def possible_revenue(self):
revenue = 0
for item in self.items:
item_temp = ItemTemplate.query.filter_by(id=item.item_template_id).first()
revenue += item.quantity * (item_temp.price - item_temp.cost)
return revenue
#hybrid_property
def total_production_cost(self):
total_production_cost = 0
for item in self.items:
total_production_cost += item.quantity * ItemTemplate.query.filter_by(id=item.item_template_id).first().cost
return total_production_cost
#hybrid_property
def low_stock_items(self):
low_stock = []
for item in self.items:
if item.quantity <= ItemTemplate.query.filter_by(id=item.item_template_id).first().low_threshold:
low_stock.append(item)
return low_stock
#hybrid_property
def large_scale_items(self):
large = []
for item in self.items:
if ItemTemplate.query.filter_by(id=item.item_template_id).first().size >= LARGE_SCALE_SIZE:
large.append(item)
return large
#hybrid_property
def item_names(self):
return [ItemTemplate.query.filter_by(id=item.item_template_id).first().name for item in self.items]
Error message:
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1799, in _execute_context
self.dialect.do_execute(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: no such table: warehouse
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\waitress\channel.py", line 397, in service
task.service()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\waitress\task.py", line 168, in service
self.execute()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\waitress\task.py", line 434, in execute
app_iter = self.channel.server.application(environ, start_response)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 2091, in __call__
return self.wsgi_app(environ, start_response)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 2076, in wsgi_app
response = self.handle_exception(e)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 2073, in wsgi_app
response = self.full_dispatch_request()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 1518, in full_dispatch_request
rv = self.handle_user_exception(e)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 1516, in full_dispatch_request
rv = self.dispatch_request()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\flask\app.py", line 1502, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "C:\coding\python\projects\WMS\wms\routes.py", line 22, in home
warehouseList = Warehouse.query.all()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2711, in all
return self._iter().all()
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\query.py", line 2846, in _iter
result = self.session.execute(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\orm\session.py", line 1689, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1611, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\sql\elements.py", line 325, in _execute_on_connection
return connection._execute_clauseelement(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1478, in _execute_clauseelement
ret = self._execute_context(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1842, in _execute_context
self._handle_dbapi_exception(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 2023, in _handle_dbapi_exception
util.raise_(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\base.py", line 1799, in _execute_context
self.dialect.do_execute(
File "C:\Users\golin\AppData\Local\Programs\Python\Python310\lib\site-packages\sqlalchemy\engine\default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) no such table: warehouse
[SQL: SELECT warehouse.id AS warehouse_id, warehouse.name AS warehouse_name, warehouse.capacity AS warehouse_capacity
FROM warehouse]
(Background on this error at: https://sqlalche.me/e/14/e3q8)

How to properly handle many to many in async sqlalchemy?

I was trying to implement many to many relationship between tables.
When I use backpopulates all tags for a specific user must be in the tags field.
The tables are successfully created.
Users and tags are added.
Link table too.
import asyncio
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy.util import await_only, greenlet_spawn
from sqlalchemy import Column, Table, ForeignKey
from sqlalchemy.orm import declarative_base, relationship
from sqlalchemy.dialects.postgresql import VARCHAR, INTEGER
Base = declarative_base()
user_tag = Table('user_tag', Base.metadata,
Column('user_id', INTEGER, ForeignKey('users.id')),
Column('tag_id', INTEGER, ForeignKey('tags.id'))
)
class User(Base):
__tablename__ = 'users'
id = Column(INTEGER, primary_key=True)
name = Column(VARCHAR(32), nullable=False, unique=True)
tags = relationship("Tag",
secondary=user_tag,
back_populates="users")
class Tag(Base):
__tablename__ = 'tags'
id = Column(INTEGER, primary_key=True)
tag = Column(VARCHAR(255), nullable=False, unique=True)
users = relationship("User",
secondary=user_tag,
back_populates="tags")
async def main():
engine = create_async_engine(
"postgresql+asyncpg://postgres:pgs12345#localhost/test",
echo=False,
)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.drop_all)
await conn.run_sync(Base.metadata.create_all)
users = [User(name="p1"), User(name="p2"), User(name="p3")]
tags = [Tag(tag="tag1"), Tag(tag="tag2"), Tag(tag="tag3")]
async with AsyncSession(engine) as session:
async with session.begin():
session.add_all(users)
session.add_all(tags)
for user in users:
await session.refresh(user)
for tag in tags:
await session.refresh(tag)
for user in users:
for i in range(3, user.id - 1, -1):
await session.execute(user_tag.insert().values(user_id=user.id, tag_id=i))
await session.commit()
for user in users:
await session.refresh(user)
for tag in tags:
await session.refresh(tag)
tags = await greenlet_spawn(users[0].tags)
print(tags)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I run the program, it crashes with:
File "C:\Sources\asyncSQLAl test\main.py", line 48, in <module>
loop.run_until_complete(main())
File "C:\Users\Stanislav\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line
642, in run_until_complete
return future.result()
File "C:\Sources\asyncSQLAl test\main.py", line 41, in main
tags = await greenlet_spawn(await users[0].tags)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\orm\attributes.py", line 480, in __get__
return self.impl.get(state, dict_)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\orm\attributes.py", line 931, in get
value = self.callable_(state, passive)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\orm\strategies.py", line 879, in _load_for_state
return self._emit_lazyload(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\orm\strategies.py", line 1036,
in _emit_lazyload
result = session.execute(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\orm\session.py", line 1689, in
execute
result = conn._execute_20(statement, params or {}, execution_options)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1582, in
_execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\sql\lambdas.py", line 481, in _execute_on_connection
return connection._execute_clauseelement(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1451, in
_execute_clauseelement
ret = self._execute_context(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1813, in
_execute_context
self._handle_dbapi_exception(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1998, in
_handle_dbapi_exception
util.raise_(exc_info[1], with_traceback=exc_info[2])
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\util\compat.py", line 207, in raise_
raise exception
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1770, in
_execute_context
self.dialect.do_execute(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\engine\default.py", line 717, in do_execute
cursor.execute(statement, parameters)
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\dialects\postgresql\asyncpg.py", line 449, in execute
self._adapt_connection.await_(
File "C:\Sources\asyncSQLAl test\venv\lib\site-packages\sqlalchemy\util\_concurrency_py3k.py", line 60, in await_only
raise exc.MissingGreenlet(
sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: http://sqlalche.me/e/14/xd2s)
sys:1: RuntimeWarning: coroutine 'AsyncAdapt_asyncpg_cursor._prepare_and_execute' was never awaited
I don't quite understand how greenlet_spawn works here and where it should be used in this example.
For example, same program, but in sync style
from sqlalchemy import Column, Table, ForeignKey
from sqlalchemy.orm import declarative_base, relationship, sessionmaker
from sqlalchemy import create_engine
from sqlalchemy.dialects.postgresql import VARCHAR, INTEGER
Base = declarative_base()
user_tag = Table('user_tag', Base.metadata,
Column('user_id', INTEGER, ForeignKey('users.id')),
Column('tag_id', INTEGER, ForeignKey('tags.id'))
)
class User(Base):
__tablename__ = 'users'
id = Column(INTEGER, primary_key=True)
name = Column(VARCHAR(32), nullable=False, unique=True)
tags = relationship("Tag",
secondary=user_tag,
back_populates="users")
class Tag(Base):
__tablename__ = 'tags'
id = Column(INTEGER, primary_key=True)
tag = Column(VARCHAR(255), nullable=False, unique=True)
users = relationship("User",
secondary=user_tag,
back_populates="tags")
def __str__(self):
return self.tag
def main():
engine = create_engine(
"postgresql+psycopg2://postgres:pgs12345#localhost/test",
echo=False,
)
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()
users = [User(name="p1"), User(name="p2"), User(name="p3")]
tags = [Tag(tag="tag1"), Tag(tag="tag2"), Tag(tag="tag3")]
with session.begin():
session.add_all(users)
session.add_all(tags)
for user in users:
for i in range(3, user.id - 1, -1):
session.execute(user_tag.insert().values(
user_id=user.id, tag_id=i))
session.commit()
for tag in users[0].tags:
print(tag, end=" ")
main()
Gives me:
tag1 tag2 tag3

I've been stuck on this today too and I've narrowed it down to the fact that a lazyload is attempted, which GreenLet is not happy about. I wasn't sure whether this was just my lack of skill but I've found this article that details some of the common errors:
https://matt.sh/sqlalchemy-the-async-ening, where it's mentioned that this very issue will occur in this way. Furthermore, the docs go into detail about needing to avoid lazyloading: https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html.
My solution at the moment is to effectively prefetch the Child relation upon the initial query of the Parent object and then manipulate it from there. Whether this is a true bug, in the sense that it should work in async when it already works in sync or simply a limitation of the async method, I've no idea.
Edit 06/08/21, here is how I am prefetching relationships:
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from . import models
async def get_parent_prefetch_children(db: AsyncSession, parent_id: int) -> models.Parent:
result = await db.execute(
sa.select(models.Parent).where(models.Parent.id == parent_id).options(
selectinload(models.Parent.children)
)
)
return result.scalar()
In your case, you call users[0].tags, which makes a lazyload and fails. In order for you to avoid this, you must refetch users with their tags eagerly loaded.

From their official doc for relationship, you can use lazy="joined" to ask the SQLAlchemy do prefetch for you on session.refresh(...) or session.execute(select(...)). This will do joined style eagerly load at the time you query for the object. Note that this may introduce performance issue to your application since sometime you are not going to access the foreign'd objects while the database will still do joined style query for you.
tags = relationship("Tag",
secondary=user_tag,
back_populates="users",
lazy="joined")
The results and SQL stmt looks like:
>>>tags = users[0].tags
>>>print(tags)
[<__main__.Tag object at 0x10bc68610>, <__main__.Tag object at 0x10bc41d00>, <__main__.Tag object at 0x10bc12b80>]
>>>from sqlalchemy.future import select
>>>print(select(User))
FROM users LEFT OUTER JOIN (user_tag AS user_tag_1 JOIN tags AS tags_1 ON tags_1.id = user_tag_1.tag_id) ON users.id = user_tag_1.user_id

session is already flushing when I use #observes

I'm building a flask app and I want to use sqlalchemy observers to update a shipment status once all products inside that shipment become available.
Here's my datamodel:
from app import db
from sqlalchemy_utils import observes
class Shipment(db.Model):
__tablename__ = 'shipment'
id = db.Column(db.Integer, primary_key=True)
products = db.relationship('Product', backref='shipment', lazy='dynamic')
all_products_ready = db.Column(db.Boolean)
#observes('products')
def product_observer(self, products):
for p in self.products:
if p.status != 'ready':
self.all_products_ready = False
return False
self.all_products_ready = True
return True
class Product(db.Model):
__tablename__ = 'product'
id = db.Column(db.Integer, primary_key=True)
shipment_id = db.Column(db.Integer, db.ForeignKey('shipment.id'))
status = db.Column(db.String(120), index=True)
And here is some code I run to test it:
shipment = models.Shipment(products=[models.Product(status='ready'), models.Product(status='not_ready')])
db.session.add(shipment)
db.session.commit()
print(shipment.all_products_ready)
When I run this code I get an InvalidRequestError: Session is already flushing.
Here is the stack trace:
Traceback (most recent call last):
File "test.py", line 5, in <module>
db.session.commit()
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\scoping.py",
line 150, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 788, in commit
self.transaction.commit()
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 384, in commit
self._prepare_impl()
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 364, in _prepare_impl
self.session.flush()
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 1985, in flush
self._flush(objects)
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 2012, in _flush
self.dispatch.before_flush(self, flush_context, objects)
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\event\attr.py", l
ine 221, in __call__
fn(*args, **kw)
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy_utils\observer.py
", line 272, in invoke_callbacks
for (root_obj, func, objects) in args:
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy_utils\observer.py
", line 252, in gather_callback_args
lambda obj: obj not in session.deleted
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy_utils\functions\o
rm.py", line 741, in getdotattr
last = [v for v in last if condition(v)]
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\dynamic.py",
line 245, in __iter__
sess = self.session
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\dynamic.py",
line 237, in session
sess.flush()
File "U:\dev\observertest\flask\lib\site-packages\sqlalchemy\orm\session.py",
line 1979, in flush
raise sa_exc.InvalidRequestError("Session is already flushing")
sqlalchemy.exc.InvalidRequestError: Session is already flushing
How can I use my models without getting this error?

I'm not completely sure why, but I think loading the relations using dynamic is causing problems here. In this line:
products = db.relationship('Product', backref='shipment', lazy='dynamic')
you need to change the lazy parameter to select instead of dynamic (or you can take out the lazy parameter alltogether as select is its default).
See the sqlalchemy reference for all the available options.

Adding an object to a one-to-many relationship in Pyramid, SQLAlchemy

I have set up a simple system for adding comments to items in my database using a one-to-many relation (one item can have many comments), but I can't seem to get it to work. It should be simple, I know. It's probably something simple that I've overlooked. I would very much appriciate a second set of eyes on this. Can anyone help?
I have a model defined with SQLAlchemy that looks like this:
class Item(Base):
__tablename__ = 'items' # Parent
id = Column(Integer, primary_key=True)
comments = relationship("Comment", backref='items')
class Comment(Base):
__tablename__ = 'comments' # Child
id = Column(Integer, primary_key=True)
comment_text = Column(Text, nullable=False)
item_id = Column(Integer, ForeignKey('items.id'), nullable=False)
def __init__(self, comment_text, item_id):
self.comment_text = comment_text
self.item_id = item_id
In my view, I do some work, then try to add a comment object:
item = DBSession.query(Item).filter(Item.id == item_id).first()
try:
print('Item id:', item.id, 'Comment text:', comment)
print('Item Comments:', item.comments)
cm = Comment(comment_text=comment,
item_id=item.id)
print('a')
item.comments.append(cm)
#DBSession.add(cm)
print('b')
DBSession.commit()
except:
DBSession.rollback()
print('c')
Note that I tried both item.comments.append(cm) and DBSession.add(cm) with identical results. That's why one of the two is commented out in the above code block. I also tried item.comments.append(Comment(...)) with identical results.
Now, when I try to add a comment, I get a stacktrace, culminating in:
sqlalchemy.exc.ResourceClosedError: This transaction is closed
The whole trace, including the debug prints, look like this:
Item id: 1 Comment text: test
Item Comments: []
a
c
Traceback (most recent call last):
File "C:\Python33\lib\wsgiref\handlers.py", line 137, in run
self.result = application(self.environ, self.start_response)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid\router.py", line 251, in __call__
response = self.invoke_subrequest(request, use_tweens=True)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid\router.py", line 227, in invoke_subrequest
response = handle_request(request)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid\tweens.py", line 21, in excview_tween
response = handler(request)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid_tm\__init__.py", line 82, in tm_tween
reraise(*exc_info)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid_tm\compat.py", line 13, in reraise
raise value
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\pyramid_tm\__init__.py", line 70, in tm_tween
manager.commit()
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_manager.py", line 111, in commit
return self.get().commit()
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_transaction.py", line 280, in commit
reraise(t, v, tb)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_compat.py", line 55, in reraise
raise value
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_transaction.py", line 271, in commit
self._commitResources()
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_transaction.py", line 417, in _commitResources
reraise(t, v, tb)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_compat.py", line 55, in reraise
raise value
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\transaction\_transaction.py", line 394, in _commitResources
rm.tpc_vote(self)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\zope\sqlalchemy\datamanager.py", line 100, in tpc_vote
self.tx.commit()
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\sqlalchemy\orm\session.py", line 352, in commit
self._assert_active(prepared_ok=True)
File "C:\Users\[user]\PYTHON~1.5\lib\site-packages\sqlalchemy\orm\session.py", line 203, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg)
sqlalchemy.exc.ResourceClosedError: This transaction is closed

Well, turns out the problem was a few corrupt files, not a programming error. Sigh. Well, problem solved. :)