Foreign key in SQLAlchemy (flask) changing before insert into DB? - python

I am trying to use SQLAlchemy to insert some records into a database with a foreign key but the foreign key does not seem to be changing as I would expect.
I have a simple 1-M relationship between two objects, Account & Transaction. The Account objects have been inserted into the database fine but when I try to iterate through these accounts and add transaction objects to them, the foreign key for all of the transactions is the id of the last account being iterated through.
Can anyone help me figure out why this is? it has been driving me crazy!
My account and transaction objects:
class Account(db.Model):
number = db.Column(db.String(80), primary_key=True)
bsb = db.Column(db.String(80), primary_key=True)
name = db.Column(db.String(80))
description = db.Column(db.String(80))
class Transaction(db.Model):
id = db.Column(db.Integer, primary_key=True)
account_id = db.Column(db.String(80), db.ForeignKey('account.number'))
amount = db.Column(db.Float(precprecision=2))
Here is where I am iterating through the accounts in the DB and trying to add transaction objects to them:
def Sync_Transactions():
#The following definately returns two account objects from the DB
accounts = [db.session.query(Account).filter(Account.number == '999999999').first(), db.session.query(Account).filter(Account.number == '222222222').first()]
for acc in accounts:
#The following parses a CSV file for the account and returns transaction objecs as a list
transactions = myLib.ParseCsvFile('C:\\transactions_%s.csv' % (acc.number))
acc.transactions = transactions
db.session.merge(acc)
db.session.commit()
The above, if only 1 account is retrieved from db, works fine. As soon as I start iterating over multiple accounts all of the transactions get given the same foreign key (the key of the last account - 222222222 in the above case)
Here is where the issue is
def ParseCsvFile(self, fileLocation, existingTransactionList=[]):
with open(fileLocation, 'rb') as f:
reader = csv.DictReader(f,['Date','TransactionAmount','C','D','TransactionType','TransactionDescription','Balance'])
for row in reader:
if not row['TransactionDescription'] == '':
existingTransactionList.append(Transaction(
float(row['TransactionAmount'])
)
)
return existingTransactionList
For some reason having the parameter existingTransactionList causes the issue. If I change the above code to the following the problem goes away but I still don't understand why due to my lack of python knowledge I am guessing :)
def ParseCsvFile(self, fileLocation, existingTransactionList=[]):
existingTransactionList=[]
with open(fileLocation, 'rb') as f:
reader = csv.DictReader(f,['Date','TransactionAmount','C','D','TransactionType','TransactionDescription','Balance'])
for row in reader:
if not row['TransactionDescription'] == '':
existingTransactionList.append(Transaction(
float(row['TransactionAmount'])
)
)
return existingTransactionList
The reason I have the existingTransactionList variable as a parameter is because I will eventually want to pass in a list of existing transactions and only the unique ones will get returned by using something like the following:
transactionList = list(set(existingTransactionList))


The issue is that you are adding all of your transactions to the last account. Change:
def ParseCsvFile(self, fileLocation, existingTransactionList=[]):
to
def ParseCsvFile(self, fileLocation, existingTransactionList=None):
if existingTransactionList is None:
existingTransactionList = []
Why does this happen
Python only parses the declaration for a function once and all of the default arguments are bound to their values during this compilation stage. That means that instead of every invocation of ParseCsvFile being given a new list instead every call to ParseCsvFile uses the same list.
To see what is going on, try the following on the command line:
def test_mutable_defaults(value, x=[], y={}):
print("x's memory id this run is", id(x))
print("Contents of x", x)
print("y's memory id this run is", id(y))
print("Contents of y", y)
x.append(value)
y[value] = value
return x, y
then use this function:
test_mutable_defaults(1)
test_mutable_defaults(2)
test_mutable_defaults(3)
If Python re-evaluated x and y on each call you would see different memory IDs for each call and x would be [1], then [2], then [3]. Because Python only evaluates x and y when it first compiles test_mutable_defaults you will see the same memory ID for x each call (and likewise for y), and x will be [1], then [1, 2] and then [1, 2, 3].

Related

How to merge/overwrite identical rows without error sqlalchemy

I have a function that checks the table for duplicate rows and overwrites the previous ones.
It has worked for me in the past when moving the id to another row but now this error occurs:
raise orm_exc.FlushError(
sqlalchemy.orm.exc.FlushError: New instance <Usage at 0x7fb3f2a035e0>
with identity key (<class 'blueprints.site.usage.Usage'>, (859,), None)
conflicts with persistent instance <Usage at 0x7fb3d896ee50>
And when I assign a new id to the updated row there's no error. How can I reassign the old id to the new row?
def save_as_single_row(self):
"""
Save usage as a single row in usage table.
"""
query = Usage.query.filter_by(site_id=self.site_id, invoice_num=self.invoice_num,
supply_charge=self.supply_charge)
this_usage = query.all()
if not this_usage or (len(this_usage) == 1 and this_usage[0] == self):
print(self.start_date, 'self')
self.save()
print(self.start_date)
return self
print('Usage already exists', this_usage[-1].__dict__)
self.id = this_usage[-1].id
self.credit = this_usage[-1].credit
self.awaiting_bill_combine = this_usage[-1].awaiting_bill_combine
self.awaiting_site_combine = this_usage[-1].awaiting_site_combine
query.delete()
db.session.add(self)
db.session.commit()
return self

Looks like the answer was much simpler than I thought, I removed db.session.add(self) and added:
new_obj = db.session.merge(this_usage[-1])
db.session.commit()
It seems like this merges the values of the most recent row with the id of the original row.
For anyone else wondering - here is some more info.
EDIT:
session.merge() stopped playing nicely for multiple rows so I stopped using db.session.add(self) and the changes committed correctly.

Flask Babel convert to original language when insert to database

I have a set of SelectField with Flask-WTF and I convert the default language with Flask-Babel.
Here is the snippet of my code:
from flask_babel import _, lazy_gettext as _l
class PaymentStatus(enum.Enum):
PENDING = _l('PENDING')
COMPLETED = _l('COMPLETED')
EXPIRED = _l('EXPIRED')
def __str__(self):
return '{}'.format(self.value)
payment_status = [(str(_l(y)), y) for y in (PaymentStatus)]
class PaymentForm(FlaskForm):
status_of_payment = SelectField(_l('Payment Status'), choices=payment_status)
# ...
# ...
And here is my model look like:
class Payment(db.Model):
__tablename__ = 'payment'
id = db.Column(db.Integer, primary_key=True)
status_of_payment = db.Column(db.Enum(PaymentStatus, name='status_of_payment'))
# ...
# ...
And when I try to insert the value from the Flask-WTF form to my database, I got some error.
Here is the snippet how I insert it to database:
if form.validate_on_submit():
payment = Payment(
# payment_status=form.status_of_payment.data
payment_status=PaymentStatus.PENDING.value
# ...
# ...
)
The value of the enum PENDING also converted to the language on preferred language on my browser, so I got this error message:
sqlalchemy.exc.StatementError: (builtins.LookupError) "MENGUNGGU" is
not among the defined enum values
for more information: "MENGUNGGU" = is Indonesian language for "PENDING" in English.
So the problem here is, when I insert the SelectField value, it also converts the language to my preferred browser language, and my database which is PostgreSQL block it, because I don't define the value on my enum type.
So, the point of my question is, can we excluded the i18n & l10n value from Flask-Babel when we want to insert the value to a database..?, or what should I do to face this..?

You should swap values in your choices because the first element in the tuple is the actual value that will be submitted and the second one is the presentation:
payment_status = [(y.name, _l(str(y.value))) for y in PaymentStatus]
Thanks to this you'll have translated names and proper values submitted.
Enum's name should be stored in your database instead of the value.

How to update object returned in query

So I'm a flask/sqlalchemy newbie but this seems like it should be a pretty simple. Yet for the life of me I can't get it to work and I can't find any documentation for this anywhere online. I have a somewhat complex query I run that returns me a list of database objects.
items = db.session.query(X, func.count(Y.x_id).label('total')).filter(X.size >= size).outerjoin(Y, X.x_id == Y.x_id).group_by(X.x_id).order_by('total ASC')\
.limit(20).all()
after I get this list of items I want to loop through the list and for each item update some property on it.
for it in items:
it.some_property = 'xyz'
db.session.commit()
However what's happening is that I'm getting an error
it.some_property = 'xyz'
AttributeError: 'result' object has no attribute 'some_property'
I'm not crazy. I'm positive that the property does exist on model X which is subclassed from db.Model. Something about the query is preventing me from accessing the attributes even though I can clearly see they exist in the debugger. Any help would be appreciated.
class X(db.Model):
x_id = db.Column(db.Integer, primary_key=True)
size = db.Column(db.Integer, nullable=False)
oords = db.relationship('Oords', lazy=True, backref=db.backref('x', lazy='joined'))
def __init__(self, capacity):
self.size = size

Given your example your result objects do not have the attribute some_property, just like the exception says. (Neither do model X objects, but I hope that's just an error in the example.)
They have the explicitly labeled total as second column and the model X instance as the first column. If you mean to access a property of the X instance, access that first from the result row, either using index, or the implicit label X:
items = db.session.query(X, func.count(Y.x_id).label('total')).\
filter(X.size >= size).\
outerjoin(Y, X.x_id == Y.x_id).\
group_by(X.x_id).\
order_by('total ASC').\
limit(20).\
all()
# Unpack a result object
for x, total in items:
x.some_property = 'xyz'
# Please commit after *all* the changes.
db.session.commit()
As noted in the other answer you could use bulk operations as well, though your limit(20) will make that a lot more challenging.

You should use the update function.
Like that:
from sqlalchemy import update
stmt = update(users).where(users.c.id==5).\
values(name='user #5')
Or :
session = self.db.get_session()
session.query(Organisation).filter_by(id_organisation = organisation.id_organisation).\
update(
{
"name" : organisation.name,
"type" : organisation.type,
}, synchronize_session = False)
session.commit();
session.close()
The sqlAlchemy doc : http://docs.sqlalchemy.org/en/latest/core/dml.html

How do I order by date when using ReferenceProperty?

I have a simple one-to-many structure like this:
class User(db.Model):
userEmail = db.StringProperty()
class Comment(db.Model):
user = db.ReferenceProperty(User, collection_name="comments")
comment = db.StringProperty()
date = db.DateTimeProperty()
I fetch a user from by his email:
q = User.all() # prepare User table for querying
q.filter("userEmail =", "az#example.com") # apply filter, email lookup
results = q.fetch(1) # execute the query, apply limit 1
the_user = results[0] # the results is a list of objects, grab the first one
this_users_comments = the_user.comments # get the user's comments
How can I order the user's comments by date, and limit it to 10 comments?

You will want to use the key keyword argument of the built-in sorted function, and use the "date" property as the key:
import operator
sorted_comments = sorted(this_users_comments, key=operator.attrgetter("date"))
# The comments will probably be sorted with earlier comments at the front of the list
# If you want ten most recent, also add the following line:
# sorted_comments.reverse()
ten_comments = sorted_comments[:10]

That query fetches the user. You need to do another query for the comments:
this_users_comments.order('date').limit(10)
for comment in this_users_comments:
...

What is an efficient way of inserting thousands of records into an SQLite table using Django?

I have to insert 8000+ records into a SQLite database using Django's ORM. This operation needs to be run as a cronjob about once per minute.
At the moment I'm using a for loop to iterate through all the items and then insert them one by one.
Example:
for item in items:
entry = Entry(a1=item.a1, a2=item.a2)
entry.save()
What is an efficient way of doing this?
Edit: A little comparison between the two insertion methods.
Without commit_manually decorator (11245 records):
nox#noxdevel marinetraffic]$ time python manage.py insrec
real 1m50.288s
user 0m6.710s
sys 0m23.445s
Using commit_manually decorator (11245 records):
[nox#noxdevel marinetraffic]$ time python manage.py insrec
real 0m18.464s
user 0m5.433s
sys 0m10.163s
Note: The test script also does some other operations besides inserting into the database (downloads a ZIP file, extracts an XML file from the ZIP archive, parses the XML file) so the time needed for execution does not necessarily represent the time needed to insert the records.

You want to check out django.db.transaction.commit_manually.
http://docs.djangoproject.com/en/dev/topics/db/transactions/#django-db-transaction-commit-manually
So it would be something like:
from django.db import transaction
#transaction.commit_manually
def viewfunc(request):
...
for item in items:
entry = Entry(a1=item.a1, a2=item.a2)
entry.save()
transaction.commit()
Which will only commit once, instead at each save().
In django 1.3 context managers were introduced.
So now you can use transaction.commit_on_success() in a similar way:
from django.db import transaction
def viewfunc(request):
...
with transaction.commit_on_success():
for item in items:
entry = Entry(a1=item.a1, a2=item.a2)
entry.save()
In django 1.4, bulk_create was added, allowing you to create lists of your model objects and then commit them all at once.
NOTE the save method will not be called when using bulk create.
>>> Entry.objects.bulk_create([
... Entry(headline="Django 1.0 Released"),
... Entry(headline="Django 1.1 Announced"),
... Entry(headline="Breaking: Django is awesome")
... ])
In django 1.6, transaction.atomic was introduced, intended to replace now legacy functions commit_on_success and commit_manually.
from the django documentation on atomic:
atomic is usable both as a decorator:
from django.db import transaction
#transaction.atomic
def viewfunc(request):
# This code executes inside a transaction.
do_stuff()
and as a context manager:
from django.db import transaction
def viewfunc(request):
# This code executes in autocommit mode (Django's default).
do_stuff()
with transaction.atomic():
# This code executes inside a transaction.
do_more_stuff()

Bulk creation is available in Django 1.4:
https://django.readthedocs.io/en/1.4/ref/models/querysets.html#bulk-create

Have a look at this. It's meant for use out-of-the-box with MySQL only, but there are pointers on what to do for other databases.

You might be better off bulk-loading the items - prepare a file and use a bulk load tool. This will be vastly more efficient than 8000 individual inserts.

To answer the question particularly with regard to SQLite, as asked, while I have just now confirmed that bulk_create does provide a tremendous speedup there is a limitation with SQLite: "The default is to create all objects in one batch, except for SQLite where the default is such that at maximum 999 variables per query is used."
The quoted stuff is from the docs--- A-IV provided a link.
What I have to add is that this djangosnippets entry by alpar also seems to be working for me. It's a little wrapper that breaks the big batch that you want to process into smaller batches, managing the 999 variables limit.

You should check out DSE. I wrote DSE to solve these kinds of problems ( massive insert or updates ). Using the django orm is a dead-end, you got to do it in plain SQL and DSE takes care of much of that for you.
Thomas

def order(request):
if request.method=="GET":
cust_name = request.GET.get('cust_name', '')
cust_cont = request.GET.get('cust_cont', '')
pincode = request.GET.get('pincode', '')
city_name = request.GET.get('city_name', '')
state = request.GET.get('state', '')
contry = request.GET.get('contry', '')
gender = request.GET.get('gender', '')
paid_amt = request.GET.get('paid_amt', '')
due_amt = request.GET.get('due_amt', '')
order_date = request.GET.get('order_date', '')
print(order_date)
prod_name = request.GET.getlist('prod_name[]', '')
prod_qty = request.GET.getlist('prod_qty[]', '')
prod_price = request.GET.getlist('prod_price[]', '')
print(prod_name)
print(prod_qty)
print(prod_price)
# insert customer information into customer table
try:
# Insert Data into customer table
cust_tab = Customer(customer_name=cust_name, customer_contact=cust_cont, gender=gender, city_name=city_name, pincode=pincode, state_name=state, contry_name=contry)
cust_tab.save()
# Retrive Id from customer table
custo_id = Customer.objects.values_list('customer_id').last() #It is return
Tuple as result from Queryset
custo_id = int(custo_id[0]) #It is convert the Tuple in INT
# Insert Data into Order table
order_tab = Orders(order_date=order_date, paid_amt=paid_amt, due_amt=due_amt, customer_id=custo_id)
order_tab.save()
# Insert Data into Products table
# insert multiple data at a one time from djanog using while loop
i=0
while(i<len(prod_name)):
p_n = prod_name[i]
p_q = prod_qty[i]
p_p = prod_price[i]
# this is checking the variable, if variable is null so fill the varable value in database
if p_n != "" and p_q != "" and p_p != "":
prod_tab = Products(product_name=p_n, product_qty=p_q, product_price=p_p, customer_id=custo_id)
prod_tab.save()
i=i+1

I recommend using plain SQL (not ORM) you can insert multiple rows with a single insert:
insert into A select from B;
The select from B portion of your sql could be as complicated as you want it to get as long as the results match the columns in table A and there are no constraint conflicts.

def order(request):
if request.method=="GET":
# get the value from html page
cust_name = request.GET.get('cust_name', '')
cust_cont = request.GET.get('cust_cont', '')
pincode = request.GET.get('pincode', '')
city_name = request.GET.get('city_name', '')
state = request.GET.get('state', '')
contry = request.GET.get('contry', '')
gender = request.GET.get('gender', '')
paid_amt = request.GET.get('paid_amt', '')
due_amt = request.GET.get('due_amt', '')
order_date = request.GET.get('order_date', '')
prod_name = request.GET.getlist('prod_name[]', '')
prod_qty = request.GET.getlist('prod_qty[]', '')
prod_price = request.GET.getlist('prod_price[]', '')
# insert customer information into customer table
try:
# Insert Data into customer table
cust_tab = Customer(customer_name=cust_name, customer_contact=cust_cont, gender=gender, city_name=city_name, pincode=pincode, state_name=state, contry_name=contry)
cust_tab.save()
# Retrive Id from customer table
custo_id = Customer.objects.values_list('customer_id').last() #It is return Tuple as result from Queryset
custo_id = int(custo_id[0]) #It is convert the Tuple in INT
# Insert Data into Order table
order_tab = Orders(order_date=order_date, paid_amt=paid_amt, due_amt=due_amt, customer_id=custo_id)
order_tab.save()
# Insert Data into Products table
# insert multiple data at a one time from djanog using while loop
i=0
while(i<len(prod_name)):
p_n = prod_name[i]
p_q = prod_qty[i]
p_p = prod_price[i]
# this is checking the variable, if variable is null so fill the varable value in database
if p_n != "" and p_q != "" and p_p != "":
prod_tab = Products(product_name=p_n, product_qty=p_q, product_price=p_p, customer_id=custo_id)
prod_tab.save()
i=i+1
return HttpResponse('Your Record Has been Saved')
except Exception as e:
return HttpResponse(e)
return render(request, 'invoice_system/order.html')

Categories