Related
I'm trying to be able to dynamically sort a query by a given column name and have nulls be placed at the end of the list. It works fine when using desc ordering, but raises a ProgrammingError when trying to use nulls_last with asc order. It appears this error is being raised as the result of an InvalidArgument error (see Traces below).
Dependencies
sqlalchemy = "~1.4.37"
sqlalchemy-spanner = "~1.2.0"
We're using Google Cloud Spanner for our database.
Original Query
Generating Method
#classmethod
def get_my_models(cls, sort_by: str, order_by: str) -> List["MyModel"]:
with read_session_scope(get_engine()) as session:
sort_attr = getattr(MyModel, sort_by)
query = (
session.query(MyModel)
.filter(func.coalesce(MyModel.is_deleted, False).is_(False))
.order_by(nulls_last(getattr(sort_attr, order_by)()))
)
return [MyModel.from_orm(model) for model in query.all()]
Translated Query
When called with sort_by="my_column" and order_by="desc"
SELECT ... FROM my_model WHERE coalesce(my_model.is_deleted, false) IS false ORDER BY
my_model.my_column DESC NULLS LAST
The above works fine. So as long as I want to order by descending, I'm good. Unfortunately, that's not the requirement.
When called with sort_by="my_column" and order_by="asc"
SELECT ... FROM my_model WHERE coalesce(my_model.is_deleted, false) IS false ORDER BY
my_model.my_column ASC NULLS LAST
it looks great, but it raises:
sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError)
The error is raised in the last line of the query method during query.all()
Traces
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1712, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1631, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 332, in _execute_on_connection
return connection._execute_clauseelement(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1498, in _execute_clauseelement
ret = self._execute_context(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1862, in _execute_context
self._handle_dbapi_exception(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2043, in _handle_dbapi_exception
util.raise_(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1006, in do_execute
cursor.execute(statement, parameters)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/spanner_dbapi/cursor.py", line 286, in execute
raise ProgrammingError(getattr(e, "details", e))
sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError) []
self = <google.cloud.spanner_dbapi.cursor.Cursor object at 0xffffacfca220>
sql = 'SELECT ... coalesce(my_model.is_deleted, #a0) IS false ORDER BY my_model.my_column ASC NULLS LAST'
args = {'a0': False}
#check_not_closed
def execute(self, sql, args=None):
"""Prepares and executes a Spanner database operation.
:type sql: str
:param sql: A SQL query statement.
:type args: list
:param args: Additional parameters to supplement the SQL query.
"""
self._result_set = None
try:
if self.connection.read_only:
self._handle_DQL(sql, args or None)
return
class_ = parse_utils.classify_stmt(sql)
if class_ == parse_utils.STMT_DDL:
self._batch_DDLs(sql)
if self.connection.autocommit:
self.connection.run_prior_DDL_statements()
return
# For every other operation, we've got to ensure that
# any prior DDL statements were run.
# self._run_prior_DDL_statements()
self.connection.run_prior_DDL_statements()
if class_ == parse_utils.STMT_UPDATING:
sql = parse_utils.ensure_where_clause(sql)
if class_ != parse_utils.STMT_INSERT:
sql, args = sql_pyformat_args_to_spanner(sql, args or None)
if not self.connection.autocommit:
statement = Statement(
sql,
args,
get_param_types(args or None)
if class_ != parse_utils.STMT_INSERT
else {},
ResultsChecksum(),
class_ == parse_utils.STMT_INSERT,
)
(
self._result_set,
self._checksum,
) = self.connection.run_statement(statement)
while True:
try:
self._itr = PeekIterator(self._result_set)
break
except Aborted:
self.connection.retry_transaction()
return
if class_ == parse_utils.STMT_NON_UPDATING:
self._handle_DQL(sql, args or None)
elif class_ == parse_utils.STMT_INSERT:
_helpers.handle_insert(self.connection, sql, args or None)
else:
self.connection.database.run_in_transaction(
self._do_execute_update, sql, args or None
)
except (AlreadyExists, FailedPrecondition, OutOfRange) as e:
raise IntegrityError(getattr(e, "details", e))
except InvalidArgument as e:
> raise ProgrammingError(getattr(e, "details", e))
E sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError) []
Google Cloud Spanner databases using the standard GoogleSQL dialect always sort NULL first when the sort order is ascending, and NULL last when the sort order is descending. The SQL dialect allows you to specify ASC NULLS FIRST and DESC NULLS LAST, as those correspond with the default, but not to actually change the sort order.
A workaround for this would be to use a COALESCE(my_column, <large-value>) expression in your ORDER BY clause. The feasibility of this workaround depends a little on the data type of the column that you are sorting. If it is for example an INT64 column, you could just use the max value for INT64. Be aware though that such a workaround can make your query less efficient, as the ORDER BY clause will not be able to use any index that you might have defined on the column.
I have a strange issue in my Python project. It uses SQLAlchemy and Bakery to have prepared queries. I have a function that takes the connection (db), bakery, and an array of objects.
This function is called several times by an other function in a for loop and here is my issue (at least what I understand):
Let's assume that the first time it receives an array with two elements.
The next time it is called the function will also expect an array with two elements
import sqlalchemy as sa
def cpe_filter(db, bakery, iterable):
cpes = []
try:
query = bakery(lambda s: s.query(Cpe))
query += lambda y: y.filter(
sa.or_(*[
Cpe.cpe.like(sa.bindparam('cpe_{}'.format(i)))
for i, _ in enumerate(iterable)
])
)
query += lambda y: y.filter_by(active=sa.bindparam('active'))
cpes = query(db).params(active=True,
**{'cpe_{}'.format(i): e for i, e in enumerate(iterable)}) \
.all()
except NoResultFound:
log.info("Found no CPE matching list {}.".format(iterable))
If the next array is smaller than the previous, I get this kind of error (Pastebin):
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1151] SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd
FROM cpe
WHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1154] {'cpe_0': 'cpe:/o:sun:solaris', 'cpe_1': 'cpe:/a:tritreal:ted_cde', 'cpe_2': 'cpe:/o:hp:hp-ux', 'active': 1}
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1151] SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd
FROM cpe
WHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1154] {'cpe_0': 'cpe:/a:hp:dtmail', 'cpe_1': 'cpe:/a:university_of_washington:pine', 'cpe_2': 'cpe:/o:sco:unixware', 'active': 1}
[2018-12-17 16:35:16 - ERROR/scap.abc:66] An error has occurred during task execution.
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in _init_compiled
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in <listcomp>
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 547, in construct_params
% bindparam.key, code="cd3x")
sqlalchemy.exc.InvalidRequestError: A value is required for bind parameter 'cpe_2' (Background on this error at: http://sqlalche.me/e/cd3x)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/scap/project/scap/abc.py", line 64, in run
self(*args, **kwargs)
File "/root/scap/project/scap/tasks.py", line 362, in __call__
q.cve_insert_or_update(self.db, self.bakery, self.parse(name))
File "/root/scap/project/scap/queries.py", line 148, in cve_insert_or_update
cpes = list(cpe_filter(db, bakery, cpes))
File "/root/scap/project/scap/queries.py", line 68, in cpe_filter
**{'cpe_{}'.format(i): e for i, e in enumerate(products)}) \
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/ext/baked.py", line 457, in all
return list(self)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/ext/baked.py", line 364, in __iter__
return q._execute_and_instances(context)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3018, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1132, in _execute_context
None, None)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 265, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 248, in reraise
raise value.with_traceback(tb)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in _init_compiled
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in <listcomp>
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 547, in construct_params
% bindparam.key, code="cd3x")
sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) A value is required for bind parameter 'cpe_2' [SQL: 'SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd \nFROM cpe \nWHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s'] [parameters: [{'active': True, 'cpe_0': 'cpe:/a:university_of_washington:imap', 'cpe_1': 'cpe:/a:netscape:messaging_server'}]] (Background on this error at: http://sqlalche.me/e/cd3x)
As you can see, the function is called three times, the first two times it works without any issue (3 elements each time), and the third time it has only two elements and it expect a third element according to the error.
NB: The iterable can reach about 50 elements most of the time.
The problem stems from the observations 4. and 5. under "Synopsis" in the baked queries documentation:
In the above code, even though our application may call upon search_for_user() many times, and even though within each invocation we build up an entirely new BakedQuery object, all of the lambdas are only called once. Each lambda is never called a second time for as long as this query is cached in the bakery.
The caching is achieved by storing references to the lambda objects themselves in order to formulate a cache key; that is, the fact that the Python interpreter assigns an in-Python identity to these functions is what determines how to identify the query on successive runs. For those invocations of search_for_user() where the email parameter is specified, the callable lambda q: q.filter(User.email == bindparam('email')) will be part of the cache key that’s retrieved; when email is None, this callable is not part of the cache key.
If you inspect your cpe_filter() function using dis you'll note that the lambda-functions are constants and so keep their identity between calls. As explained in the referenced documentation, SQLAlchemy caches queries based on those identities and calls
query += lambda y: y.filter(
sa.or_(*[
Cpe.cpe.like(sa.bindparam('cpe_{}'.format(i)))
for i, _ in enumerate(iterable)
])
)
only once. In other words the placeholders will be set the first time you call cpe_filter(), based on iterable. They will be "reset" only when this query has been evicted from the cache.
The solution depends on your DBMS in use. For example Postgresql has the ANY array comparison that could be used:
query += lambda y: y.filter(Cpe.cpe.like(sa.any_(sa.bindparam('cpe'))))
and the parameter would be passed as
# This relies on Psycopg2's lists adaptation:
# http://initd.org/psycopg/docs/usage.html#lists-adaptation
cpes = query(db).params(active=True, cpe=list(iterable)).all()
On MS SQL Server you could perhaps create a full-text index and use CONTAINS:
query += lambda y: y.filter(func.contains(Cpe.cpe, sa.bindparam('cpe')))
The bind param cpe should pass the search condition, which must be formed from iterable:
search_cond = " OR ".join(iterable)
cpes = query(db).params(active=True, cpe=search_cond).all()
This of course requires that the items in iterable are valid full-text search terms.
Here's my models.py:
from django.db import models
class Location(models.Model):
location_name = models.CharField(max_length=100)
def __unicode__(self):
return self.location_name
class Menu(models.Model):
location = models.ManyToManyField(Location)
restaurant_name = models.CharField(max_length=200)
dish_category = models.CharField(max_length=100)
dish_name = models.CharField(max_length=200)
VNV_tag = models.CharField(max_length=100)
quantity = models.CharField(max_length=10, default='FULL')
price = models.CharField(max_length=5, default=0)
def __unicode__(self):
return self.restaurant_name
I am trying to populate my database from an excel file using xlrd python module.
Here's my populate_db.py script:
import os
import xlrd
from menusearch.models import Menu, Location
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'midnitepro.settings')
import django
django.setup()
def populate():
book = xlrd.open_workbook('/Users/Sahil/Desktop/MNFP1.xlsx')
sheet = book.sheet_by_index(0)
count1 = 0
while count1<sheet.nrows:
value_list = []
count2 = 0
while count2<sheet.ncols:
value_list.append(sheet.row_values(count1, start_colx=count2, end_colx=count2+1))
count2 +=1
add_menuitem(value_list)
count1 += 1
def add_menuitem(value_list):
split_query = []
m = Menu.objects.create()
value_list = [str(x[0]).encode('utf-8') for x in value_list if x]
m.restaurant_name = (value_list[0])
m.dish_category = (value_list[1])
m.dish_name = (value_list[2])
m.VNV_tag = (value_list[3])
m.quantity = (value_list[4])
m.price = (value_list[5])
# m.location.add(int(value_list[6]))
split_query = (value_list[6]).split(",")
for sq in split_query:
l = Location.objects.get_or_create(location_name=sq)
try:
m.location.add(l)
except ValueError:
print "Problem Adding Location to Menu."
m.save()
if __name__ == '__main__':
print "Starting population script..."
from menusearch.models import Menu, Location
populate()
Now when I run the populate_db.py script, the output on the terminal reads: "Problem Adding Location to Menu.". I see that the problem exists here in the populate_db script: m.location.add(l)
But this queryset is in line with the syntax given here: https://docs.djangoproject.com/en/1.8/topics/db/examples/many_to_many/
After removing the try/except, my traceback reads:
TypeError: int() argument must be a string or a number, not 'Location'
Please explain what is happening here.
And how to properly add a a value to the ManyToManyField via a queryset.
Possible duplicate: Django Error: TypeError: int() argument must be a string or a number, not 'BuildsTable'
But I would still like to know the answer in my case as I'm using a ManyToManyField.
Full traceback:
l= (<Location: Indiranagar>, False)
Inside Loop - sq= Indiranagar
Traceback (most recent call last):
File "populate_db.py", line 62, in <module>
populate()
File "populate_db.py", line 23, in populate
add_menuitem(value_list)
File "populate_db.py", line 50, in add_menuitem
m.location.add(l)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 973, in add
self._add_items(self.source_field_name, self.target_field_name, *objs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 1079, in _add_items
'%s__in' % target_field_name: new_ids,
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/query.py", line 679, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/query.py", line 697, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1304, in add_q
clause, require_inner = self._add_q(where_part, self.used_aliases)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1332, in _add_q
allow_joins=allow_joins, split_subq=split_subq,
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1194, in build_filter
lookups, value)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 1745, in get_lookup_constraint
root_constraint.add(lookup_class(targets[0].get_col(alias, sources[0]), value), AND)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/lookups.py", line 96, in __init__
self.rhs = self.get_prep_lookup()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/lookups.py", line 134, in get_prep_lookup
return self.lhs.output_field.get_prep_lookup(self.lookup_name, self.rhs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/__init__.py", line 729, in get_prep_lookup
return [self.get_prep_value(v) for v in value]
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/__init__.py", line 985, in get_prep_value
return int(value)
TypeError: int() argument must be a string or a number, not 'Location'.
To add a value to a ManyToManyField, you need to add the objects by passing them as arguments.
add(obj1, obj2, obj3, ...)
This will add the specified objects to the related object set.
You can call the add() like below:
menu_object.location.add(loc_obj1, loc_obj2, ...)
In your case, you need to do:
l = Location.objects.get_or_create(location_name=sq)[0]
You need to access the instance by index 0 as get_or_create returns a tuple (instance, created). Then pass this instance to the add() method.
Note:
Using add() with a many-to-many relationship will not call any .save() for each object, but rather create the relationships using .bulk_create().
I understand that Rahul Gupta has answered this already. But on a more theoretical level I think this is what happened when you use get_or_create here:
l = Location.objects.get_or_create(location_name=sq)
It returns a Tuple, and hence you have to call it's first element by modifying it to:
l = Location.objects.get_or_create(location_name=sq)[0]
Which returns only the index element of the tuple/dictionary!
Suppose a NDB(version 0.7 & 0.8) model with key which can be None:
class Test(model.Model):
k = model.KeyProperty()
value = model.IntegerProperty()
count = Test.query(Test.k == None).count() # error
#count = Test.query(Test.k != None).count() # error also
Only filtering key with None throws error.
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/ext/webapp/_webapp25.py", line 701, in __call__
handler.get(*groups)
File "/Users/reiot/Documents/Works/ndbtest/main.py", line 91, in get
count = Test.query(Test.k == None).count() # error
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 500, in __eq__
return self._comparison('=', value)
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 491, in _comparison
return FilterNode(self._name, op, self._datastore_type(value))
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 1138, in _datastore_type
return datastore_types.Key(value.urlsafe())
AttributeError: 'NoneType' object has no attribute 'urlsafe'
How can I query KeyProperty with None?
This looks like a bug in NDB. Can you file one on the issue tracker?
I want to filter some database objects by a concatenated string.
The normal SQL query would be:
SELECT concat(firstName, ' ', name) FROM person WHERE CONCAT(firstName, ' ', name) LIKE "a%";
In the model, I have created a manager called PersonObjects:
class PersonObjects(Manager):
attrs = {
'fullName': "CONCAT(firstName, ' ', name)"
}
def get_query_set(self):
return super(PersonObjects, self).get_query_set().extra(
select=self.attrs)
I also configured this in my model:
objects = managers.PersonObjects()
Now accessing fullName works for single objects:
>>> p = models.Person.objects.get(pk=4)
>>> p.fullName
u'Fred Borminski'
But it does not work in a filter:
>>> p = models.Person.objects.filter(fullName__startswith='Alexei')
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "/usr/lib/python2.7/site-packages/django/db/models/manager.py", line 141, in filter
return self.get_query_set().filter(*args, **kwargs)
File "/usr/lib/python2.7/site-packages/django/db/models/query.py", line 550, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/usr/lib/python2.7/site-packages/django/db/models/query.py", line 568, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/usr/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1128, in add_q
can_reuse=used_aliases)
File "/usr/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1026, in add_filter
negate=negate, process_extras=process_extras)
File "/usr/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1191, in setup_joins
"Choices are: %s" % (name, ", ".join(names)))
FieldError: Cannot resolve keyword 'fullName' into field. Choices are: firstName, gender, name, (...)
Is this a bug or a feature? How can I fix this?
Thanks.
It's not a bug. filter() only inspects model definitions, so it doesn't recognize fullName as a declared field (because it's not - it's an extra argument in a query).
You can add the fullName to WHERE using extra():
Person.objects.extra(where=["fullName LIKE %s"], params=["Alexei%"])
I solved this by implementing a custom Aggregate function.
In this case I needed to concatenate individual fields into a street address to be able to filter/search for matches.
The following aggregate function allows to specify a field and one or more others to perform a SQL CONCAT_WS.
Edit 3 Aug 2015:
A better implementation with details gleaned from https://stackoverflow.com/a/19529861/3230522. The previous implementation would fail if the queryset was used in a subquery. The table names are now correct, although I note that this just works for concatenation of columns from the same table.
from django.db.models import Aggregate
from django.db.models.sql.aggregates import Aggregate as SQLAggregate
class SqlAggregate(SQLAggregate):
sql_function = 'CONCAT_WS'
sql_template = u'%(function)s(" ", %(field)s, %(columns_to_concatenate)s)'
def as_sql(self, qn, connection):
self.extra['columns_to_concatenate'] = ', '.join(
['.'.join([qn(self.col[0]), qn(c.strip())]) for c in self.extra['with_columns'].split(',')])
return super(SqlAggregate, self).as_sql(qn, connection)
class Concatenate(Aggregate):
sql = SqlAggregate
def __init__(self, expression, **extra):
super(Concatenate, self).__init__(
expression,
**extra)
def add_to_query(self, query, alias, col, source, is_summary):
aggregate = self.sql(col,
source=source,
is_summary=is_summary,
**self.extra)
query.aggregates[alias] = aggregate
The proposed solution worked great with postgresql and JSONB fields in the code below. Only records that have the 'partner' key under the 'key' jsonb field are returned:
query_partner = "select key->>'partner' from accounting_subaccount " \
"where accounting_subaccount.id = subaccount_id and key ? 'partner'"
qs = queryset.extra(select={'partner': query_partner}, where=["key ? 'partner'"])