I have a strange issue in my Python project. It uses SQLAlchemy and Bakery to have prepared queries. I have a function that takes the connection (db), bakery, and an array of objects.
This function is called several times by an other function in a for loop and here is my issue (at least what I understand):
Let's assume that the first time it receives an array with two elements.
The next time it is called the function will also expect an array with two elements
import sqlalchemy as sa
def cpe_filter(db, bakery, iterable):
cpes = []
try:
query = bakery(lambda s: s.query(Cpe))
query += lambda y: y.filter(
sa.or_(*[
Cpe.cpe.like(sa.bindparam('cpe_{}'.format(i)))
for i, _ in enumerate(iterable)
])
)
query += lambda y: y.filter_by(active=sa.bindparam('active'))
cpes = query(db).params(active=True,
**{'cpe_{}'.format(i): e for i, e in enumerate(iterable)}) \
.all()
except NoResultFound:
log.info("Found no CPE matching list {}.".format(iterable))
If the next array is smaller than the previous, I get this kind of error (Pastebin):
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1151] SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd
FROM cpe
WHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1154] {'cpe_0': 'cpe:/o:sun:solaris', 'cpe_1': 'cpe:/a:tritreal:ted_cde', 'cpe_2': 'cpe:/o:hp:hp-ux', 'active': 1}
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1151] SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd
FROM cpe
WHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s
[2018-12-17 16:35:16 - INFO/sqlalchemy.engine.base.Engine:1154] {'cpe_0': 'cpe:/a:hp:dtmail', 'cpe_1': 'cpe:/a:university_of_washington:pine', 'cpe_2': 'cpe:/o:sco:unixware', 'active': 1}
[2018-12-17 16:35:16 - ERROR/scap.abc:66] An error has occurred during task execution.
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in _init_compiled
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in <listcomp>
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 547, in construct_params
% bindparam.key, code="cd3x")
sqlalchemy.exc.InvalidRequestError: A value is required for bind parameter 'cpe_2' (Background on this error at: http://sqlalche.me/e/cd3x)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/root/scap/project/scap/abc.py", line 64, in run
self(*args, **kwargs)
File "/root/scap/project/scap/tasks.py", line 362, in __call__
q.cve_insert_or_update(self.db, self.bakery, self.parse(name))
File "/root/scap/project/scap/queries.py", line 148, in cve_insert_or_update
cpes = list(cpe_filter(db, bakery, cpes))
File "/root/scap/project/scap/queries.py", line 68, in cpe_filter
**{'cpe_{}'.format(i): e for i, e in enumerate(products)}) \
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/ext/baked.py", line 457, in all
return list(self)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/ext/baked.py", line 364, in __iter__
return q._execute_and_instances(context)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/orm/query.py", line 3018, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1132, in _execute_context
None, None)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 265, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 248, in reraise
raise value.with_traceback(tb)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1127, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in _init_compiled
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 635, in <listcomp>
grp, m in enumerate(parameters)]
File "/root/.local/share/virtualenvs/scap-TS2Ah8Sl/lib/python3.6/site-packages/sqlalchemy/sql/compiler.py", line 547, in construct_params
% bindparam.key, code="cd3x")
sqlalchemy.exc.StatementError: (sqlalchemy.exc.InvalidRequestError) A value is required for bind parameter 'cpe_2' [SQL: 'SELECT cpe.id AS cpe_id, cpe.active AS cpe_active, cpe.date_created AS cpe_date_created, cpe.timestamp AS cpe_timestamp, cpe.cpe_part_id AS cpe_cpe_part_id, cpe.device_id AS cpe_device_id, cpe.cpe AS cpe_cpe, cpe.match_nvd AS cpe_match_nvd \nFROM cpe \nWHERE (cpe.cpe LIKE %(cpe_0)s OR cpe.cpe LIKE %(cpe_1)s OR cpe.cpe LIKE %(cpe_2)s) AND cpe.active = %(active)s'] [parameters: [{'active': True, 'cpe_0': 'cpe:/a:university_of_washington:imap', 'cpe_1': 'cpe:/a:netscape:messaging_server'}]] (Background on this error at: http://sqlalche.me/e/cd3x)
As you can see, the function is called three times, the first two times it works without any issue (3 elements each time), and the third time it has only two elements and it expect a third element according to the error.
NB: The iterable can reach about 50 elements most of the time.
The problem stems from the observations 4. and 5. under "Synopsis" in the baked queries documentation:
In the above code, even though our application may call upon search_for_user() many times, and even though within each invocation we build up an entirely new BakedQuery object, all of the lambdas are only called once. Each lambda is never called a second time for as long as this query is cached in the bakery.
The caching is achieved by storing references to the lambda objects themselves in order to formulate a cache key; that is, the fact that the Python interpreter assigns an in-Python identity to these functions is what determines how to identify the query on successive runs. For those invocations of search_for_user() where the email parameter is specified, the callable lambda q: q.filter(User.email == bindparam('email')) will be part of the cache key that’s retrieved; when email is None, this callable is not part of the cache key.
If you inspect your cpe_filter() function using dis you'll note that the lambda-functions are constants and so keep their identity between calls. As explained in the referenced documentation, SQLAlchemy caches queries based on those identities and calls
query += lambda y: y.filter(
sa.or_(*[
Cpe.cpe.like(sa.bindparam('cpe_{}'.format(i)))
for i, _ in enumerate(iterable)
])
)
only once. In other words the placeholders will be set the first time you call cpe_filter(), based on iterable. They will be "reset" only when this query has been evicted from the cache.
The solution depends on your DBMS in use. For example Postgresql has the ANY array comparison that could be used:
query += lambda y: y.filter(Cpe.cpe.like(sa.any_(sa.bindparam('cpe'))))
and the parameter would be passed as
# This relies on Psycopg2's lists adaptation:
# http://initd.org/psycopg/docs/usage.html#lists-adaptation
cpes = query(db).params(active=True, cpe=list(iterable)).all()
On MS SQL Server you could perhaps create a full-text index and use CONTAINS:
query += lambda y: y.filter(func.contains(Cpe.cpe, sa.bindparam('cpe')))
The bind param cpe should pass the search condition, which must be formed from iterable:
search_cond = " OR ".join(iterable)
cpes = query(db).params(active=True, cpe=search_cond).all()
This of course requires that the items in iterable are valid full-text search terms.
Related
I'm trying to be able to dynamically sort a query by a given column name and have nulls be placed at the end of the list. It works fine when using desc ordering, but raises a ProgrammingError when trying to use nulls_last with asc order. It appears this error is being raised as the result of an InvalidArgument error (see Traces below).
Dependencies
sqlalchemy = "~1.4.37"
sqlalchemy-spanner = "~1.2.0"
We're using Google Cloud Spanner for our database.
Original Query
Generating Method
#classmethod
def get_my_models(cls, sort_by: str, order_by: str) -> List["MyModel"]:
with read_session_scope(get_engine()) as session:
sort_attr = getattr(MyModel, sort_by)
query = (
session.query(MyModel)
.filter(func.coalesce(MyModel.is_deleted, False).is_(False))
.order_by(nulls_last(getattr(sort_attr, order_by)()))
)
return [MyModel.from_orm(model) for model in query.all()]
Translated Query
When called with sort_by="my_column" and order_by="desc"
SELECT ... FROM my_model WHERE coalesce(my_model.is_deleted, false) IS false ORDER BY
my_model.my_column DESC NULLS LAST
The above works fine. So as long as I want to order by descending, I'm good. Unfortunately, that's not the requirement.
When called with sort_by="my_column" and order_by="asc"
SELECT ... FROM my_model WHERE coalesce(my_model.is_deleted, false) IS false ORDER BY
my_model.my_column ASC NULLS LAST
it looks great, but it raises:
sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError)
The error is raised in the last line of the query method during query.all()
Traces
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1712, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1631, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/sql/elements.py", line 332, in _execute_on_connection
return connection._execute_clauseelement(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1498, in _execute_clauseelement
ret = self._execute_context(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1862, in _execute_context
self._handle_dbapi_exception(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 2043, in _handle_dbapi_exception
util.raise_(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "/opt/pysetup/.venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/sqlalchemy_spanner/sqlalchemy_spanner.py", line 1006, in do_execute
cursor.execute(statement, parameters)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/spanner_dbapi/cursor.py", line 70, in wrapper
return function(cursor, *args, **kwargs)
File "/opt/pysetup/.venv/lib/python3.8/site-packages/google/cloud/spanner_dbapi/cursor.py", line 286, in execute
raise ProgrammingError(getattr(e, "details", e))
sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError) []
self = <google.cloud.spanner_dbapi.cursor.Cursor object at 0xffffacfca220>
sql = 'SELECT ... coalesce(my_model.is_deleted, #a0) IS false ORDER BY my_model.my_column ASC NULLS LAST'
args = {'a0': False}
#check_not_closed
def execute(self, sql, args=None):
"""Prepares and executes a Spanner database operation.
:type sql: str
:param sql: A SQL query statement.
:type args: list
:param args: Additional parameters to supplement the SQL query.
"""
self._result_set = None
try:
if self.connection.read_only:
self._handle_DQL(sql, args or None)
return
class_ = parse_utils.classify_stmt(sql)
if class_ == parse_utils.STMT_DDL:
self._batch_DDLs(sql)
if self.connection.autocommit:
self.connection.run_prior_DDL_statements()
return
# For every other operation, we've got to ensure that
# any prior DDL statements were run.
# self._run_prior_DDL_statements()
self.connection.run_prior_DDL_statements()
if class_ == parse_utils.STMT_UPDATING:
sql = parse_utils.ensure_where_clause(sql)
if class_ != parse_utils.STMT_INSERT:
sql, args = sql_pyformat_args_to_spanner(sql, args or None)
if not self.connection.autocommit:
statement = Statement(
sql,
args,
get_param_types(args or None)
if class_ != parse_utils.STMT_INSERT
else {},
ResultsChecksum(),
class_ == parse_utils.STMT_INSERT,
)
(
self._result_set,
self._checksum,
) = self.connection.run_statement(statement)
while True:
try:
self._itr = PeekIterator(self._result_set)
break
except Aborted:
self.connection.retry_transaction()
return
if class_ == parse_utils.STMT_NON_UPDATING:
self._handle_DQL(sql, args or None)
elif class_ == parse_utils.STMT_INSERT:
_helpers.handle_insert(self.connection, sql, args or None)
else:
self.connection.database.run_in_transaction(
self._do_execute_update, sql, args or None
)
except (AlreadyExists, FailedPrecondition, OutOfRange) as e:
raise IntegrityError(getattr(e, "details", e))
except InvalidArgument as e:
> raise ProgrammingError(getattr(e, "details", e))
E sqlalchemy.exc.ProgrammingError: (google.cloud.spanner_dbapi.exceptions.ProgrammingError) []
Google Cloud Spanner databases using the standard GoogleSQL dialect always sort NULL first when the sort order is ascending, and NULL last when the sort order is descending. The SQL dialect allows you to specify ASC NULLS FIRST and DESC NULLS LAST, as those correspond with the default, but not to actually change the sort order.
A workaround for this would be to use a COALESCE(my_column, <large-value>) expression in your ORDER BY clause. The feasibility of this workaround depends a little on the data type of the column that you are sorting. If it is for example an INT64 column, you could just use the max value for INT64. Be aware though that such a workaround can make your query less efficient, as the ORDER BY clause will not be able to use any index that you might have defined on the column.
I was trying this tutorial given in the MistQL for a personal work but this below code is throwing exception as given below
import mistql
data="{\"foo\": \"bar\"}"
query = '#.foo'
results = mistql.query(query, data)
print(results)
Traceback (most recent call last): File "C:\Temp\sample.py", line 4,
in
purchaserEmails = mistql.query(query, data) File "C:\Python3.10.0\lib\site-packages\mistql\query.py", line 18, in query
result = execute_outer(ast, data) File "C:\Python3.10.0\lib\site-packages\mistql\execute.py", line 73, in
execute_outer
return execute(ast, build_initial_stack(data, builtins)) File "C:\Python3.10.0\lib\site-packages\typeguard_init_.py", line 1033,
in wrapper
retval = func(*args, **kwargs) File "C:\Python3.10.0\lib\site-packages\mistql\execute.py", line 60, in
execute
return execute_fncall(ast.fn, ast.args, stack) File "C:\Python3.10.0\lib\site-packages\typeguard_init_.py", line 1033,
in wrapper
retval = func(*args, **kwargs) File "C:\Python3.10.0\lib\site-packages\mistql\execute.py", line 28, in
execute_fncall
return function_definition(arguments, stack, execute) File "C:\Python3.10.0\lib\site-packages\mistql\builtins.py", line 37, in
wrapped
return fn(arguments, stack, exec) File "C:\Python3.10.0\lib\site-packages\mistql\builtins.py", line 168, in
dot
return _index_single(RuntimeValue.of(right.name), left) File "C:\Python3.10.0\lib\site-packages\mistql\builtins.py", line 295, in
_index_single
assert_type(index, RVT.Number) File "C:\Python3.10.0\lib\site-packages\mistql\runtime_value.py", line 344,
in assert_type
raise MistQLTypeError(f"Expected {expected_type}, got {value.type}") mistql.exceptions.MistQLTypeError: Expected
RuntimeValueType.Number, got RuntimeValueType.String
Strings passed as data into MistQL's query method correspond to MistQL string types, and thus the #.foo query is attempting to index the string "{\"foo\": \"bar\"}") using the "foo" value, leading to the error above.
MistQL is expecting native dictionaries and lists. The correspondence between Python data types and MistQL data types can be seen here
That being said, the error message is indeed very confusing and should be fixed post-haste! Issue here
So to preface this: the goal is to split a sqlite database into three shards through code. I have a majority of the background of storing and accessing the databases, but seem to have run into an issue trying to query more than one of those shards in a row to find a specific row. Code more than likely speaks for itself, so please look at it below.
def get_db():
"""Opens a new database connection if there is none yet for the
current application context.
"""
top = _app_ctx_stack.top
if not hasattr(top, 'sqlite_db'):
top.sqlite_db = [ sqlite3.connect(DATABASE_1, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_2, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_3, detect_types=sqlite3.PARSE_DECLTYPES) ]
top.sqlite_db[0].row_factory = sqlite3.Row
top.sqlite_db[1].row_factory = sqlite3.Row
top.sqlite_db[2].row_factory = sqlite3.Row
return top.sqlite_db
def query_db(query, shard, args=(), one=False):
"""Queries the database and returns a list of dictionaries."""
db = get_db()
cur = db[shard].execute(query, args)
rv = cur.fetchall()
return (rv[0] if rv else None) if one else rv
def get_user_id(username):
"""Convenience method to look up the id for a username."""
rb = query_db('select user_id from user where username = ?', 1, [username], one=True)
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
return rv[0] if rv else None
#app.teardown_appcontext
def close_database(exception):
"""Closes the database again at the end of the request."""
top = _app_ctx_stack.top
if hasattr(top, 'sqlite_db'):
top.sqlite_db[0].close()
top.sqlite_db[1].close()
top.sqlite_db[2].close()
I tried to include everything that may be needed. get_user_id is where I'm encountering an issue; it seems that calling into query_db twice in arrow is causing the problem below (it seems that it doesn't matter what they're assigned to either, as it encounters the same problem either way).
TypeError: 'NoneType' object has no attribute '__getitem__'
If i remove either of the lines (whether it queries the right shard or not) it works. Whether there's an actual row in the database with that username shouldn't matter, since it is confirmed to return None when I remove on of the lines in get_user_id. The problem seems to arise from calling query_db twice in a row. This further leads me to believe that there's an issue with calling the db connection. Once again, I've tried to include everything that I believe to be relevant, but I can add anything else if this isn't enough.
EDIT 1:
Adding full traceback as requested.
Traceback (most recent call last):
File "/home/me/.local/bin/flask", line 9, in <module>
load_entry_point('Flask', 'console_scripts', 'flask')()
File "/home/me/Desktop/project/flask/flask/cli.py", line 881, in main
cli.main(args=args, prog_name=name)
File "/home/me/Desktop/project/flask/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/me/Desktop/project/flask/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 149, in dummy_command
dummy_db()
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 143, in dummy_db
rv = get_user_id('dbrewer5')
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 86, in get_user_id
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 75, in query_db
cur = db[shard].execute(query, args)
TypeError: 'NoneType' object has no attribute '__getitem__'
I am currently developing a game. This game store data in a sqlite database. I'm using dataset to manage the database, so I don't have to worry about sql queries. I have a method that access the database to update player info :
def updatePlayerInfo(channel, info): # Context at https://github.com/DuckHunt-discord/DuckHunt-Discord/blob/master/database.py#L33
table = getChannelTable(channel)
table.upsert(info, ["id_"])
# An UPSERT is a smart combination of insert and update.
# If rows with matching keys exist they will be updated, otherwise a new row is inserted in the table.
This function works fine for almost everything. Only one thing create an error : using munAP_ as a column name ! (storing only integers timestamps inside)
Some other columns work the same way, but aren't affected by a single bug !
Exception raised is the following :
Ignoring exception in on_message
Traceback (most recent call last):
File "/usr/local/lib/python3.4/dist-packages/discord/client.py", line 245, in _run_event
yield from getattr(self, event)(*args, **kwargs)
File "./main.py", line 1022, in on_message
if database.getStat(message.channel, message.author, "chargeurs",
File "/home/cloudbot/discord-bot/database.py", line 45, in setStat
updatePlayerInfo(channel, dict_)
File "/home/cloudbot/discord-bot/database.py", line 35, in updatePlayerInfo
table.upsert(info, ["id_"])
File "/usr/local/lib/python3.4/dist-packages/dataset/persistence/table.py", line 185, in upsert
row_count = self.update(row, keys, ensure=ensure, types=types)
File "/usr/local/lib/python3.4/dist-packages/dataset/persistence/table.py", line 154, in update
rp = self.database.executable.execute(stmt)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 1991, in execute
return connection.execute(statement, *multiparams, **params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 1003, in _execute_clauseelement
inline=len(distilled_params) > 1)
File "<string>", line 1, in <lambda>
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 494, in compile
return self._compiler(dialect, bind=bind, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 500, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 395, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 190, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 1958, in visit_update
crud_params = crud._get_crud_params(self, update_stmt, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/crud.py", line 109, in _get_crud_params
(", ".join("%s" % c for c in check))
sqlalchemy.exc.CompileError: Unconsumed column names: munAP_
https://github.com/DuckHunt-discord/DuckHunt-Discord/issues/8
I already tried to change the column name (it was munAP before) but it changed nothing !
What else can I try ? I suspect that the problem is in my code, but maybe it's dataset fault ?
I'm trying to select specific columns from a table like this:
users = Table('users', metadata, autoload=True)
s = users.select([users.c.email])
results = s.execute()
print results
and I'm getting this error:
> Traceback (most recent call last): File "my_mailer.py", line 35, in
> <module>
> s = users.select([users.c.email]) File "/task/__pips__/sqlalchemy/sql/selectable.py", line 175, in select
> return Select([self], whereclause, **params) File "/task/__pips__/sqlalchemy/sql/selectable.py", line 2082, in __init__
> self._whereclause = _literal_as_text(whereclause) File "/task/__pips__/sqlalchemy/sql/elements.py", line 2745, in
> _literal_as_text
> "SQL expression object or string expected." sqlalchemy.exc.ArgumentError: SQL expression object or string
> expected.
So I tried this:
users = Table('users', metadata, autoload=True)
s = users.select('email')
results = s.execute()
print results
And got this in response:
> Traceback (most recent call last): File "my_mailer.py", line 36, in
> <module>
> results = s.execute() File "/task/__pips__/sqlalchemy/sql/base.py", line 124, in execute
> return e._execute_clauseelement(self, multiparams, params) File "/task/__pips__/sqlalchemy/engine/base.py", line 1605, in
> _execute_clauseelement
> return connection._execute_clauseelement(elem, multiparams, params) File "/task/__pips__/sqlalchemy/engine/base.py", line 761,
> in _execute_clauseelement
> compiled_sql, distilled_params File "/task/__pips__/sqlalchemy/engine/base.py", line 874, in
> _execute_context
> context) File "/task/__pips__/sqlalchemy/engine/base.py", line 1023, in _handle_dbapi_exception
> exc_info File "/task/__pips__/sqlalchemy/util/compat.py", line 185, in raise_from_cause
> reraise(type(exception), exception, tb=exc_tb) File "/task/__pips__/sqlalchemy/engine/base.py", line 867, in
> _execute_context
> context) File "/task/__pips__/sqlalchemy/engine/default.py", line 388, in do_execute
> cursor.execute(statement, parameters) sqlalchemy.exc.ProgrammingError: (ProgrammingError) argument of WHERE
> must be type boolean, not type character varying LINE 3: WHERE email
Sure enough, the first argument here is the 'whereclause', not 'columns' like everywhere else, this is reflected in the documentation:
This argument is not present on the form of select() available on
Table.
Question: how can I select only specific columns using select on the table? And generally, why on earth the columns argument is not available on select on the table? I can't understand why somebody made the decision to make this different than the standard select.
Use general purpose select instead of Table.select:
stmt = select([users.c.email])
result = conn.execute(stmt)
With metadata you can do this:
metadata = MetaData(bind=engine)
tblusers = metadata.tables['users']
tblproducts = metadata.tables['products']
# example 1: only columns you want.
data_users = tblusers.select().with_only_columns([tblusers.c.id, tblusers.c.name]).execute()
# example 2: w/ where & order_by
data_products = tblproducts.select().with_only_columns([tblproducts.c.id, tblproducts.c.price, tblproductos.c.description]).where(tblproducts.c.stock > 0).order_by(tblproducts.c.brand).execute()
Be well...
SqlAlchemy Docs