Related
I'm trying to build a query to count occurrences from distinct months in my SQL DB, using annotate and extract for this purpose.
Here is the code used:
Model
class Results(model.Model):
trip = models.BooleanField(default=False)
created_on = models.DateTimeField(default=datetime.now)
Query
from django.db.models.functions import Extract, ExtractYear
from django.db.models import Count
res = Results.objects.annotate(month=ExtractMonth('created_on'))
.values('month').annotate(count=Count('month'))
OR
res = Results.objects.annotate(month=Extract('created_on','month'))
.values('month').annotate(count=Count('month'))
Both return:
Error
Traceback (most recent call last):
File "<console>", line 1, in <module>
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 244, in __repr__
data = list(self[:REPR_OUTPUT_SIZE + 1])
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 268, in __iter__
self._fetch_all()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 1186, in _fetch_all
self._result_cache = list(self._iterable_class(self))
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\query.py", line 106, in __iter__
for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size):
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1017, in results_iter
results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 1052, in execute_sql
sql, params = self.as_sql()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 449, in as_sql
extra_select, order_by, group_by = self.pre_sql_setup()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 50, in pre_sql_setup
self.setup_query()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 41, in setup_query
self.select, self.klass_info, self.annotation_col_map = self.get_select()
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 244, in get_select
sql, params = self.compile(col, select_format=True)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\sql\compiler.py", line 390, in compile
sql, params = node.as_sql(self, self.connection)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\aggregates.py", line 76, in as_sql
return super().as_sql(compiler, connection, **extra_context)
File "C:\Users\VIBRITO\Desktop\Projetos\webapp\venv\lib\site-packages\django\db\models\expressions.py", line 618, in as_sql
data['expressions'] = data['field'] = arg_joiner.join(sql_parts)
TypeError: sequence item 0: expected str instance, tuple found
Can anybody see what is wrong in my approach?
I've followed the documentation for this query.
I'm using Django 2.1.5 and Python 3.7.0 on Windows.
If you see the documentation of ExtractMonth it is postgreSQL only function, If you use a postgreSQL function with MySQL Database, of course its going to give error.
I would suggest using the raw query, or changing the database to postgres.
So to preface this: the goal is to split a sqlite database into three shards through code. I have a majority of the background of storing and accessing the databases, but seem to have run into an issue trying to query more than one of those shards in a row to find a specific row. Code more than likely speaks for itself, so please look at it below.
def get_db():
"""Opens a new database connection if there is none yet for the
current application context.
"""
top = _app_ctx_stack.top
if not hasattr(top, 'sqlite_db'):
top.sqlite_db = [ sqlite3.connect(DATABASE_1, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_2, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_3, detect_types=sqlite3.PARSE_DECLTYPES) ]
top.sqlite_db[0].row_factory = sqlite3.Row
top.sqlite_db[1].row_factory = sqlite3.Row
top.sqlite_db[2].row_factory = sqlite3.Row
return top.sqlite_db
def query_db(query, shard, args=(), one=False):
"""Queries the database and returns a list of dictionaries."""
db = get_db()
cur = db[shard].execute(query, args)
rv = cur.fetchall()
return (rv[0] if rv else None) if one else rv
def get_user_id(username):
"""Convenience method to look up the id for a username."""
rb = query_db('select user_id from user where username = ?', 1, [username], one=True)
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
return rv[0] if rv else None
#app.teardown_appcontext
def close_database(exception):
"""Closes the database again at the end of the request."""
top = _app_ctx_stack.top
if hasattr(top, 'sqlite_db'):
top.sqlite_db[0].close()
top.sqlite_db[1].close()
top.sqlite_db[2].close()
I tried to include everything that may be needed. get_user_id is where I'm encountering an issue; it seems that calling into query_db twice in arrow is causing the problem below (it seems that it doesn't matter what they're assigned to either, as it encounters the same problem either way).
TypeError: 'NoneType' object has no attribute '__getitem__'
If i remove either of the lines (whether it queries the right shard or not) it works. Whether there's an actual row in the database with that username shouldn't matter, since it is confirmed to return None when I remove on of the lines in get_user_id. The problem seems to arise from calling query_db twice in a row. This further leads me to believe that there's an issue with calling the db connection. Once again, I've tried to include everything that I believe to be relevant, but I can add anything else if this isn't enough.
EDIT 1:
Adding full traceback as requested.
Traceback (most recent call last):
File "/home/me/.local/bin/flask", line 9, in <module>
load_entry_point('Flask', 'console_scripts', 'flask')()
File "/home/me/Desktop/project/flask/flask/cli.py", line 881, in main
cli.main(args=args, prog_name=name)
File "/home/me/Desktop/project/flask/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/me/Desktop/project/flask/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 149, in dummy_command
dummy_db()
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 143, in dummy_db
rv = get_user_id('dbrewer5')
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 86, in get_user_id
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 75, in query_db
cur = db[shard].execute(query, args)
TypeError: 'NoneType' object has no attribute '__getitem__'
Basically, I need to build a function that will filter a query according to given dates and return a new query. I'm new to SQLAlchemy, I looked up similar questions but I still got the same error:
`Don't know how to literal-quote value datetime.datetime(2018, 1, 1, 8, 3, 1, 438278)`
Here's my code:
def filter_dates(query_obj, datecols, start = None, end = None):
if end is None:
end = datetime.datetime.now()
if start is None:
start = end - datetime.timedelta(weeks=12)
print("%s to %s" % (start, end))
for datecol in datecols:
print("Filtrando datas!")
query_obj = query_obj.filter(datecol >= start)
query_obj = query_obj.filter(datecol <= end)
ReevTable.print_query(query_obj)
return query_obj
datecols is an orm.attributes object. Suppose I have an Object called User with a Datetime attribute named created_at. This is the expected behaviour:
query = session.query(Company.name, Company.created_at, Company.number_of_employees, Company.email_bounce_rate)
query = filter_dates(query_obj=query, datecols = [Company.created_at, Company.email_events.created_at])
query.all()
Expected output is a table with Companies that were only created within the date range, and the bounce rate should only be calculated during that specified date range. This might seem weird, but I calculate a not just emails, but other kinds of interactions too, so I need to input a list of attributes instead of just a single one. This is why I need to separate this filtering with a method.
I've tried using pandas datetime and timedelta, the built-in python datetime module, and simple strings with pd.to_datetime, but without success. The same error gets raised everytime. My Company column is in DateTime, so I don't know what else to do.
class Company(Base)
created_at = Column(DateTime, nullable=False)
I'm completely new to SQLAlchemy, what am I doing wrong?
Full traceback:
`Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "reev-data-science/tables/reevtable.py", line 128, in import_data
self.print_query(query_obj)
File "reev-data-science/tables/reevtable.py", line 107, in print_query
print(bcolors.OKBLUE + bcolors.BOLD + str(query_obj.statement.compile(compile_kwargs={"literal_binds":True})) + bcolors.ENDC)
File "<string>", line 1, in <lambda>
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/elements.py", line 442, in compile
return self._compiler(dialect, bind=bind, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/elements.py", line 448, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 453, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 219, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 245, in process
return obj._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/annotation.py", line 80, in _compiler_dispatch
self, visitor, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1815, in visit_select
text, select, inner_columns, froms, byfrom, kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1899, in _compose_select_body
t = select._whereclause._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 829, in visit_clauselist
for c in clauselist.clauses)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 826, in <genexpr>
s for s in
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 829, in <genexpr>
for c in clauselist.clauses)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 829, in visit_clauselist
for c in clauselist.clauses)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 826, in <genexpr>
s for s in
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 829, in <genexpr>
for c in clauselist.clauses)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1080, in visit_binary
return self._generate_generic_binary(binary, opstring, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1113, in _generate_generic_binary
self, eager_grouping=eager_grouping, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1244, in visit_bindparam
bindparam, within_columns_clause=True, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1277, in render_literal_bindparam
return self.render_literal_value(value, bindparam.type)
File "/usr/local/lib/python3.5/dist-packages/sqlalchemy/sql/compiler.py", line 1295, in render_literal_value
"Don't know how to literal-quote value %r" % value)
NotImplementedError: Don't know how to literal-quote value datetime.datetime(2018, 1, 1, 9, 24, 46, 54634)`
The print_query() method:
`def print_query(query_obj):
print(bcolors.OKBLUE + bcolors.BOLD + str(query_obj.statement.compile(compile_kwargs={"literal_binds":True})) + bcolors.ENDC)`
Convert datatime object to str in your filter clause
def filter_dates(query_obj, datecols, start = None, end = None):
if end is None:
end = datetime.datetime.now()
if start is None:
start = end - datetime.timedelta(weeks=12)
print("%s to %s" % (start, end))
for datecol in datecols:
print("Filtrando datas!")
query_obj = query_obj.filter(datecol >= str(start))
query_obj = query_obj.filter(datecol <= str(end))
ReevTable.print_query(query_obj)
return query_obj
I had the same problem with the DateTime type. This error occurs if you want to compile your statement with only literal_binds: True argument and you have complex (not int or string) fields in query.
stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
Solved it with TypeDecorator.
Solution:
from sqlalchemy import insert, create_engine, MetaData, Table, Column, DateTime, TypeDecorator
from sqlalchemy.dialects import postgresql
import psycopg2
from datetime import datetime
engine = create_engine('postgresql+psycopg2://postgres:password#127.0.0.1/postgres')
conn = engine.connect()
class MyDateTimeType(TypeDecorator):
impl = DateTime
def process_literal_param(self, value, dialect):
return value.strftime("'%Y-%m-%d %H:%M:%S'")
meta = MetaData()
dates = Table('dates', meta, Column('created_at', MyDateTimeType(), nullable=False))
stmt = (
insert(dates).
values(created_at=datetime.now())
)
raw_sql = stmt.compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
conn.execute(raw_sql)
I am currently developing a game. This game store data in a sqlite database. I'm using dataset to manage the database, so I don't have to worry about sql queries. I have a method that access the database to update player info :
def updatePlayerInfo(channel, info): # Context at https://github.com/DuckHunt-discord/DuckHunt-Discord/blob/master/database.py#L33
table = getChannelTable(channel)
table.upsert(info, ["id_"])
# An UPSERT is a smart combination of insert and update.
# If rows with matching keys exist they will be updated, otherwise a new row is inserted in the table.
This function works fine for almost everything. Only one thing create an error : using munAP_ as a column name ! (storing only integers timestamps inside)
Some other columns work the same way, but aren't affected by a single bug !
Exception raised is the following :
Ignoring exception in on_message
Traceback (most recent call last):
File "/usr/local/lib/python3.4/dist-packages/discord/client.py", line 245, in _run_event
yield from getattr(self, event)(*args, **kwargs)
File "./main.py", line 1022, in on_message
if database.getStat(message.channel, message.author, "chargeurs",
File "/home/cloudbot/discord-bot/database.py", line 45, in setStat
updatePlayerInfo(channel, dict_)
File "/home/cloudbot/discord-bot/database.py", line 35, in updatePlayerInfo
table.upsert(info, ["id_"])
File "/usr/local/lib/python3.4/dist-packages/dataset/persistence/table.py", line 185, in upsert
row_count = self.update(row, keys, ensure=ensure, types=types)
File "/usr/local/lib/python3.4/dist-packages/dataset/persistence/table.py", line 154, in update
rp = self.database.executable.execute(stmt)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 1991, in execute
return connection.execute(statement, *multiparams, **params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/engine/base.py", line 1003, in _execute_clauseelement
inline=len(distilled_params) > 1)
File "<string>", line 1, in <lambda>
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 494, in compile
return self._compiler(dialect, bind=bind, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/elements.py", line 500, in _compiler
return dialect.statement_compiler(dialect, self, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 395, in __init__
Compiled.__init__(self, dialect, statement, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 190, in __init__
self.string = self.process(self.statement, **compile_kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 213, in process
return obj._compiler_dispatch(self, **kwargs)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/visitors.py", line 81, in _compiler_dispatch
return meth(self, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/compiler.py", line 1958, in visit_update
crud_params = crud._get_crud_params(self, update_stmt, **kw)
File "/usr/local/lib/python3.4/dist-packages/sqlalchemy/sql/crud.py", line 109, in _get_crud_params
(", ".join("%s" % c for c in check))
sqlalchemy.exc.CompileError: Unconsumed column names: munAP_
https://github.com/DuckHunt-discord/DuckHunt-Discord/issues/8
I already tried to change the column name (it was munAP before) but it changed nothing !
What else can I try ? I suspect that the problem is in my code, but maybe it's dataset fault ?
I have an application that has worked successfully in the past. Today however, it's started throwing an error when I try write to datastore. For example, I'm creating a new entity of this model
class EventInstallment(ndb.Model):
somekey = ndb.KeyProperty()
somename = ndb.StringProperty(default = "")
start_date = ndb.DateTimeProperty()
notes = ndb.StringProperty("")
moderator_approved = ndb.BooleanProperty(default = True)
added_by = ndb.KeyProperty()
created = ndb.DateTimeProperty(auto_now_add = True)
using this code
ins = somemodel()
ins.somename = "26-september-2016"
ins.somekey = the_key
ins.start_date = datetime.datetime.now()
ins.put()
and this exception gets thrown.
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/admin/__init__.py", line 363, in post
exec(compiled_code, globals())
File "<string>", line 8, in <module>
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3451, in _put
return self._put_async(**ctx_options).get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 383, in get_result
self.check_success()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
value = gen.throw(exc.__class__, exc, tb)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 824, in put
key = yield self._put_batcher.add(entity, options)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
value = gen.throw(exc.__class__, exc, tb)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 358, in _put_tasklet
keys = yield self._conn.async_put(options, datastore_entities)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 513, in _on_rpc_completion
result = rpc.get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 613, in get_result
return self.__get_result_hook(self)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1881, in __put_hook
self.check_rpc_success(rpc)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1373, in check_rpc_success
raise _ToDatastoreError(err)
BadRequestError: The property.name is the empty string.
Any idea what this issue might be? It looks like a change in GAE ndb - as it worked as recently as 1 month ago...
Shouldn't notes = ndb.StringProperty("") be: notes = ndb.StringProperty(default = "") ?