Suppose a NDB(version 0.7 & 0.8) model with key which can be None:
class Test(model.Model):
k = model.KeyProperty()
value = model.IntegerProperty()
count = Test.query(Test.k == None).count() # error
#count = Test.query(Test.k != None).count() # error also
Only filtering key with None throws error.
File "/Applications/GoogleAppEngineLauncher.app/Contents/Resources/GoogleAppEngine-default.bundle/Contents/Resources/google_appengine/google/appengine/ext/webapp/_webapp25.py", line 701, in __call__
handler.get(*groups)
File "/Users/reiot/Documents/Works/ndbtest/main.py", line 91, in get
count = Test.query(Test.k == None).count() # error
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 500, in __eq__
return self._comparison('=', value)
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 491, in _comparison
return FilterNode(self._name, op, self._datastore_type(value))
File "/Users/reiot/Documents/Works/ndbtest/ndb/model.py", line 1138, in _datastore_type
return datastore_types.Key(value.urlsafe())
AttributeError: 'NoneType' object has no attribute 'urlsafe'
How can I query KeyProperty with None?
This looks like a bug in NDB. Can you file one on the issue tracker?
Related
I have a scrapy script that works locally, but when I deploy it to Scrapinghub, it's giving all errors. Upon debugging, the error is coming from Yielding the item.
This is the error I get.
ERROR [scrapy.utils.signal] Error caught on signal handler: <bound method ?.item_scraped of <sh_scrapy.extension.HubstorageExtension object at 0x7fd39e6141d0>> Less
Traceback (most recent call last):
File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 150, in maybeDeferred
result = f(*args, **kw)
File "/usr/local/lib/python2.7/site-packages/pydispatch/robustapply.py", line 55, in robustApply
return receiver(*arguments, **named)
File "/usr/local/lib/python2.7/site-packages/sh_scrapy/extension.py", line 45, in item_scraped
item = self.exporter.export_item(item)
File "/usr/local/lib/python2.7/site-packages/scrapy/exporters.py", line 304, in export_item
result = dict(self._get_serialized_fields(item))
File "/usr/local/lib/python2.7/site-packages/scrapy/exporters.py", line 75, in _get_serialized_fields
value = self.serialize_field(field, field_name, item[field_name])
File "/usr/local/lib/python2.7/site-packages/scrapy/exporters.py", line 284, in serialize_field
return serializer(value)
File "/usr/local/lib/python2.7/site-packages/scrapy/exporters.py", line 290, in _serialize_value
return dict(self._serialize_dict(value))
File "/usr/local/lib/python2.7/site-packages/scrapy/exporters.py", line 300, in _serialize_dict
key = to_bytes(key) if self.binary else key
File "/usr/local/lib/python2.7/site-packages/scrapy/utils/python.py", line 117, in to_bytes
'object, got %s' % type(text).__name__)
TypeError: to_bytes must receive a unicode, str or bytes object, got int
It doesn't specify the field with issues, but by process of elimination, I came to realize it's this part of the code:
try:
item["media"] = {}
media_index = 0
media_content = response.xpath("//audio/source/#src").extract_first()
if media_content is not None:
item["media"][media_index] = {}
preview = item["media"][media_index]
preview["Media URL"] = media_content
preview["Media Type"] = "Audio"
media_index += 1
except IndexError:
print "Index error for media " + item["asset_url"]
I cleared some parts up to make it easier to tackle, but basically this part is the issue. Something it doesn't like about the item media.
I'm beginner in both Python and Scrapy. So sorry if this turns out to be silly basic Python mistake. Any idea?
EDIT: So after getting the answer from ThunderMind, the solution was to simply do str(media_index) for key
Yeah, right here:
item["media"][media_index] = {}
media_index is a mutable. and Keys can't be mutable.
Read Python dict, to know what should be used as keys.
So to preface this: the goal is to split a sqlite database into three shards through code. I have a majority of the background of storing and accessing the databases, but seem to have run into an issue trying to query more than one of those shards in a row to find a specific row. Code more than likely speaks for itself, so please look at it below.
def get_db():
"""Opens a new database connection if there is none yet for the
current application context.
"""
top = _app_ctx_stack.top
if not hasattr(top, 'sqlite_db'):
top.sqlite_db = [ sqlite3.connect(DATABASE_1, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_2, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_3, detect_types=sqlite3.PARSE_DECLTYPES) ]
top.sqlite_db[0].row_factory = sqlite3.Row
top.sqlite_db[1].row_factory = sqlite3.Row
top.sqlite_db[2].row_factory = sqlite3.Row
return top.sqlite_db
def query_db(query, shard, args=(), one=False):
"""Queries the database and returns a list of dictionaries."""
db = get_db()
cur = db[shard].execute(query, args)
rv = cur.fetchall()
return (rv[0] if rv else None) if one else rv
def get_user_id(username):
"""Convenience method to look up the id for a username."""
rb = query_db('select user_id from user where username = ?', 1, [username], one=True)
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
return rv[0] if rv else None
#app.teardown_appcontext
def close_database(exception):
"""Closes the database again at the end of the request."""
top = _app_ctx_stack.top
if hasattr(top, 'sqlite_db'):
top.sqlite_db[0].close()
top.sqlite_db[1].close()
top.sqlite_db[2].close()
I tried to include everything that may be needed. get_user_id is where I'm encountering an issue; it seems that calling into query_db twice in arrow is causing the problem below (it seems that it doesn't matter what they're assigned to either, as it encounters the same problem either way).
TypeError: 'NoneType' object has no attribute '__getitem__'
If i remove either of the lines (whether it queries the right shard or not) it works. Whether there's an actual row in the database with that username shouldn't matter, since it is confirmed to return None when I remove on of the lines in get_user_id. The problem seems to arise from calling query_db twice in a row. This further leads me to believe that there's an issue with calling the db connection. Once again, I've tried to include everything that I believe to be relevant, but I can add anything else if this isn't enough.
EDIT 1:
Adding full traceback as requested.
Traceback (most recent call last):
File "/home/me/.local/bin/flask", line 9, in <module>
load_entry_point('Flask', 'console_scripts', 'flask')()
File "/home/me/Desktop/project/flask/flask/cli.py", line 881, in main
cli.main(args=args, prog_name=name)
File "/home/me/Desktop/project/flask/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/me/Desktop/project/flask/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 149, in dummy_command
dummy_db()
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 143, in dummy_db
rv = get_user_id('dbrewer5')
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 86, in get_user_id
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 75, in query_db
cur = db[shard].execute(query, args)
TypeError: 'NoneType' object has no attribute '__getitem__'
I have an application that has worked successfully in the past. Today however, it's started throwing an error when I try write to datastore. For example, I'm creating a new entity of this model
class EventInstallment(ndb.Model):
somekey = ndb.KeyProperty()
somename = ndb.StringProperty(default = "")
start_date = ndb.DateTimeProperty()
notes = ndb.StringProperty("")
moderator_approved = ndb.BooleanProperty(default = True)
added_by = ndb.KeyProperty()
created = ndb.DateTimeProperty(auto_now_add = True)
using this code
ins = somemodel()
ins.somename = "26-september-2016"
ins.somekey = the_key
ins.start_date = datetime.datetime.now()
ins.put()
and this exception gets thrown.
Traceback (most recent call last):
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/admin/__init__.py", line 363, in post
exec(compiled_code, globals())
File "<string>", line 8, in <module>
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/model.py", line 3451, in _put
return self._put_async(**ctx_options).get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 383, in get_result
self.check_success()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
value = gen.throw(exc.__class__, exc, tb)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 824, in put
key = yield self._put_batcher.add(entity, options)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 427, in _help_tasklet_along
value = gen.throw(exc.__class__, exc, tb)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/context.py", line 358, in _put_tasklet
keys = yield self._conn.async_put(options, datastore_entities)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/ext/ndb/tasklets.py", line 513, in _on_rpc_completion
result = rpc.get_result()
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/api/apiproxy_stub_map.py", line 613, in get_result
return self.__get_result_hook(self)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1881, in __put_hook
self.check_rpc_success(rpc)
File "/base/data/home/runtimes/python27/python27_lib/versions/1/google/appengine/datastore/datastore_rpc.py", line 1373, in check_rpc_success
raise _ToDatastoreError(err)
BadRequestError: The property.name is the empty string.
Any idea what this issue might be? It looks like a change in GAE ndb - as it worked as recently as 1 month ago...
Shouldn't notes = ndb.StringProperty("") be: notes = ndb.StringProperty(default = "") ?
Here's my models.py:
from django.db import models
class Location(models.Model):
location_name = models.CharField(max_length=100)
def __unicode__(self):
return self.location_name
class Menu(models.Model):
location = models.ManyToManyField(Location)
restaurant_name = models.CharField(max_length=200)
dish_category = models.CharField(max_length=100)
dish_name = models.CharField(max_length=200)
VNV_tag = models.CharField(max_length=100)
quantity = models.CharField(max_length=10, default='FULL')
price = models.CharField(max_length=5, default=0)
def __unicode__(self):
return self.restaurant_name
I am trying to populate my database from an excel file using xlrd python module.
Here's my populate_db.py script:
import os
import xlrd
from menusearch.models import Menu, Location
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'midnitepro.settings')
import django
django.setup()
def populate():
book = xlrd.open_workbook('/Users/Sahil/Desktop/MNFP1.xlsx')
sheet = book.sheet_by_index(0)
count1 = 0
while count1<sheet.nrows:
value_list = []
count2 = 0
while count2<sheet.ncols:
value_list.append(sheet.row_values(count1, start_colx=count2, end_colx=count2+1))
count2 +=1
add_menuitem(value_list)
count1 += 1
def add_menuitem(value_list):
split_query = []
m = Menu.objects.create()
value_list = [str(x[0]).encode('utf-8') for x in value_list if x]
m.restaurant_name = (value_list[0])
m.dish_category = (value_list[1])
m.dish_name = (value_list[2])
m.VNV_tag = (value_list[3])
m.quantity = (value_list[4])
m.price = (value_list[5])
# m.location.add(int(value_list[6]))
split_query = (value_list[6]).split(",")
for sq in split_query:
l = Location.objects.get_or_create(location_name=sq)
try:
m.location.add(l)
except ValueError:
print "Problem Adding Location to Menu."
m.save()
if __name__ == '__main__':
print "Starting population script..."
from menusearch.models import Menu, Location
populate()
Now when I run the populate_db.py script, the output on the terminal reads: "Problem Adding Location to Menu.". I see that the problem exists here in the populate_db script: m.location.add(l)
But this queryset is in line with the syntax given here: https://docs.djangoproject.com/en/1.8/topics/db/examples/many_to_many/
After removing the try/except, my traceback reads:
TypeError: int() argument must be a string or a number, not 'Location'
Please explain what is happening here.
And how to properly add a a value to the ManyToManyField via a queryset.
Possible duplicate: Django Error: TypeError: int() argument must be a string or a number, not 'BuildsTable'
But I would still like to know the answer in my case as I'm using a ManyToManyField.
Full traceback:
l= (<Location: Indiranagar>, False)
Inside Loop - sq= Indiranagar
Traceback (most recent call last):
File "populate_db.py", line 62, in <module>
populate()
File "populate_db.py", line 23, in populate
add_menuitem(value_list)
File "populate_db.py", line 50, in add_menuitem
m.location.add(l)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 973, in add
self._add_items(self.source_field_name, self.target_field_name, *objs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 1079, in _add_items
'%s__in' % target_field_name: new_ids,
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/query.py", line 679, in filter
return self._filter_or_exclude(False, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/query.py", line 697, in _filter_or_exclude
clone.query.add_q(Q(*args, **kwargs))
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1304, in add_q
clause, require_inner = self._add_q(where_part, self.used_aliases)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1332, in _add_q
allow_joins=allow_joins, split_subq=split_subq,
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/sql/query.py", line 1194, in build_filter
lookups, value)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/related.py", line 1745, in get_lookup_constraint
root_constraint.add(lookup_class(targets[0].get_col(alias, sources[0]), value), AND)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/lookups.py", line 96, in __init__
self.rhs = self.get_prep_lookup()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/lookups.py", line 134, in get_prep_lookup
return self.lhs.output_field.get_prep_lookup(self.lookup_name, self.rhs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/__init__.py", line 729, in get_prep_lookup
return [self.get_prep_value(v) for v in value]
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/django/db/models/fields/__init__.py", line 985, in get_prep_value
return int(value)
TypeError: int() argument must be a string or a number, not 'Location'.
To add a value to a ManyToManyField, you need to add the objects by passing them as arguments.
add(obj1, obj2, obj3, ...)
This will add the specified objects to the related object set.
You can call the add() like below:
menu_object.location.add(loc_obj1, loc_obj2, ...)
In your case, you need to do:
l = Location.objects.get_or_create(location_name=sq)[0]
You need to access the instance by index 0 as get_or_create returns a tuple (instance, created). Then pass this instance to the add() method.
Note:
Using add() with a many-to-many relationship will not call any .save() for each object, but rather create the relationships using .bulk_create().
I understand that Rahul Gupta has answered this already. But on a more theoretical level I think this is what happened when you use get_or_create here:
l = Location.objects.get_or_create(location_name=sq)
It returns a Tuple, and hence you have to call it's first element by modifying it to:
l = Location.objects.get_or_create(location_name=sq)[0]
Which returns only the index element of the tuple/dictionary!
I'm storing file paths as relative paths in the database, but I'm then using hybrid properties to turn in into an absolute path when its mapped. When I query using this property it throws an error. Here's the model:
class File(Base):
__tablename__ = 'files'
...
_f_path = Column(Unicode(30))
...
#hybrid_property
def f_path(self):
env = shelve.open('environment')
return os.path.join(env['project_dir'], self._f_path)
#f_path.setter
def f_path(self, _f_path):
self._f_path = _f_path
When I run this query (where ref is a unicode string):
session.query(File).filter_by(f_path=ref).first()
It gives me this error:
File "/Users/Ben/Dropbox/Giraffe/giraffe_server/giraffe/file_handlers/maya.py", line 135, in process_file
rf = session.query(File).filter_by(f_path=str(ref)).first()
File "build/bdist.macosx-10.7-intel/egg/sqlalchemy/orm/query.py", line 1211, in filter_by
for key, value in kwargs.iteritems()]
File "build/bdist.macosx-10.7-intel/egg/sqlalchemy/orm/util.py", line 597, in _entity_descriptor
return getattr(entity, key)
File "build/bdist.macosx-10.7-intel/egg/sqlalchemy/ext/hybrid.py", line 681, in __get__
return self.expr(owner)
File "/Users/Ben/Dropbox/Giraffe/giraffe_server/giraffe/model.py", line 133, in f_path
print "\n\n\n[model.py:File#f_path hybrid_property] returning: ", os.path.join(env['project_dir'], self._f_path)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/posixpath.py", line 66, in join
if b.startswith('/'):
File "build/bdist.macosx-10.7-intel/egg/sqlalchemy/sql/expression.py", line 3426, in __nonzero__
raise TypeError("Boolean value of this clause is not defined")
TypeError: Boolean value of this clause is not defined
Your hybrid property must return a sql expression; yours does not, it returns a python string instead.
To resolve that for this case, don't do the path join in python but in a SQL expression instead:
return env['project_dir'] + os.path.sep + self._f_path
which will resolve to self._f_path.__radd__(result_of_project_dir_plus_os_path_sep), which can be used both in queries and as a return value.