We Keep Coding

Validate/coerce values when replacing an entire collection in SQLAlchemy

Given a my_obj instance of MyType with a my_collection relationship for RelType, I have a validation method decorated with #validates('my_collection') that coerces appended dicts with a primary-key key/value pair into instances of RelType.
So, it works perfectly in this case:
my_obj.my_collection.append(RelType(rel_type_id=x))
And in this case, automatically coercing the values:
my_obj.my_collection.append({"rel_type_id": x})
However, the validation method is not called when the whole collection is replaced. It doesn't work for this case:
my_obj.my_collection = [{"rel_type_id": x}]
I get a TypeError: unhashable type: 'dict' because I'm trying to assign the dict directly, not an instance of RelType.
From the documentation, it looks like the only way to get it to work in that case is to have a custom collection class with a method tagged as #collection.converter, but besides the extra complication of using a custom collection, it looks like that would just duplicate code that's already in the validator.
Am I missing something? Is there a better/easier way?
UPDATE
Here's a minimal example reproducing the problem, SQLAlchemy 1.1.5
from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import validates
engine = create_engine('sqlite:///:memory:')
Session = sessionmaker(bind=engine)
Base = declarative_base()
class RelType(Base):
__tablename__ = 'rel_type'
rel_type_id = Column(Integer, primary_key=True)
my_type_id = Column(Integer, ForeignKey('my_type.my_type_id'))
class MyType(Base):
__tablename__ = 'my_type'
my_type_id = Column(Integer, primary_key=True)
my_collection = relationship('RelType')
#validates('my_collection')
def validate_my_collection(self, key, value):
if value is not None and not isinstance(value, RelType):
value = RelType(**value)
return value
def main():
Base.metadata.create_all(engine)
obj = MyType(my_type_id=1)
# this works
obj.my_collection.append({'rel_type_id': 2})
# but this immediately raises TypeError: unhashable type: 'dict'
obj.my_collection = [{'rel_type_id': 1}]
if __name__ == '__main__':
main()
This is the exception:
Traceback (most recent call last):
File "sqlamin.py", line 57, in <module>
main()
File "sqlamin.py", line 51, in main
obj.my_collection = [{'rel_type_id': 1}]
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/attributes.py", line 224, in __set__
instance_dict(instance), value, None)
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/attributes.py", line 1081, in set
new_values, old_collection, new_collection)
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/collections.py", line 748, in bulk_replace
constants = existing_idset.intersection(values or ())
File ".env/lib/python3.4/site-packages/sqlalchemy/util/_collections.py", line 612, in intersection
result._members.update(self._working_set(members).intersection(other))
TypeError: unhashable type: 'dict'

Indeed, this is unexpected behavior, more of a design flaw than a bug. There's an issue opened for a better fix on 1.2, but meanwhile the workaround is using a custom collection with the #collection.converter decorator:
class MyCollection(list):
#collection.converter
def convert(self, value):
return [RelType(**v) if not isinstance(v, RelType) else v for v in value]
And use that with the relationship:
my_collection = relationship('RelType', collection_class=MyCollection)
Unfortunately the #collection.appender also doesn't work for similar reasons, so you have to implement the validator and the converter to catch both append and replace cases.

peewee refers to a base class table that does not exist

The following set-up illustrates my problem:
from peewee import (Model,
SqliteDatabase)
from peewee import (CharField,
ForeignKeyField,
IntegerField)
from playhouse.shortcuts import model_to_dict
database = SqliteDatabase(':memory:')
class BaseModel(Model):
class Meta:
database = database
class Description(BaseModel):
i_am = CharField(
db_column='I_AM'
)
class Meta:
db_table = 'DESCRIPTION'
# This is not a table, just a convenient class to inherit from.
class ValueBase(BaseModel):
value = IntegerField(
db_column='VALUE'
)
description = ForeignKeyField(
db_column='I_AM_ID',
rel_model=Description,
to_field='id'
)
class SpecificValue(ValueBase):
class Meta:
db_table = 'SPECIFIC_VALUE'
if __name__ == '__main__':
models = [Description, SpecificValue]
database.create_tables(models)
description = Description(
i_am = 'prime'
)
description.save()
specific = SpecificValue(
value=7,
description=description
)
specific.save()
### This works
print model_to_dict(
specific
)
### This does NOT work
print model_to_dict(
specific,
backrefs=True
)
The problem is that when I use model_to_dict and ask for back references, then the function thinks specific is in a table valuebase and not in a table called SPECIFIC_VALUE. Does anyone know how to do this sort of inheritance and correctly set table names?
EDIT
The traceback is as follows:
Traceback (most recent call last):
File "/home/lafras/Projects/Cloud/eg.py", line 61, in <module>
backrefs=True
File "build/bdist.linux-x86_64/egg/playhouse/shortcuts.py", line 96, in model_to_dict
File "build/bdist.linux-x86_64/egg/playhouse/shortcuts.py", line 117, in model_to_dict
File "build/bdist.linux-x86_64/egg/peewee.py", line 2794, in __iter__
File "build/bdist.linux-x86_64/egg/peewee.py", line 2787, in execute
File "build/bdist.linux-x86_64/egg/peewee.py", line 2470, in _execute
File "build/bdist.linux-x86_64/egg/peewee.py", line 3199, in execute_sql
File "build/bdist.linux-x86_64/egg/peewee.py", line 3048, in __exit__
File "build/bdist.linux-x86_64/egg/peewee.py", line 3191, in execute_sql
peewee.OperationalError: no such table: valuebase
If SpecificValue inherits directly from BaseModel:
class SpecificValue(BaseModel):
value = IntegerField(
db_column='VALUE'
)
description = ForeignKeyField(
db_column='I_AM_ID',
rel_model=Description,
to_field='id'
)
class Meta:
db_table = 'SPECIFIC_VALUE'
then the output is as expected:
{'id': 1, 'value': 7, 'description': {'id': 1, 'i_am': 'prime'}}

Can you explain a bit more thoroughly what's not working correctly? I don't see any back-references that would be associated with the SpecificValue table, since there are not other models that have a foreign key to it.
EDIT:
OK, so what's happening here is that when you specify backrefs, peewee is trying to go from:
SpecificValue -> Description -> ValueBase (which has a foreign key to description).
The fix is to explicitly exclude the backref:
model_to_dict(specific, backrefs=True, exclude=[Description.valuebase_set])

Returning verbose names from query set when using json serializer

Is it possible to call
tasks = models.Conference.objects.filter(location_id=key)
data = serializers.serialize("json", tasks)
and have it return the verbose field names rather than the variable names?

One way to accomplish this, is by monkey patching the methods within the django.core.serializers.python.Serializer class to return each fields verbose_name opposed to the standard name attribute.
Take for example the following code...
models.py
from django.db import models
class RelatedNode(models.Model):
name = models.CharField(max_length=100, verbose_name="related node")
class Node(models.Model):
name = models.CharField(max_length=100, verbose_name="verbose name")
related_node = models.ForeignKey(RelatedNode, verbose_name="verbose fk related node", related_name="related_node")
related_nodes = models.ManyToManyField(RelatedNode, verbose_name="verbose related m2m nodes", related_name="related_nodes")
I create these model objects within the database...
RelatedNode.objects.create(name='related_node_1')
RelatedNode.objects.create(name='related_node_2')
RelatedNode.objects.create(name='related_node_fk')
Node.objects.create(name='node_1', related_node=RelatedNode.objects.get(name='related_node_fk'))
Node.objects.all()[0].related_nodes.add(RelatedNode.objects.get(name='related_node_1'))
Node.objects.all()[0].related_nodes.add(RelatedNode.objects.get(name='related_node_2'))
views.py
from testing.models import Node
from django.utils.encoding import smart_text, is_protected_type
from django.core.serializers.python import Serializer
from django.core import serializers
def monkey_patch_handle_field(self, obj, field):
value = field._get_val_from_obj(obj)
# Protected types (i.e., primitives like None, numbers, dates,
# and Decimals) are passed through as is. All other values are
# converted to string first.
if is_protected_type(value):
self._current[field.verbose_name] = value
else:
self._current[field.verbose_name] = field.value_to_string(obj)
def monkey_patch_handle_fk_field(self, obj, field):
if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
related = getattr(obj, field.name)
if related:
value = related.natural_key()
else:
value = None
else:
value = getattr(obj, field.get_attname())
self._current[field.verbose_name] = value
def monkey_patch_handle_m2m_field(self, obj, field):
if field.rel.through._meta.auto_created:
if self.use_natural_foreign_keys and hasattr(field.rel.to, 'natural_key'):
m2m_value = lambda value: value.natural_key()
else:
m2m_value = lambda value: smart_text(value._get_pk_val(), strings_only=True)
self._current[field.verbose_name] = [m2m_value(related)
for related in getattr(obj, field.name).iterator()]
Serializer.handle_field = monkey_patch_handle_field
Serializer.handle_fk_field = monkey_patch_handle_fk_field
Serializer.handle_m2m_field = monkey_patch_handle_m2m_field
serializers.serialize('json', Node.objects.all())
This outputs for me...
u'[{"fields": {"verbose fk related node": 3, "verbose related m2m nodes": [1, 2], "verbose name": "node_1"}, "model": "testing.node", "pk": 1}]'
As we could see, this actually gives us back the verbose_name of each field as keys in the returned dictionaries.

mongokit No collection found

I'm using mongokit with flask, and everytime I try to use a collection I created, I receive the error No collection found
I defined my collections in a separated file models.py. It looks like this:
from mongokit import Connection, Document
import os
import sys
here = os.path.dirname(os.path.abspath(__file__))
path = os.path.abspath(os.path.join(here, 'settings'))
sys.path.append(path)
from settings import base as settings
connection = Connection()
#connection.register
class Contact(Document):
__database__ = settings.MONGO_DBNAME
__collection__ = "Contact"
structure = {
"name":unicode,
"mobile_number":unicode,
}
required_fields = ["name"]
#connection.register
class User(Document):
__database__ = settings.MONGO_DBNAME
__collection__ = 'User'
structure = {
"username":unicode,
"twitter_access_token":unicode,
"twitter_token_secret":unicode,
"contacts":[Contact]
}
required_fields = ["username"]
default_values = {
"twitter_access_token": "",
"twitter_token_secret": ""
}
But then I tried:
>>> from models import User
>>> u = User()
>>> u["username"] = "somename"
>>> u.save()
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/Users/fernandocezar/.virtualenvs/contacts/lib/python2.7/site-packages/mongokit/document.py", line 404, in save
self.validate(auto_migrate=False)
File "/Users/fernandocezar/.virtualenvs/contacts/lib/python2.7/site-packages/mongokit/document.py", line 230, in validate
(size_limit, size_limit_str) = self._get_size_limit()
File "/Users/fernandocezar/.virtualenvs/contacts/lib/python2.7/site-packages/mongokit/document.py", line 214, in _get_size_limit
server_version = tuple(self.connection.server_info()['version'].split("."))
File "/Users/fernandocezar/.virtualenvs/contacts/lib/python2.7/site-packages/mongokit/document.py", line 622, in __getattribute__
raise ConnectionError('No collection found')
mongokit.mongo_exceptions.ConnectionError: No collection found
I followed this tutorial, but not even the notation connection.<dbname>.<collection>() works. And yes, there is, indeed, such a collection.
What am I missing?

To quote the tutorial you linked:
To avoid repeating ourselves though, let’s specify the database and
collection name in the Document definition:
#connection.register
class BlogPost(Document):
__collection__ = 'blog_posts'
__database__ = 'blog'
structure = {...}
>>> bp = connection.BlogPost()
In the shell example, the model object is constructed through the connection object. In your case, you were simply doing user = User(). Try creating the user through the same connection instance that you used to register the model (e.g. user = connection.User()).

SQLAlchemy mapping table with non-ascii columns to class

item = Table('Item', metadata, autoload=True, autoload_with=engine, encoding = 'cp1257')
class Item(object):
pass
from sqlalchemy.orm import mapper
mapper(Item, item)
I get error:
line 43, in <module>
mapper(Item, item)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\__init__.py", line 890, in mapper
return Mapper(class_, local_table, *args, **params)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 211, in __init__
self._configure_properties()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 578, in _configure_properties
setparent=True)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 618, in _configure_property
self._log("_configure_property(%s, %s)", key, prop.__class__.__name__)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 877, in _log
(self.non_primary and "|non-primary" or "") + ") " +
File "C:\Python27\lib\site-packages\sqlalchemy\util.py", line 1510, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "C:\Python27\lib\site-packages\sqlalchemy\sql\expression.py", line 3544, in description
return self.name.encode('ascii', 'backslashreplace')
UnicodeDecodeError: 'ascii' codec can't decode byte 0xeb in position 7: ordinal not in range(128)
I am connecting to MSSQL. table autoload seems to work. I only get this error while trying to map.
Thank you all for help!

Mapping the table to a class creates mapped properties on the class. The properties have the same name of the columns, by default. Since python 2.x only allows ascii identifiers, that fails if you have non-ascii column names.
The only solution I can think of is to give the identifiers a different name when mapping the table to a class.
The example below does that. Note that I'm creating the table on the code for simplicity, so anyone can run the code without having existing table. But you could do the same with a reflected table.
#-*- coding:utf-8 -*-
import sqlalchemy as sa
import sqlalchemy.orm
engine = sa.create_engine('sqlite://', echo=True) # new memory-only database
metadata = sa.MetaData(bind=engine)
# create a table. This could be reflected from the database instead:
tb = sa.Table('foo', metadata,
sa.Column(u'id', sa.Integer, primary_key=True),
sa.Column(u'nomé', sa.Unicode(100)),
sa.Column(u'ãéìöû', sa.Unicode(100))
)
tb.create()
class Foo(object):
pass
# maps the table to the class, defining different property names
# for some columns:
sa.orm.mapper(Foo, tb, properties={
'nome': tb.c[u'nomé'],
'aeiou': tb.c[u'ãéìöû']
})
After that you can use Foo.nome to refer to the nomé column and Foo.aeiou to refer to the ãéìöû column.

I faced the same problem and finally managed to do it replacing table['column'].key after autoloading it, just make all your table classes inherit this one and then modify the column name replacement in mapTo method or override manually the desired names with a dictionary and columns_descriptor method. I don't know if this is not the right way to do it but after searching for hours is the best aproach I've got.
class SageProxy(object):
#classmethod
def ismapped(cls, table_name=None):
if mappings:
if table_name:
if mappings.has_key(table_name):
tmap=mappings[table_name]
if tmap.has_key('class'):
tclass=tmap['class']
if tclass is cls:
return True
else:
for m in mappings:
if cls is m['class']:
return True
return False
#classmethod
def mappingprops(cls):
#override this to pass properties to sqlalchemy mapper function
return None
#classmethod
def columns_descriptors(cls):
#override this to map columns to different class properties names
#return dictionary where key is the column name and value is the desired property name
return {}
#classmethod
def mapTo(cls, table_name, map_opts=None):
if not cls.ismapped(table_name):
tab_obj=Table(table_name,sage_md,autoload=True)
for c in tab_obj.c:
#clean field names
tab_obj.c[c.name].key=c.key.replace(u'%',u'Porcentaje').replace(u'ñ',u'ny').replace(u'Ñ',u'NY').replace(u'-',u'_')
for k,v in cls.columns_descriptors():
if tab_obj.c[k]:
tab_obj.c[k].key=v
mapper(cls, tab_obj, properties=cls.mappingprops())
mappings[table_name]={'table':tab_obj,'class':cls}
return cls
I expect it will be usefull

I found that I could do this with a simple addition to my reflected class:
metadata = MetaData(bind=engine, reflect=True)
sm = sessionmaker(bind=engine)
class tblOrders(Base):
__table__ = metadata.tables['tblOrders']
meter = __table__.c['Meter#']
meter is now mapped to the underlying Meter# column, which allows this code to work:
currOrder = tblOrders()
currOrder.meter = '5'
Without the mapping, python sees it as a broken statement becase Meter followed by a comment does not exist in the object.