item = Table('Item', metadata, autoload=True, autoload_with=engine, encoding = 'cp1257')
class Item(object):
pass
from sqlalchemy.orm import mapper
mapper(Item, item)
I get error:
line 43, in <module>
mapper(Item, item)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\__init__.py", line 890, in mapper
return Mapper(class_, local_table, *args, **params)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 211, in __init__
self._configure_properties()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 578, in _configure_properties
setparent=True)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 618, in _configure_property
self._log("_configure_property(%s, %s)", key, prop.__class__.__name__)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 877, in _log
(self.non_primary and "|non-primary" or "") + ") " +
File "C:\Python27\lib\site-packages\sqlalchemy\util.py", line 1510, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "C:\Python27\lib\site-packages\sqlalchemy\sql\expression.py", line 3544, in description
return self.name.encode('ascii', 'backslashreplace')
UnicodeDecodeError: 'ascii' codec can't decode byte 0xeb in position 7: ordinal not in range(128)
I am connecting to MSSQL. table autoload seems to work. I only get this error while trying to map.
Thank you all for help!
Mapping the table to a class creates mapped properties on the class. The properties have the same name of the columns, by default. Since python 2.x only allows ascii identifiers, that fails if you have non-ascii column names.
The only solution I can think of is to give the identifiers a different name when mapping the table to a class.
The example below does that. Note that I'm creating the table on the code for simplicity, so anyone can run the code without having existing table. But you could do the same with a reflected table.
#-*- coding:utf-8 -*-
import sqlalchemy as sa
import sqlalchemy.orm
engine = sa.create_engine('sqlite://', echo=True) # new memory-only database
metadata = sa.MetaData(bind=engine)
# create a table. This could be reflected from the database instead:
tb = sa.Table('foo', metadata,
sa.Column(u'id', sa.Integer, primary_key=True),
sa.Column(u'nomé', sa.Unicode(100)),
sa.Column(u'ãéìöû', sa.Unicode(100))
)
tb.create()
class Foo(object):
pass
# maps the table to the class, defining different property names
# for some columns:
sa.orm.mapper(Foo, tb, properties={
'nome': tb.c[u'nomé'],
'aeiou': tb.c[u'ãéìöû']
})
After that you can use Foo.nome to refer to the nomé column and Foo.aeiou to refer to the ãéìöû column.
I faced the same problem and finally managed to do it replacing table['column'].key after autoloading it, just make all your table classes inherit this one and then modify the column name replacement in mapTo method or override manually the desired names with a dictionary and columns_descriptor method. I don't know if this is not the right way to do it but after searching for hours is the best aproach I've got.
class SageProxy(object):
#classmethod
def ismapped(cls, table_name=None):
if mappings:
if table_name:
if mappings.has_key(table_name):
tmap=mappings[table_name]
if tmap.has_key('class'):
tclass=tmap['class']
if tclass is cls:
return True
else:
for m in mappings:
if cls is m['class']:
return True
return False
#classmethod
def mappingprops(cls):
#override this to pass properties to sqlalchemy mapper function
return None
#classmethod
def columns_descriptors(cls):
#override this to map columns to different class properties names
#return dictionary where key is the column name and value is the desired property name
return {}
#classmethod
def mapTo(cls, table_name, map_opts=None):
if not cls.ismapped(table_name):
tab_obj=Table(table_name,sage_md,autoload=True)
for c in tab_obj.c:
#clean field names
tab_obj.c[c.name].key=c.key.replace(u'%',u'Porcentaje').replace(u'ñ',u'ny').replace(u'Ñ',u'NY').replace(u'-',u'_')
for k,v in cls.columns_descriptors():
if tab_obj.c[k]:
tab_obj.c[k].key=v
mapper(cls, tab_obj, properties=cls.mappingprops())
mappings[table_name]={'table':tab_obj,'class':cls}
return cls
I expect it will be usefull
I found that I could do this with a simple addition to my reflected class:
metadata = MetaData(bind=engine, reflect=True)
sm = sessionmaker(bind=engine)
class tblOrders(Base):
__table__ = metadata.tables['tblOrders']
meter = __table__.c['Meter#']
meter is now mapped to the underlying Meter# column, which allows this code to work:
currOrder = tblOrders()
currOrder.meter = '5'
Without the mapping, python sees it as a broken statement becase Meter followed by a comment does not exist in the object.
Related
Given a my_obj instance of MyType with a my_collection relationship for RelType, I have a validation method decorated with #validates('my_collection') that coerces appended dicts with a primary-key key/value pair into instances of RelType.
So, it works perfectly in this case:
my_obj.my_collection.append(RelType(rel_type_id=x))
And in this case, automatically coercing the values:
my_obj.my_collection.append({"rel_type_id": x})
However, the validation method is not called when the whole collection is replaced. It doesn't work for this case:
my_obj.my_collection = [{"rel_type_id": x}]
I get a TypeError: unhashable type: 'dict' because I'm trying to assign the dict directly, not an instance of RelType.
From the documentation, it looks like the only way to get it to work in that case is to have a custom collection class with a method tagged as #collection.converter, but besides the extra complication of using a custom collection, it looks like that would just duplicate code that's already in the validator.
Am I missing something? Is there a better/easier way?
UPDATE
Here's a minimal example reproducing the problem, SQLAlchemy 1.1.5
from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import validates
engine = create_engine('sqlite:///:memory:')
Session = sessionmaker(bind=engine)
Base = declarative_base()
class RelType(Base):
__tablename__ = 'rel_type'
rel_type_id = Column(Integer, primary_key=True)
my_type_id = Column(Integer, ForeignKey('my_type.my_type_id'))
class MyType(Base):
__tablename__ = 'my_type'
my_type_id = Column(Integer, primary_key=True)
my_collection = relationship('RelType')
#validates('my_collection')
def validate_my_collection(self, key, value):
if value is not None and not isinstance(value, RelType):
value = RelType(**value)
return value
def main():
Base.metadata.create_all(engine)
obj = MyType(my_type_id=1)
# this works
obj.my_collection.append({'rel_type_id': 2})
# but this immediately raises TypeError: unhashable type: 'dict'
obj.my_collection = [{'rel_type_id': 1}]
if __name__ == '__main__':
main()
This is the exception:
Traceback (most recent call last):
File "sqlamin.py", line 57, in <module>
main()
File "sqlamin.py", line 51, in main
obj.my_collection = [{'rel_type_id': 1}]
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/attributes.py", line 224, in __set__
instance_dict(instance), value, None)
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/attributes.py", line 1081, in set
new_values, old_collection, new_collection)
File ".env/lib/python3.4/site-packages/sqlalchemy/orm/collections.py", line 748, in bulk_replace
constants = existing_idset.intersection(values or ())
File ".env/lib/python3.4/site-packages/sqlalchemy/util/_collections.py", line 612, in intersection
result._members.update(self._working_set(members).intersection(other))
TypeError: unhashable type: 'dict'
Indeed, this is unexpected behavior, more of a design flaw than a bug. There's an issue opened for a better fix on 1.2, but meanwhile the workaround is using a custom collection with the #collection.converter decorator:
class MyCollection(list):
#collection.converter
def convert(self, value):
return [RelType(**v) if not isinstance(v, RelType) else v for v in value]
And use that with the relationship:
my_collection = relationship('RelType', collection_class=MyCollection)
Unfortunately the #collection.appender also doesn't work for similar reasons, so you have to implement the validator and the converter to catch both append and replace cases.
I can't use sqlite function group_concat() in peewee. Here is complete snipet. Somehow peewee want to convert result of group_concat() to integer, while it is string ("1,2"). I can't find the way to suppress it.
from peewee import *
db = SqliteDatabase(':memory:')
class Test(Model):
name = CharField()
score = IntegerField()
class Meta:
database = db
db.create_tables([Test])
Test.create(name='A', score=1).save()
Test.create(name='A', score=2).save()
#select name, group_concat(score) from Test group by name
for t in Test.select(Test.name, fn.group_concat(Test.score)).order_by(Test.name):
pass
It produces following error:
Traceback (most recent call last):
File "C:\Users\u_tem0m\Dropbox\Wrk\sgo\broken.py", line 17, in <module>
for t in Test.select(Test.name, fn.group_concat(Test.score)).order_by(Test.name):
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 1938, in next
obj = self.qrw.iterate()
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 1995, in iterate
return self.process_row(row)
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 2070, in process_row
setattr(instance, column, func(row[i]))
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 874, in python_value
return value if value is None else self.coerce(value)
ValueError: invalid literal for int() with base 10: '1,2'
Try adding a coerce(False) to your call to group_concat:
query = (Test
.select(Test.name, fn.GROUP_CONCAT(Test.score).coerce(False))
.order_by(Test.name))
for t in query:
pass
Peewee sees that Test.score is an integer field, so whenever a function is called on that column, Peewee will try to convert the result back to an int. The problem is that group_concat returns a string, so we must tell Peewee not to mess with the return value.
Just found what result of fn.group_concat(""+Test.score) don't cast to integer. But I think resulting sql maybe less optimal
SELECT "t1"."name", group_concat(? + "t1"."score") AS allscore FROM "test" AS t1 ORDER BY "t1"."name" ['']
Do anybody knows more elegant way?
I just start a simple project called flask_wiki this days and I'm using some flask extensions as the follows:
Flask-SQLAlchemy
Flask-Restful
MarshMallow
Well, I just discovered that the MarshMallow project provides a class called 'ModelSchema', which reads all fields from my SQLAlchemy Model and provide a fully automated (de)serialializer.
In my case, I created a 'GUID' Field which is RDBM agnostic and inserted it on my Sqlalchemy model as follows:
from flask.ext.sqlalchemy import SQLAlchemy
from flask_wiki.backend.custom_fields import GUIDField
class Page(db.Model):
"""
Implements the Page Model.
"""
guid = db.Column(GUIDField, primary_key=True, default=uuid.uuid4)
name = db.Column(db.String, nullable=False)
raw_content = db.Column(db.Text)
rendered_content = db.Column(db.Text)
def __repr__(self):
return self.__str__()
def __str__(self):
return self.name
The GUIDField is implemented is this way:
from sqlalchemy.types import TypeDecorator, CHAR
import uuid
class GUIDField(TypeDecorator):
# Platform independent GUID Implementation that uses little endianess.
impl = CHAR
def load_dialect_impl(self, dialect):
return dialect.type_descriptor(CHAR(32))
def process_bind_param(self, value, dialect):
if value is None:
return value
else:
if isinstance(value, uuid.UUID):
return value.bytes_le
def process_result_value(self, value, dialect):
if value is None:
return value
else:
return uuid.UUID(bytes_le=value)
The code for create test objects (through mixer) is working; all the guids are generated and verified correctly.
With this in mind, a just created the following MarshMallow Serializer Field:
from marshmallow import fields
import uuid
class GUIDSerializationField(fields.Field):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if isinstance(value, uuid.UUID):
return str(value)
else:
return None
Finally, I created the SerializerClass:
from flask_wiki.backend.backend import marsh
from flask_wiki.backend.custom_serialization_fields import GUIDSerializationField
from flask_wiki.backend.models import Page
from marshmallow import fields
class PageSchema(marsh.ModelSchema):
class Meta:
model = Page
guid = GUIDSerializationField()
page_schema = PageSchema()
pages_schema = PageSchema(many=True)
I tried to use this last code with and without inserting the guid field, but in all cases the following error occurs:
Traceback (most recent call last):
File "/home/arthas/dev/flask-wiki/flask_wiki/wiki.py", line 3, in <module>
from flask_wiki.backend import backend
File "/home/arthas/dev/flask-wiki/flask_wiki/backend/__init__.py", line 1, in <module>
import flask_wiki.backend.routes
File "/home/arthas/dev/flask-wiki/flask_wiki/backend/routes.py", line 2, in <module>
from flask_wiki.backend.views import PageView
File "/home/arthas/dev/flask-wiki/flask_wiki/backend/views.py", line 3, in <module>
from flask_wiki.backend.serializers import pages_schema, page_schema
File "/home/arthas/dev/flask-wiki/flask_wiki/backend/serializers.py", line 7, in <module>
class PageSchema(marsh.ModelSchema):
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow/schema.py", line 116, in __new__
dict_cls=dict_cls
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/schema.py", line 53, in get_declared_fields
declared_fields = mcs.get_fields(converter, opts)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/schema.py", line 77, in get_fields
return converter.fields_for_model(opts.model, fields=opts.fields, exclude=opts.exclude)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/convert.py", line 75, in fields_for_model
field = self.property2field(prop)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/convert.py", line 93, in property2field
field_class = self._get_field_class_for_property(prop)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/convert.py", line 151, in _get_field_class_for_property
field_cls = self._get_field_class_for_column(column)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/convert.py", line 121, in _get_field_class_for_column
return self._get_field_class_for_data_type(column.type)
File "/home/arthas/env/wiki/lib/python3.5/site-packages/marshmallow_sqlalchemy/convert.py", line 143, in _get_field_class_for_data_type
'Could not find field column of type {0}.'.format(types[0]))
marshmallow_sqlalchemy.exceptions.ModelConversionError: Could not find field column of type <class 'flask_wiki.backend.custom_fields.GUIDField'>.
So, I finally ask: how to use marshmallow to serialize a custom sqlalchemy field?
You need to create your own Converter.
Try something like this:
import uuid
from flask_wiki.backend.models import Page
from flask_wiki.backend.backend import marsh
from marshmallow_sqlalchemy.convert import ModelConverter
from flask_wiki.backend.custom_fields import GUIDField
from marshmallow import fields
#Here you are overwriting the list of types of the SQLAlchemy, adding your custom type
class GUIDConverter(ModelConverter):
SQLA_TYPE_MAPPING = dict(
list(ModelConverter.SQLA_TYPE_MAPPING.items()) +
[(GUIDField, fields.Str)]
)
class GUIDSerializationField(fields.Field):
def _serialize(self, value, attr, obj):
if value is None:
return value
else:
if isinstance(value, uuid.UUID):
return str(value)
else:
return None
class PageSchema(marsh.ModelSchema):
class Meta:
model = Page
model_converter = GUIDConverter #Tell to Marshmallow to use your custom converter for this model
guid = GUIDSerializationField(attribute="guid")
You can also see this link for help.
I hope it helped and sorry for my bad english
According to what I've read in the docs, you can specify a model_converter attribute in your ModelSchema Meta class. The ModelConverter class has a SQLA_TYPE_MAPPING attribute you can override in a subclass to add your custom GUID field to the types detected by the automatic schema generator.
That said, I've never used it so I don't know if this will work or not.
I'm using python-storm as orm. The many-to-many reference set is giving me headaches :(
These are the relevant objects:
class Author(object):
__storm_table__ = "author"
id = Int(primary=True)
name = Unicode()
institution_id = Int()
institution = Reference(institution_id, Institution.id)
def __init__(self, name):
self.name = name
class Paper(object):
__storm_table__ = "paper"
id = Int(primary=True)
name = Unicode()
conference_id = Int()
conference = Reference(conference_id, Conference.id)
def __init__(self, name):
self.name = name
class AuthorPapers(object):
__storm_table__ = "authorpapers"
__storm_primary__ = "author_id", "paper_id"
author_id = Int()
paper_id = Int()
The respective sqlite table look like this
store.execute("CREATE TABLE if not exists author (id INTEGER PRIMARY KEY, name VARCHAR, institution_id INTEGER, FOREIGN KEY (institution_id) REFERENCES institution(id))")
store.execute("CREATE TABLE if not exists paper (id INTEGER PRIMARY KEY, name VARCHAR, conference_id INTEGER, FOREIGN KEY (conference_id) REFERENCES conference(id))")
store.execute("CREATE TABLE if not exists authorpapers (author_id INTEGER, paper_id INTEGER, PRIMARY KEY (author_id, paper_id))")
Now say if a have two author the collaborated on a paper
a = Author(u"Steve Rogers")
b = Author(u"Captain America")
and a paper
p6 = Paper(u"Bunga Bunga")
So now I want to associate both author to the paper using
Author.papers = ReferenceSet(Author.id, AuthorPapers.author_id, Paper.id, AuthorPapers.paper_id)
and doing this
a.papers.add(p6)
b.papers.add(p6)
This is btw it says it is supposed to work in the storm tutorial...but I get
File "/usr/lib64/python2.7/site-packages/storm/references.py", line 376, in add
self._relation2.link(remote, link, True)
File "/usr/lib64/python2.7/site-packages/storm/references.py", line 624, in link
pairs = zip(self._get_local_columns(local.__class__),
File "/usr/lib64/python2.7/site-packages/storm/references.py", line 870, in _get_local_columns
for prop in self.local_key)
File "/usr/lib64/python2.7/site-packages/storm/references.py", line 870, in <genexpr>
for prop in self.local_key)
File "/usr/lib64/python2.7/site-packages/storm/properties.py", line 53, in __get__
return self._get_column(cls)
File "/usr/lib64/python2.7/site-packages/storm/properties.py", line 97, in _get_column
attr = self._detect_attr_name(cls)
File "/usr/lib64/python2.7/site-packages/storm/properties.py", line 82, in _detect_attr_name
raise RuntimeError("Property used in an unknown class")
RuntimeError: Property used in an unknown class
And I'm not really able to make sense of this right now.
I'm not really, familiar with storm, but looking at the documentation example, looks like is just an issue related to the order in which the arguments to ReferenceSet are passed. I tried to use this:
Author.papers = ReferenceSet(Author.id, AuthorPapers.author_id, AuthorPapers.paper_id, Paper.id)
instead of this:
Author.papers = ReferenceSet(Author.id, AuthorPapers.author_id, Paper.id, AuthorPapers.paper_id)
and no exception was raised.
I am trying to create a program that loads in over 100 tables from a database so that I can change all appearances of a user's user id.
Rather than map all of the tables individually, I decided to use a loop to map each of the tables using an array of objects. This way, the table definitions can be stored in a config file and later updated.
Here is my code so far:
def init_model(engine):
"""Call me before using any of the tables or classes in the model"""
meta.Session.configure(bind=engine)
meta.engine = engine
class Table:
tableID = ''
primaryKey = ''
pkType = sa.types.String()
class mappedClass(object):
pass
WIW_TBL = Table()
LOCATIONS_TBL = Table()
WIW_TBL.tableID = "wiw_tbl"
WIW_TBL.primaryKey = "PORTAL_USERID"
WIW_TBL.pkType = sa.types.String()
LOCATIONS_TBL.tableID = "locations_tbl"
LOCATIONS_TBL.primaryKey = "LOCATION_CODE"
LOCATIONS_TBL.pkType = sa.types.Integer()
tableList = ([WIW_TBL, LOCATIONS_TBL])
for i in tableList:
i.tableID = sa.Table(i.tableID.upper(), meta.metadata,
sa.Column(i.primaryKey, i.pkType, primary_key=True),
autoload=True,
autoload_with=engine)
orm.mapper(i.mappedClass, i.tableID)
The error that this code returns is:
sqlalchemy.exc.ArgumentError: Class '<class 'changeofname.model.mappedClass'>' already has a primary mapper defined. Use non_primary=True to create a non primary Mapper. clear_mappers() will remove *all* current mappers from all classes.
I cant use clear_mappers as it wipes all of the classes and the entity_name scheme doesn't seem to apply here.
It seems that every object wants to use the same class, although they all should have their own instance of it.
Does anyone have any ideas?
Well, in your case it *is the same Class you try to map to different Tables. To solve this, create a class dynamically for each Table:
class Table(object):
tableID = ''
primaryKey = ''
pkType = sa.types.String()
def __init__(self):
self.mappedClass = type('TempClass', (object,), {})
But I would prefer slightly cleaner version:
class Table2(object):
def __init__(self, table_id, pk_name, pk_type):
self.tableID = table_id
self.primaryKey = pk_name
self.pkType = pk_type
self.mappedClass = type('Class_' + self.tableID, (object,), {})
# ...
WIW_TBL = Table2("wiw_tbl", "PORTAL_USERID", sa.types.String())
LOCATIONS_TBL = Table2("locations_tbl", "LOCATION_CODE", sa.types.Integer())