I can't use sqlite function group_concat() in peewee. Here is complete snipet. Somehow peewee want to convert result of group_concat() to integer, while it is string ("1,2"). I can't find the way to suppress it.
from peewee import *
db = SqliteDatabase(':memory:')
class Test(Model):
name = CharField()
score = IntegerField()
class Meta:
database = db
db.create_tables([Test])
Test.create(name='A', score=1).save()
Test.create(name='A', score=2).save()
#select name, group_concat(score) from Test group by name
for t in Test.select(Test.name, fn.group_concat(Test.score)).order_by(Test.name):
pass
It produces following error:
Traceback (most recent call last):
File "C:\Users\u_tem0m\Dropbox\Wrk\sgo\broken.py", line 17, in <module>
for t in Test.select(Test.name, fn.group_concat(Test.score)).order_by(Test.name):
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 1938, in next
obj = self.qrw.iterate()
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 1995, in iterate
return self.process_row(row)
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 2070, in process_row
setattr(instance, column, func(row[i]))
File "C:\Program Files\Python 3.5\lib\site-packages\peewee.py", line 874, in python_value
return value if value is None else self.coerce(value)
ValueError: invalid literal for int() with base 10: '1,2'
Try adding a coerce(False) to your call to group_concat:
query = (Test
.select(Test.name, fn.GROUP_CONCAT(Test.score).coerce(False))
.order_by(Test.name))
for t in query:
pass
Peewee sees that Test.score is an integer field, so whenever a function is called on that column, Peewee will try to convert the result back to an int. The problem is that group_concat returns a string, so we must tell Peewee not to mess with the return value.
Just found what result of fn.group_concat(""+Test.score) don't cast to integer. But I think resulting sql maybe less optimal
SELECT "t1"."name", group_concat(? + "t1"."score") AS allscore FROM "test" AS t1 ORDER BY "t1"."name" ['']
Do anybody knows more elegant way?
Related
My script migrates data from MySQL to mongodb. It runs perfectly well when there are no unicode columns included. But throws me below error when OrgLanguages column is added.
mongoImp = dbo.insert_many(odbcArray)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/collection.py", line 711, in insert_many
blk.execute(self.write_concern.document)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/bulk.py", line 493, in execute
return self.execute_command(sock_info, generator, write_concern)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/bulk.py", line 319, in execute_command
run.ops, True, self.collection.codec_options, bwc)
bson.errors.InvalidStringData: strings in documents must be valid UTF-8: 'Portugu\xeas do Brasil, ?????, English, Deutsch, Espa\xf1ol latinoamericano, Polish'
My code:
import MySQLdb, MySQLdb.cursors, sys, pymongo, collections
odbcArray=[]
mongoConStr = '192.168.10.107:36006'
sqlConnect = MySQLdb.connect(host = "54.175.170.187", user = "testuser", passwd = "testuser", db = "testdb", cursorclass=MySQLdb.cursors.DictCursor)
mongoConnect = pymongo.MongoClient(mongoConStr)
sqlCur = sqlConnect.cursor()
sqlCur.execute("SELECT ID,OrgID,OrgLanguages,APILoginID,TransactionKey,SMTPSpeed,TimeZoneName,IsVideoWatched FROM organizations")
dbo = mongoConnect.eaedw.mysqlData
tuples = sqlCur.fetchall()
for tuple in tuples:
odbcArray.append(collections.OrderedDict(tuple))
mongoImp = dbo.insert_many(odbcArray)
sqlCur.close()
mongoConnect.close()
sqlConnect.close()
sys.exit()
Above script migraates data perfectly when tried without OrgLanguages column in the SELECT query.
To overcome this, I have tried to use the OrderedDict() in another way but gives me a different type of error
Changed Code:
for tuple in tuples:
doc = collections.OrderedDict()
doc['oid'] = tuple.OrgID
doc['APILoginID'] = tuple.APILoginID
doc['lang'] = unicode(tuple.OrgLanguages)
odbcArray.append(doc)
mongoImp = dbo.insert_many(odbcArray)
Error Received:
Traceback (most recent call last):
File "pymsql.py", line 19, in <module>
doc['oid'] = tuple.OrgID
AttributeError: 'dict' object has no attribute 'OrgID'
Your MySQL connection is returning characters in a different encoding than UTF-8, which is the encoding that all BSON strings must be in. Try your original code but pass charset='utf8' to MySQLdb.connect.
I used some very simple code to create a database with peewee, I am new to using python ORMs so I can't really tell why I'm getting a whole bunch of errors.
What this code does is: First I create a database 'diary.db'
the data types used are entries, which is a Text Field, and date, which is a DateTimeField. I created some functions: 'initialize' to run basic commands and initialize the database, 'menu_loop' that will show a menu that works with an infinite loop and may call the function 'add_entry' that adds new entries to the database.
Heres the code:
#!/usr/bin/env python3
from collections import OrderedDict
from peewee import *
import datetime
import sys
db = SqliteDatabase('diary.db')
class Diary(Model):
entries = TextField()
date = DateTimeField(default = datetime.datetime.now)
class Meta:
database = db
def initialize():
"""initializes the database"""
db.connect()
db.create_tables([Diary], safe = True)
#end of initialize
def menu_loop():
"""show menu"""
choice = 0
while choice != 2:
print("Enter '2' to quit")
print('1) to add an entry')
choice = input()
if choice == 1:
add_entry()
#end of menu_loop
def add_entry():
"""add an entry"""
print("Enter your entry, press ctrl+d when done")
data = sys.stdin.read().strip()
if data:
while(True):
option = input('\nSave entry?[1 = yes 0 = no] ')
if option == 1:
Diary.create(content=data)
print ("Saved sucessfully!")
if option == 0:
print ("Program exited")
break;
#end of add_entry
if __name__ == '__main__':
initialize()
menu_loop()
and the error log
Enter '2' to quit
1) to add an entry
1
Enter your entry, press ctrl+d when done
this is my new entry
hello world^D
Save entry?[1 = yes 0 = no] 1
Traceback (most recent call last):
File "ispythonsmart.py", line 50, in <module>
menu_loop()
File "ispythonsmart.py", line 30, in menu_loop
add_entry()
File "ispythonsmart.py", line 41, in add_entry
Diary.create(content=data)
File "/Library/Python/2.7/site-packages/peewee.py", line 4494, in create
inst.save(force_insert=True)
File "/Library/Python/2.7/site-packages/peewee.py", line 4680, in save
pk_from_cursor = self.insert(**field_dict).execute()
File "/Library/Python/2.7/site-packages/peewee.py", line 3213, in execute
cursor = self._execute()
File "/Library/Python/2.7/site-packages/peewee.py", line 2628, in _execute
return self.database.execute_sql(sql, params, self.require_commit)
File "/Library/Python/2.7/site-packages/peewee.py", line 3461, in execute_sql
self.commit()
File "/Library/Python/2.7/site-packages/peewee.py", line 3285, in __exit__
reraise(new_type, new_type(*exc_args), traceback)
File "/Library/Python/2.7/site-packages/peewee.py", line 3454, in execute_sql
cursor.execute(sql, params or ())
peewee.IntegrityError: NOT NULL constraint failed: diary.entries
You need to set entries to null=True or use a default value for entries:
class Diary(Model):
entries = TextField(null=True)
Output:
Enter '2' to quit
1) to add an entry
1
Enter your entry, press ctrl+d when done
foobar
Save entry?[1 = yes 0 = no] 1
Saved successfully!
You want to see "entries" TEXT in the db not "entries" TEXT NOT NULL. If a column is set to NOT NULL you must insert a value or you will get an integrity error, an alternative is to give a default value for the column i.e TextField(default="foo"). On a sidenote you have #!/usr/bin/env python3 as your shebang but your code is written for python 2 so you may want to correct that.
I have been trying to add items to a DynamoDB table using boto, but somehow it doesn't seem to work. I tried using users.Item() and users.put_item but nothing worked. Below is the script that I have in use.
import boto.dynamodb2
import boto.dynamodb2.items
import json
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex
from boto.dynamodb2.layer1 import DynamoDBConnection
from boto.dynamodb2.table import Table
from boto.dynamodb2.items import Item
from boto.dynamodb2.types import NUMBER
region = "us-east-1"
con = boto.dynamodb2.connect_to_region(region)
gettables = con.list_tables()
mytable = "my_table"
if mytable not in gettables['TableNames']:
print "The table *%s* is not in the list of tables created. A new table will be created." % req_table
Table.create(req_table,
schema = [HashKey('username'),
RangeKey('ID', data_type = NUMBER)],
throughput = {'read': 1, 'write': 1})
else:
print "The table *%s* exists." % req_table
con2table = Table(req_table,connection=con)
con2table.put_item(data={'username': 'abcd',
'ID': '001',
'logins':'10',
'timeouts':'20'
'daysabsent': '30'
})
I tried this, the table gets created and it is fine. But when I try to put in the items, I get the following error message.
Traceback (most recent call last):
File "/home/ec2-user/DynamoDB_script.py", line 29, in <module>
'daysabsent':'30'
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 821, in put_item
return item.save(overwrite=overwrite)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/items.py", line 455, in save
returned = self.table._put_item(final_data, expects=expects)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 835, in _put_item
self.connection.put_item(self.table_name, item_data, **kwargs)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 1510, in put_item
body=json.dumps(params))
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2842, in make_request
retry_handler=self._retry_handler)
File "/usr/lib/python2.7/dist-packages/boto/connection.py", line 954, in _mexe
status = retry_handler(response, i, next_sleep)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2882, in _retry_handler
response.status, response.reason, data)
boto.dynamodb2.exceptions.ValidationException: ValidationException: 400 Bad Request
{u'message': u'One or more parameter values were invalid: Type mismatch for key version expected: N actual: S', u'__type': u'com.amazon.coral.validate#ValidationException'}
Thank you.
From the error message you are getting, it sounds like you are trying to send string values for an attribute that is defined as numeric in DynamoDB.
The specific issue looks to be related to your Range Key ID which is defined as a numeric value N but you are sending it a string value '001'.
Looks like of of the values you are trying to load has empty value.
I got the same error when I was trying to load this. I got exception when partner_name property was a empty string.
try:
item_old = self.table.get_item(hash_key=term)
except BotoClientError as ex:
# if partner alias does not exist then create a new entry!
if ex.message == "Key does not exist.":
item_old = self.table.new_item(term)
else:
raise ex
item_old['partner_code'] = partner_code
item_old['partner_name'] = partner_name
item_old.put()
I try to use these both prepared statements in my django app:
READINGS = "SELECT * FROM readings"
READINGS_BY_USER_ID = "SELECT * FROM readings WHERE user_id=?"
I query against the db with:
def get_all(self):
query = self.session.prepare(ps.ALL_READINGS)
all_readings = self.session.execute(query)
return all_readings
def get_all_by_user_id(self, user_id):
query = self.session.prepare(ps.READINGS_BY_USER_ID)
readings = self.session.execute(query, [user_id])
return readings
The first of both works pretty well. But the second gives me:
ERROR 2015-07-08 09:42:56,634 | views::exception_handler 47 | ('Unable to complete the operation against any hosts', {<Host: localhost data1>: TypeError("'unicode' does not have the buffer interface",)})
Can anyone tell me what happened here? I understand, that there must be a unicode string somewhere that does not have a buffer interface. But which string is meant? My prepared statement?
Here is the stacktrace in addition:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/rest_framework/views.py", line 448, in dispatch
response = handler(request, *args, **kwargs)
File "/Users/me/Workspace/project/Readings/views.py", line 36, in get_by_user_id
readings = self.tr_dao.get_all_by_user_id(user_id)
File "/Users/me/Workspace/project/Readings/dao.py", line 22, in get_all_by_user_id
readings = self.session.execute(query, [user_id], timeout=60)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/cassandra/cluster.py", line 1405, in execute
result = future.result(timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/cassandra/cluster.py", line 2967, in result
raise self._final_exception
if you are on python 2 this will probably fix it
def get_all_by_user_id(self, user_id):
query = self.session.prepare(ps.READINGS_BY_USER_ID)
readings = self.session.execute(query, [str(user_id)])
return readings
This is not working because your user_id is of type unicode. You can check it using
type(user_id)
If that is the case you should encode it to string:
str(user_id)
It will solve the issue.
item = Table('Item', metadata, autoload=True, autoload_with=engine, encoding = 'cp1257')
class Item(object):
pass
from sqlalchemy.orm import mapper
mapper(Item, item)
I get error:
line 43, in <module>
mapper(Item, item)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\__init__.py", line 890, in mapper
return Mapper(class_, local_table, *args, **params)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 211, in __init__
self._configure_properties()
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 578, in _configure_properties
setparent=True)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 618, in _configure_property
self._log("_configure_property(%s, %s)", key, prop.__class__.__name__)
File "C:\Python27\lib\site-packages\sqlalchemy\orm\mapper.py", line 877, in _log
(self.non_primary and "|non-primary" or "") + ") " +
File "C:\Python27\lib\site-packages\sqlalchemy\util.py", line 1510, in __get__
obj.__dict__[self.__name__] = result = self.fget(obj)
File "C:\Python27\lib\site-packages\sqlalchemy\sql\expression.py", line 3544, in description
return self.name.encode('ascii', 'backslashreplace')
UnicodeDecodeError: 'ascii' codec can't decode byte 0xeb in position 7: ordinal not in range(128)
I am connecting to MSSQL. table autoload seems to work. I only get this error while trying to map.
Thank you all for help!
Mapping the table to a class creates mapped properties on the class. The properties have the same name of the columns, by default. Since python 2.x only allows ascii identifiers, that fails if you have non-ascii column names.
The only solution I can think of is to give the identifiers a different name when mapping the table to a class.
The example below does that. Note that I'm creating the table on the code for simplicity, so anyone can run the code without having existing table. But you could do the same with a reflected table.
#-*- coding:utf-8 -*-
import sqlalchemy as sa
import sqlalchemy.orm
engine = sa.create_engine('sqlite://', echo=True) # new memory-only database
metadata = sa.MetaData(bind=engine)
# create a table. This could be reflected from the database instead:
tb = sa.Table('foo', metadata,
sa.Column(u'id', sa.Integer, primary_key=True),
sa.Column(u'nomé', sa.Unicode(100)),
sa.Column(u'ãéìöû', sa.Unicode(100))
)
tb.create()
class Foo(object):
pass
# maps the table to the class, defining different property names
# for some columns:
sa.orm.mapper(Foo, tb, properties={
'nome': tb.c[u'nomé'],
'aeiou': tb.c[u'ãéìöû']
})
After that you can use Foo.nome to refer to the nomé column and Foo.aeiou to refer to the ãéìöû column.
I faced the same problem and finally managed to do it replacing table['column'].key after autoloading it, just make all your table classes inherit this one and then modify the column name replacement in mapTo method or override manually the desired names with a dictionary and columns_descriptor method. I don't know if this is not the right way to do it but after searching for hours is the best aproach I've got.
class SageProxy(object):
#classmethod
def ismapped(cls, table_name=None):
if mappings:
if table_name:
if mappings.has_key(table_name):
tmap=mappings[table_name]
if tmap.has_key('class'):
tclass=tmap['class']
if tclass is cls:
return True
else:
for m in mappings:
if cls is m['class']:
return True
return False
#classmethod
def mappingprops(cls):
#override this to pass properties to sqlalchemy mapper function
return None
#classmethod
def columns_descriptors(cls):
#override this to map columns to different class properties names
#return dictionary where key is the column name and value is the desired property name
return {}
#classmethod
def mapTo(cls, table_name, map_opts=None):
if not cls.ismapped(table_name):
tab_obj=Table(table_name,sage_md,autoload=True)
for c in tab_obj.c:
#clean field names
tab_obj.c[c.name].key=c.key.replace(u'%',u'Porcentaje').replace(u'ñ',u'ny').replace(u'Ñ',u'NY').replace(u'-',u'_')
for k,v in cls.columns_descriptors():
if tab_obj.c[k]:
tab_obj.c[k].key=v
mapper(cls, tab_obj, properties=cls.mappingprops())
mappings[table_name]={'table':tab_obj,'class':cls}
return cls
I expect it will be usefull
I found that I could do this with a simple addition to my reflected class:
metadata = MetaData(bind=engine, reflect=True)
sm = sessionmaker(bind=engine)
class tblOrders(Base):
__table__ = metadata.tables['tblOrders']
meter = __table__.c['Meter#']
meter is now mapped to the underlying Meter# column, which allows this code to work:
currOrder = tblOrders()
currOrder.meter = '5'
Without the mapping, python sees it as a broken statement becase Meter followed by a comment does not exist in the object.