We Keep Coding

'exceptions.TypeError' when executing python script

I'm getting the following error when executing the following script:
Error Type: <type 'exceptions.TypeError'>
Error Contents: 'NoneType' object is not iterable
Traceback (most recent call last):
File "addon.py", line 75, in <module>
plugin.run()
File "xbmcswift2/plugin.py", line 332, in run
items = self._dispatch(self.request.path)
File "/plugin.py", line 306, in _dispatch
listitems = view_func(**items)
File "/addon.py", line 42, in all_episodes
items = thisiscriminal.compile_playable_podcast(playable_podcast)
File "/lib/thisiscriminal.py", line 121, in compile_playable_podcast
for podcast in playable_podcast:
TypeError: 'NoneType' object is not iterable
The code in question is as follows, any advice would be greatly appreciated as I have no idea what I'm doing wrong:
def get_playable_podcast(soup):
"""
#param: parsed html page
"""
r = urllib.urlopen('https://thisiscriminal.com/wp-json/criminal/v1/episodes?posts=10000&page=1')
data = json.loads(r.read().decode('utf-8'))
for post in data['posts']:
print post['title']
print post['episodeNumber']
print post['audioSource']
print post['image']['medium']
subjects = []
item = {
'title': post['title'],
'audioSource': post['audioSource'],
'episodeNumber': post['episodeNumber'],
'medium': post['image']['medium']
}
subjects.append(item)
print subjects
def compile_playable_podcast(playable_podcast):
"""
#para: list containing dict of key/values pairs for playable podcasts
"""
items = []
for podcast in playable_podcast:
items.append({
post['title']: podcast['title']['episodeNumber'],
post['audioSource']: podcast['audioSource'],
post['image']['medium']: podcast['medium'],
'is_playable': True,})
return items

I assume your script does something alike to the following,
podcast = get_playable_podcast(soup)
compiled = compile_playable_podcast(podcast)
The problem is that get_playable_podcast has no return statement. In such a case, Python defaults to returning None - which you then pass into compile_playable_podcast. Since None is not iterable, compile_playable_podcast rightfully raises a TypeError.
Now, the solution is of course to return the podcast list you're building in get_playable_podcast, like so,
def get_playable_podcast(soup):
"""
#param: parsed html page
"""
r = urllib.urlopen('https://thisiscriminal.com/wp-json/criminal/v1/episodes?posts=10000&page=1')
data = json.loads(r.read().decode('utf-8'))
subjects = []
for post in data['posts']:
print post['title']
print post['episodeNumber']
print post['audioSource']
print post['image']['medium']
item = {
'title': post['title'],
'audioSource': post['audioSource'],
'episodeNumber': post['episodeNumber'],
'medium': post['image']['medium']
}
subjects.append(item)
print subjects
return subjects
Beside this, it may be worthwhile to carefully check your script for unused parameters and/or duplicate code.

API using MechanicalSoup is failing to submit form - NoneType

I'm using a python API that uses MechanicalSoup to do most of its functions and all of a sudden, it won't work anymore. I feel like the website it's utilizing may have changed or something.
Here is the API's code:
def trade(self, symbol, orderType, quantity, priceType="Market", price=False, duration=Duration.good_cancel):
"""
Executes trades on the platform. See the readme.md file
for examples on use and inputs. Returns True if the
trade was successful. Else an exception will be
raised.
client.trade("GOOG", Action.buy, 10)
client.trade("GOOG", Action.buy, 10, "Limit", 500)
"""
br = self.br
trade_page = self.fetch('/simulator/trade/tradestock.aspx')
trade_form = trade_page.soup.select("form#orderForm")[0]
# input symbol, quantity, etc.
trade_form.select("input#symbolTextbox")[0]["value"] = symbol
trade_form.select("input#quantityTextbox")[0]["value"] = str(quantity)
# input transaction type
[option.attrs.pop("selected", "") for option in trade_form.select("select#transactionTypeDropDown")[0]("option")]
trade_form.select("select#transactionTypeDropDown")[0].find("option", {"value": str(orderType.value)})["selected"] = True
# input price type
[radio.attrs.pop("checked", "") for radio in trade_form("input", {"name": "Price"})]
trade_form.find("input", {"name": "Price", "value": priceType})["checked"] = True
# input duration type
[option.attrs.pop("selected", "") for option in trade_form.select("select#durationTypeDropDown")[0]("option")]
trade_form.select("select#durationTypeDropDown")[0].find("option", {"value": str(duration.value)})["selected"] = True
# if a limit or stop order is made, we have to specify the price
if price and priceType == "Limit":
trade_form.select("input#limitPriceTextBox")[0]["value"] = str(price)
elif price and priceType == "Stop":
trade_form.select("input#stopPriceTextBox")[0]["value"] = str(price)
prev_page = br.submit(trade_form, trade_page.url)
prev_form = prev_page.soup.select("form", {"name": "simTradePreview"})
br.submit(prev_form, prev_page.url)
return True
And here is my code that I implement it with:
def buy(shares, ticker, client):
client.trade(ticker,ita.Action.buy, shares)
...
if apred[0] - arl[-1] > 0 and apred[1] - apred[0] > 0 and tickers[0] not in z:
buy(ashr ,tickers[0], client)
Here is the error message:
Traceback (most recent call last):
File "/Users/carson/mlTechnicalAnalysis/investopedia.py", line 146, in <module>
schedule.run_pending()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/schedule/__init__.py", line 493, in run_pending
default_scheduler.run_pending()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/schedule/__init__.py", line 78, in run_pending
self._run_job(job)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/schedule/__init__.py", line 131, in _run_job
ret = job.run()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/schedule/__init__.py", line 411, in run
ret = self.job_func()
File "/Users/carson/mlTechnicalAnalysis/investopedia.py", line 56, in main
buy(ashr ,tickers[0], client)
File "/Users/carson/mlTechnicalAnalysis/investopedia.py", line 16, in buy
client.trade(ticker,ita.Action.buy, shares)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/InvestopediaApi/ita.py", line 240, in trade
prev_form = prev_page.soup.select("form", {"name": "simTradePreview"})
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/bs4/element.py", line 1532, in select
for candidate in _use_candidate_generator(tag):
TypeError: 'dict' object is not callable
I've been beating my head against the wall for a few hours now and I feel like it'll take one of y'all two seconds to figure out. FYI it's an API to made trades on Investopedia's stock trading simulator.
Thanks!

In short: you're passing one too many arguments to .select(); you must pass a single CSS selector as a string.
Looking at the stack trace:
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/InvestopediaApi/ita.py", line 240, in trade
prev_form = prev_page.soup.select("form", {"name": "simTradePreview"})
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/bs4/element.py", line 1532, in select
for candidate in _use_candidate_generator(tag):
TypeError: 'dict' object is not callable
It appears that what you pass as the second argument to .select() is a dict ({"name": "simTradePreview"}). Apparently it's not expected.
From the stacktrace it appears that the soup in question is BeautifulSoup 4 (bs4); its select does not seem to accept a second argument. But the source does have more undocumented arguments with default vlues, in particular, _candidate_generator, which you clobber.

python : error handling Ordered dict with unicode data

My script migrates data from MySQL to mongodb. It runs perfectly well when there are no unicode columns included. But throws me below error when OrgLanguages column is added.
mongoImp = dbo.insert_many(odbcArray)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/collection.py", line 711, in insert_many
blk.execute(self.write_concern.document)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/bulk.py", line 493, in execute
return self.execute_command(sock_info, generator, write_concern)
File "/home/lrsa/.local/lib/python2.7/site-packages/pymongo/bulk.py", line 319, in execute_command
run.ops, True, self.collection.codec_options, bwc)
bson.errors.InvalidStringData: strings in documents must be valid UTF-8: 'Portugu\xeas do Brasil, ?????, English, Deutsch, Espa\xf1ol latinoamericano, Polish'
My code:
import MySQLdb, MySQLdb.cursors, sys, pymongo, collections
odbcArray=[]
mongoConStr = '192.168.10.107:36006'
sqlConnect = MySQLdb.connect(host = "54.175.170.187", user = "testuser", passwd = "testuser", db = "testdb", cursorclass=MySQLdb.cursors.DictCursor)
mongoConnect = pymongo.MongoClient(mongoConStr)
sqlCur = sqlConnect.cursor()
sqlCur.execute("SELECT ID,OrgID,OrgLanguages,APILoginID,TransactionKey,SMTPSpeed,TimeZoneName,IsVideoWatched FROM organizations")
dbo = mongoConnect.eaedw.mysqlData
tuples = sqlCur.fetchall()
for tuple in tuples:
odbcArray.append(collections.OrderedDict(tuple))
mongoImp = dbo.insert_many(odbcArray)
sqlCur.close()
mongoConnect.close()
sqlConnect.close()
sys.exit()
Above script migraates data perfectly when tried without OrgLanguages column in the SELECT query.
To overcome this, I have tried to use the OrderedDict() in another way but gives me a different type of error
Changed Code:
for tuple in tuples:
doc = collections.OrderedDict()
doc['oid'] = tuple.OrgID
doc['APILoginID'] = tuple.APILoginID
doc['lang'] = unicode(tuple.OrgLanguages)
odbcArray.append(doc)
mongoImp = dbo.insert_many(odbcArray)
Error Received:
Traceback (most recent call last):
File "pymsql.py", line 19, in <module>
doc['oid'] = tuple.OrgID
AttributeError: 'dict' object has no attribute 'OrgID'

Your MySQL connection is returning characters in a different encoding than UTF-8, which is the encoding that all BSON strings must be in. Try your original code but pass charset='utf8' to MySQLdb.connect.

Adding Item data to a DynamoDB table using boto does not work

I have been trying to add items to a DynamoDB table using boto, but somehow it doesn't seem to work. I tried using users.Item() and users.put_item but nothing worked. Below is the script that I have in use.
import boto.dynamodb2
import boto.dynamodb2.items
import json
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex
from boto.dynamodb2.layer1 import DynamoDBConnection
from boto.dynamodb2.table import Table
from boto.dynamodb2.items import Item
from boto.dynamodb2.types import NUMBER
region = "us-east-1"
con = boto.dynamodb2.connect_to_region(region)
gettables = con.list_tables()
mytable = "my_table"
if mytable not in gettables['TableNames']:
print "The table *%s* is not in the list of tables created. A new table will be created." % req_table
Table.create(req_table,
schema = [HashKey('username'),
RangeKey('ID', data_type = NUMBER)],
throughput = {'read': 1, 'write': 1})
else:
print "The table *%s* exists." % req_table
con2table = Table(req_table,connection=con)
con2table.put_item(data={'username': 'abcd',
'ID': '001',
'logins':'10',
'timeouts':'20'
'daysabsent': '30'
})
I tried this, the table gets created and it is fine. But when I try to put in the items, I get the following error message.
Traceback (most recent call last):
File "/home/ec2-user/DynamoDB_script.py", line 29, in <module>
'daysabsent':'30'
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 821, in put_item
return item.save(overwrite=overwrite)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/items.py", line 455, in save
returned = self.table._put_item(final_data, expects=expects)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 835, in _put_item
self.connection.put_item(self.table_name, item_data, **kwargs)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 1510, in put_item
body=json.dumps(params))
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2842, in make_request
retry_handler=self._retry_handler)
File "/usr/lib/python2.7/dist-packages/boto/connection.py", line 954, in _mexe
status = retry_handler(response, i, next_sleep)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2882, in _retry_handler
response.status, response.reason, data)
boto.dynamodb2.exceptions.ValidationException: ValidationException: 400 Bad Request
{u'message': u'One or more parameter values were invalid: Type mismatch for key version expected: N actual: S', u'__type': u'com.amazon.coral.validate#ValidationException'}
Thank you.

From the error message you are getting, it sounds like you are trying to send string values for an attribute that is defined as numeric in DynamoDB.
The specific issue looks to be related to your Range Key ID which is defined as a numeric value N but you are sending it a string value '001'.

Looks like of of the values you are trying to load has empty value.
I got the same error when I was trying to load this. I got exception when partner_name property was a empty string.
try:
item_old = self.table.get_item(hash_key=term)
except BotoClientError as ex:
# if partner alias does not exist then create a new entry!
if ex.message == "Key does not exist.":
item_old = self.table.new_item(term)
else:
raise ex
item_old['partner_code'] = partner_code
item_old['partner_name'] = partner_name
item_old.put()

Cassandra: 'unicode' does not have the buffer interface

I try to use these both prepared statements in my django app:
READINGS = "SELECT * FROM readings"
READINGS_BY_USER_ID = "SELECT * FROM readings WHERE user_id=?"
I query against the db with:
def get_all(self):
query = self.session.prepare(ps.ALL_READINGS)
all_readings = self.session.execute(query)
return all_readings
def get_all_by_user_id(self, user_id):
query = self.session.prepare(ps.READINGS_BY_USER_ID)
readings = self.session.execute(query, [user_id])
return readings
The first of both works pretty well. But the second gives me:
ERROR 2015-07-08 09:42:56,634 | views::exception_handler 47 | ('Unable to complete the operation against any hosts', {<Host: localhost data1>: TypeError("'unicode' does not have the buffer interface",)})
Can anyone tell me what happened here? I understand, that there must be a unicode string somewhere that does not have a buffer interface. But which string is meant? My prepared statement?
Here is the stacktrace in addition:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/rest_framework/views.py", line 448, in dispatch
response = handler(request, *args, **kwargs)
File "/Users/me/Workspace/project/Readings/views.py", line 36, in get_by_user_id
readings = self.tr_dao.get_all_by_user_id(user_id)
File "/Users/me/Workspace/project/Readings/dao.py", line 22, in get_all_by_user_id
readings = self.session.execute(query, [user_id], timeout=60)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/cassandra/cluster.py", line 1405, in execute
result = future.result(timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/cassandra/cluster.py", line 2967, in result
raise self._final_exception

if you are on python 2 this will probably fix it
def get_all_by_user_id(self, user_id):
query = self.session.prepare(ps.READINGS_BY_USER_ID)
readings = self.session.execute(query, [str(user_id)])
return readings

This is not working because your user_id is of type unicode. You can check it using
type(user_id)
If that is the case you should encode it to string:
str(user_id)
It will solve the issue.