i inserting data in to a table but raises the error - python

This is the code
def parse_listings(self, response):
'''
content = ''
with open('incidec1.html', 'r' ) as f:
for line in f.read():
content += line
response = Selector(text=content)
brand = response.css('div[class="fs21"]').css('span').css('a::text').get()
title = response.css('div[class="klavikab lilac"]').css('span::text').get()
t = (id, brand,title)
cur.execute('''INSERT INTO products VALUES(%s,%s,%s,%s,%s)''',(id, brand, title))
mydb.commit()
print('complete.')
if __name__ == '__main__':
incidecoderspider.parse_listings(incidecoderspider,'')'
this the output error:
Traceback (most recent call last):
File "incidecoder1.py", line 98, in <module>
incidecoderspider.parse_listings(incidecoderspider,'')
File "incidecoder1.py", line 80, in parse_listings
cur.execute('''INSERT INTO products VALUES(%s,%s,%s,%s,%s)''',(id, brand, title, photo, ingredients))
AttributeError: 'builtin_function_or_method' object has no attribute 'translate'
i tried very much but can't understanfd what is the problem.

You're trying to use a value that you didn't define: id. Normally, this would get you an "undefined symbol" error. However, id is defined: it's a built-in function.
Simply change the variable name so that you don't collide with a built-in item (such as product_id), and then give that variable a value.

Related

How to get value from JSON - Python

If anyone can help me to get a value from a json response. I intend to get only the first incidence of id where the type is CLIENTREF.
I am reading a list to get some values to do a GET and check if the response is other than null "".
This is the JSON structure:
This is my code so far
def checkParent(list):
for item in list:
cdwParenturl = f"http://cdwu/cdw/counterparties/{item}/?yyyy-mm-dd={dateinplay}"
r = requests.get(cdwParenturl).json()
jsonpath_expression = parse("$..identifier[*]")
# $..identifier[?(#.type == 'CLIENTREF')].id
parentId = []
for match in jsonpath_expression.find(r):
# print(f"match id: {match.value}")
thisdict = match.value
if thisdict["type"] == "CLIENTREF":
# print(thisdict["id"])
parentId.append(thisdict["id"])
elif thisdict["id"] == "":
print(colored(f"The parent {item} does not have ultimateParent", "red"))
print(colored(f"All counterparties have parent", "green"))
checkParent(r_mheu_trade)
print(f "match id: {match.value}") I get this, What I need is the first id related to the type: CLIENTREF
The error that appears on my console is this:
Traceback (most recent call last):
File "h:\DESKTOP\test_check\checkCounterpartie.py", line 114, in <module>
checkParent(r_mheu_trade)
File "h:\DESKTOP\test_check\checkCounterpartie.py", line 106, in checkParent
if thisdict["type"] == "CLIENTREF":
KeyError: 'type'
With the help of pyzer I found a solution to my problem. This is how the code looks like:
# Request to get all UltimateParent
def checkParent(murex):
for item in murex:
cdwParenturl = f"http://cdwu/cdw/counterparties/{item}/?yyyy-mm-dd={dateinplay}"
r = requests.get(cdwParenturl).json()
clientref = r[0]["riskUltimateParent"]["identifier"][1]
if clientref == "":
print(colored(f"The parent {item} does not have ultimateParent", "red"))
else:
print(colored(f"All counterparties have parent", "green"))
checkParent(r_mheu_trade)

'exceptions.TypeError' when executing python script

I'm getting the following error when executing the following script:
Error Type: <type 'exceptions.TypeError'>
Error Contents: 'NoneType' object is not iterable
Traceback (most recent call last):
File "addon.py", line 75, in <module>
plugin.run()
File "xbmcswift2/plugin.py", line 332, in run
items = self._dispatch(self.request.path)
File "/plugin.py", line 306, in _dispatch
listitems = view_func(**items)
File "/addon.py", line 42, in all_episodes
items = thisiscriminal.compile_playable_podcast(playable_podcast)
File "/lib/thisiscriminal.py", line 121, in compile_playable_podcast
for podcast in playable_podcast:
TypeError: 'NoneType' object is not iterable
The code in question is as follows, any advice would be greatly appreciated as I have no idea what I'm doing wrong:
def get_playable_podcast(soup):
"""
#param: parsed html page
"""
r = urllib.urlopen('https://thisiscriminal.com/wp-json/criminal/v1/episodes?posts=10000&page=1')
data = json.loads(r.read().decode('utf-8'))
for post in data['posts']:
print post['title']
print post['episodeNumber']
print post['audioSource']
print post['image']['medium']
subjects = []
item = {
'title': post['title'],
'audioSource': post['audioSource'],
'episodeNumber': post['episodeNumber'],
'medium': post['image']['medium']
}
subjects.append(item)
print subjects
def compile_playable_podcast(playable_podcast):
"""
#para: list containing dict of key/values pairs for playable podcasts
"""
items = []
for podcast in playable_podcast:
items.append({
post['title']: podcast['title']['episodeNumber'],
post['audioSource']: podcast['audioSource'],
post['image']['medium']: podcast['medium'],
'is_playable': True,})
return items
I assume your script does something alike to the following,
podcast = get_playable_podcast(soup)
compiled = compile_playable_podcast(podcast)
The problem is that get_playable_podcast has no return statement. In such a case, Python defaults to returning None - which you then pass into compile_playable_podcast. Since None is not iterable, compile_playable_podcast rightfully raises a TypeError.
Now, the solution is of course to return the podcast list you're building in get_playable_podcast, like so,
def get_playable_podcast(soup):
"""
#param: parsed html page
"""
r = urllib.urlopen('https://thisiscriminal.com/wp-json/criminal/v1/episodes?posts=10000&page=1')
data = json.loads(r.read().decode('utf-8'))
subjects = []
for post in data['posts']:
print post['title']
print post['episodeNumber']
print post['audioSource']
print post['image']['medium']
item = {
'title': post['title'],
'audioSource': post['audioSource'],
'episodeNumber': post['episodeNumber'],
'medium': post['image']['medium']
}
subjects.append(item)
print subjects
return subjects
Beside this, it may be worthwhile to carefully check your script for unused parameters and/or duplicate code.

Adding Item data to a DynamoDB table using boto does not work

I have been trying to add items to a DynamoDB table using boto, but somehow it doesn't seem to work. I tried using users.Item() and users.put_item but nothing worked. Below is the script that I have in use.
import boto.dynamodb2
import boto.dynamodb2.items
import json
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex
from boto.dynamodb2.layer1 import DynamoDBConnection
from boto.dynamodb2.table import Table
from boto.dynamodb2.items import Item
from boto.dynamodb2.types import NUMBER
region = "us-east-1"
con = boto.dynamodb2.connect_to_region(region)
gettables = con.list_tables()
mytable = "my_table"
if mytable not in gettables['TableNames']:
print "The table *%s* is not in the list of tables created. A new table will be created." % req_table
Table.create(req_table,
schema = [HashKey('username'),
RangeKey('ID', data_type = NUMBER)],
throughput = {'read': 1, 'write': 1})
else:
print "The table *%s* exists." % req_table
con2table = Table(req_table,connection=con)
con2table.put_item(data={'username': 'abcd',
'ID': '001',
'logins':'10',
'timeouts':'20'
'daysabsent': '30'
})
I tried this, the table gets created and it is fine. But when I try to put in the items, I get the following error message.
Traceback (most recent call last):
File "/home/ec2-user/DynamoDB_script.py", line 29, in <module>
'daysabsent':'30'
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 821, in put_item
return item.save(overwrite=overwrite)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/items.py", line 455, in save
returned = self.table._put_item(final_data, expects=expects)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/table.py", line 835, in _put_item
self.connection.put_item(self.table_name, item_data, **kwargs)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 1510, in put_item
body=json.dumps(params))
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2842, in make_request
retry_handler=self._retry_handler)
File "/usr/lib/python2.7/dist-packages/boto/connection.py", line 954, in _mexe
status = retry_handler(response, i, next_sleep)
File "/usr/lib/python2.7/dist-packages/boto/dynamodb2/layer1.py", line 2882, in _retry_handler
response.status, response.reason, data)
boto.dynamodb2.exceptions.ValidationException: ValidationException: 400 Bad Request
{u'message': u'One or more parameter values were invalid: Type mismatch for key version expected: N actual: S', u'__type': u'com.amazon.coral.validate#ValidationException'}
Thank you.
From the error message you are getting, it sounds like you are trying to send string values for an attribute that is defined as numeric in DynamoDB.
The specific issue looks to be related to your Range Key ID which is defined as a numeric value N but you are sending it a string value '001'.
Looks like of of the values you are trying to load has empty value.
I got the same error when I was trying to load this. I got exception when partner_name property was a empty string.
try:
item_old = self.table.get_item(hash_key=term)
except BotoClientError as ex:
# if partner alias does not exist then create a new entry!
if ex.message == "Key does not exist.":
item_old = self.table.new_item(term)
else:
raise ex
item_old['partner_code'] = partner_code
item_old['partner_name'] = partner_name
item_old.put()

RuntimeError: maximum recursion depth exceeded (without an explicit recursive call in my Python code)

I am trying to read key-value pairs from an already existing shelf to create a new class object with a updated field and write that class object to a new shelf.
My class object : SongDetails
This is the procedure which fails:
def updateShelfWithTabBody(shelfFileName, newShelfFileName):
"""this function updates songDetails with
html body i.e. just the part that contains lyrics and
chords in the tab """
#read all songDetails
shelf = shelve.open(shelfFileName)
listOfKeys = shelf.keys()
#create new songDetails object
temporaryShelfObject = SongDetails.SongDetails()
#iterate over list of keys
for key in listOfKeys:
#print "name:"+shelf[key].name
#fill details from temporaryShelfObject
temporaryShelfObject.name = shelf[key].name
temporaryShelfObject.tabHtmlPageContent = shelf[key].tabHtmlPageContent
#add new detail information
htmlPageContent = shelf[key].tabHtmlPageContent
temporaryShelfObject.htmlBodyContent = extractDataFromDocument.fetchTabBody(htmlPageContent)
#write SongDetails back to shelf
writeSongDetails.writeSongDetails(temporaryShelfObject, newShelfFileName)
Definitions for functions used in above code:
def fetchTabBody(page_contents):
soup = BeautifulSoup(page_contents)
HtmlBody = ""
try:
#The lyrics and chords of song are contained in div with id = "cont"
#Note: This assumtption is specific to ultimate-guitar.com
HtmlBody = soup.html.body.find("div",{"id":"cont"})
except:
print "Error: ",sys.exc_info()[0]
return HtmlBody
def writeSongDetails(songDetails, shelfFileName):
shelf = shelve.open(shelfFileName)
songDetails.name = str(songDetails.name).strip(' ')
shelf[songDetails.name] = songDetails
shelf.close()
SongDetails class:
class SongDetails:
name = ""
tabHtmlPageContent = ""
genre = ""
year = ""
artist = ""
chordsAndLyrics = ""
htmlBodyContent = ""
scale = ""
chordsUsed = []
This is the error that I get:
Traceback (most recent call last):
File "/l/nx/user/ndhande/Independent_Study_Project_Git/Crawler/updateSongDetailsShelfWithNewAttributes.py", line 69, in <module>
updateShelfWithTabBody(shelfFileName, newShelfFileName)
File "/l/nx/user/ndhande/Independent_Study_Project_Git/Crawler/updateSongDetailsShelfWithNewAttributes.py", line 38, in updateShelfWithTabBody
writeSongDetails.writeSongDetails(temporaryShelfObject, newShelfFileName)
File "/home/nx/user/ndhande/Independent_Study_Project_Git/Crawler/writeSongDetails.py", line 7, in writeSongDetails
shelf[songDetails.name] = songDetails
File "/usr/lib64/python2.6/shelve.py", line 132, in __setitem__
p.dump(value)
File "/usr/lib64/python2.6/copy_reg.py", line 71, in _reduce_ex
state = base(self)
File "/u/ndhande/.local/lib/python2.6/site-packages/BeautifulSoup.py", line 476, in __unicode__
return str(self).decode(DEFAULT_OUTPUT_ENCODING)
**RuntimeError: maximum recursion depth exceeded**
I couldn't find any reason why I'm getting this error even though there is no explicit recursive call in my code. I have seen this error in other stackoverflow posts, but they did have recursive calls in their case.
str(self) calls __str__ or calls __unicode__ calls str(self).

sqlalchemy and how to count row

I have the following code:
query = self.dbObj.session.query(MaterialsTable)
rowCounter = self.dbObj.session.query(MaterialsTable)
for attr , val in interfaceCodesObject.filterNameDict.items():
query = query.filter(and_(getattr(MaterialsTable,attr).like("%%%s%%" % val)))
rowCounter = rowCounter.filter(and_(getattr(MaterialsTable,attr).like("%%%s%%" % val))).count()
query.all()
When i use mutiple field, i get the following traceback:
Traceback (most recent call last):
File "/home/mohsen/codes/amlak/amlak/src/ui/materialsFindFrame.py", line 202, in <lambda>
QtCore.QObject.connect(self.pushButtonSearch, QtCore.SIGNAL(_fromUtf8("clicked()")), lambda: self.interfaceCodesConstructor.responseToRequestForData(self))
File "/home/mohsen/codes/amlak/amlak/src/ui/interface/interface.py", line 109, in responseToRequestForData
self.materialsObejct.findData(self.objectSearchMaterials,self)
File "/home/mohsen/codes/amlak/amlak/src/materials/materials.py", line 133, in findData
rowCounter = rowCounter.filter(and_(getattr(MaterialsTable,attr).like("%%%s%%" % val))).count()
AttributeError: 'long' object has no attribute 'filter'
But when i use just one field it use rowCounter as integer and real return.
My question is , how can i adapt it with multiple field?Such as : Where name = 'barak' and id != 3 But one filed such as : Where name = 'barak'
Do you just want to get the count of final result? In that case, in fact, you just need only one query object instead of both query and rowCounter.
query = self.dbObj.session.query(MaterialsTable)
for attr , val in interfaceCodesObject.filterNameDict.items():
query = query.filter(and_(getattr(MaterialsTable,attr).like("%%%s%%" % val)))
# all result rows
query.all()
# count of rows
rowCounter = query.count()
Hope this helps.

Categories