We Keep Coding

Amazon Product Advertising API 5.0 Error: "Code":"TooManyRequests","Message":"The request was denied due to request throttling

I am trying to get product information through the Amazon Product Advertising API 5.0 but I keep getting the same error:
Error calling PA-API 5.0!
Status code: 429
Errors : {"__type":"com.amazon.paapi5#TooManyRequestsException","Errors":[{"Code":"TooManyRequests","Message":"The request was denied due to request throttling. Please verify the number of requests made per second to the Amazon Product Advertising API."}]}
Request ID: ee3f3ad0-e512-463f-b34b-395af81e347d
I followed the instructions in the docs, but I can't seem to solve the problem.
I downloaded the python example package and used the sample_get_items_api.py script:
from paapi5_python_sdk.api.default_api import DefaultApi
from paapi5_python_sdk.models.condition import Condition
from paapi5_python_sdk.models.get_items_request import GetItemsRequest
from paapi5_python_sdk.models.get_items_resource import GetItemsResource
from paapi5_python_sdk.models.partner_type import PartnerType
from paapi5_python_sdk.rest import ApiException
def parse_response(item_response_list):
"""
The function parses GetItemsResponse and creates a dict of ASIN to Item object
:param item_response_list: List of Items in GetItemsResponse
:return: Dict of ASIN to Item object
"""
mapped_response = {}
for item in item_response_list:
mapped_response[item.asin] = item
return mapped_response
def get_items():
""" Following are your credentials """
""" Please add your access key here """
access_key = "<YOUR ACCESS KEY>"
""" Please add your secret key here """
secret_key = "<YOUR SECRET KEY>"
""" Please add your partner tag (store/tracking id) here """
partner_tag = "<YOUR PARTNER TAG>"
""" PAAPI host and region to which you want to send request """
""" For more details refer: https://webservices.amazon.com/paapi5/documentation/common-request-parameters.html#host-and-region"""
host = "webservices.amazon.com"
region = "us-east-1"
""" API declaration """
default_api = DefaultApi(
access_key=access_key, secret_key=secret_key, host=host, region=region
)
""" Request initialization"""
""" Choose item id(s) """
item_ids = ["059035342X", "B00X4WHP5E", "B00ZV9RDKK"]
""" Choose resources you want from GetItemsResource enum """
""" For more details, refer: https://webservices.amazon.com/paapi5/documentation/get-items.html#resources-parameter """
get_items_resource = [
GetItemsResource.ITEMINFO_TITLE,
GetItemsResource.OFFERS_LISTINGS_PRICE,
]
""" Forming request """
try:
get_items_request = GetItemsRequest(
partner_tag=partner_tag,
partner_type=PartnerType.ASSOCIATES,
marketplace="www.amazon.com",
condition=Condition.NEW,
item_ids=item_ids,
resources=get_items_resource,
)
except ValueError as exception:
print("Error in forming GetItemsRequest: ", exception)
return
try:
""" Sending request """
response = default_api.get_items(get_items_request)
print("API called Successfully")
print("Complete Response:", response)
""" Parse response """
if response.items_result is not None:
print("Printing all item information in ItemsResult:")
response_list = parse_response(response.items_result.items)
for item_id in item_ids:
print("Printing information about the item_id: ", item_id)
if item_id in response_list:
item = response_list[item_id]
if item is not None:
if item.asin is not None:
print("ASIN: ", item.asin)
if item.detail_page_url is not None:
print("DetailPageURL: ", item.detail_page_url)
if (
item.item_info is not None
and item.item_info.title is not None
and item.item_info.title.display_value is not None
):
print("Title: ", item.item_info.title.display_value)
if (
item.offers is not None
and item.offers.listings is not None
and item.offers.listings[0].price is not None
and item.offers.listings[0].price.display_amount is not None
):
print(
"Buying Price: ",
item.offers.listings[0].price.display_amount,
)
else:
print("Item not found, check errors")
if response.errors is not None:
print("\nPrinting Errors:\nPrinting First Error Object from list of Errors")
print("Error code", response.errors[0].code)
print("Error message", response.errors[0].message)
except ApiException as exception:
print("Error calling PA-API 5.0!")
print("Status code:", exception.status)
print("Errors :", exception.body)
print("Request ID:", exception.headers["x-amzn-RequestId"])
except TypeError as exception:
print("TypeError :", exception)
except ValueError as exception:
print("ValueError :", exception)
except Exception as exception:
print("Exception :", exception)
Anybody know how to get it to work?

pytest will not raise exception

I can not get the exception to throw on this unit test.
def test_something(monkeypatch):
# Arrange
os.environ["ENTITY"] = "JURKAT" # should monkeypatch this, but ignore for now.
opgave_or_sag_id = "S7777777"
url = "https://testdriven.io"
auth = ("test", "test")
tx = 999999999
patch_called = False
def mock_get(*args, **kwargs):
nonlocal patch_called
patch_called = True
return MockResponseGet410()
monkeypatch.setattr(Session, "get", mock_get)
# Act
with pytest.raises(Exception) as exc_info:
dsu.fetch_entity_from_green(opgave_or_sag_id, url, auth, tx)
# Assert
assert patch_called
assert exc_info.value.args[0] == "Expected status code 200 (or 410 with invalid opgaver), but got 410 for entity JURKAT. Error: Invalid opgave - It's gone - https://testdriven.io"
class MockResponseGet410:
def __init__(self):
self.status_code = 410
self.text = "the opgave has status: INVALID, it's gone now."
self.reason = "It's gone"
self.url = "https://testdriven.io"
self.headers = {
"Content-Type": "application/json",
"Content-Length": "123",
"Content-Location": "tx=123",
}
# From dsu
def fetch_entity_from_green(opgave_or_sag_id, url, auth, tx):
"""Retrieve missing entity from green's api.
Parameters
----------
opgave_or_sag_id : string, required
tx : int, required
A tx number.
url : str, required
auth : AWSAuthObject, required
Returns
-------
entity : dict
A dict representing the entity from green.
status_code : int
"""
try:
ENTITY = os.environ["ENTITY"]
url_with_id = url + str(opgave_or_sag_id)
s = fetch_request_session_with_retries()
r = s.get(url_with_id, auth=auth)
handle_non_200_response_for_invalid_opgaver(r, opgave_or_sag_id, ENTITY)
# I deleted the rest, not relevant in this test as the above function should throw an exception.
except Exception as e:
print(f"An exception occured on request with id {opgave_or_sag_id}: {e}")
The exception should be thrown in handle_non_200_response_for_invalid_opgaver because mock_get returns a 410 status code and ENTITY is set to JURKAT:
def handle_non_200_response_for_invalid_opgaver(request, opgave_or_sag_id, ENTITY):
"""
Handles a non-200 response from the API but allows 410 responses on invalid opgaver.
"""
# 410 because Team Green returns this for invalid opgaver, which becomes a valid response.
if request.status_code != 200 and (
request.status_code == 410 and ENTITY != "OPGAVER"
):
print(f"Status code for request on {opgave_or_sag_id}: {request.status_code}")
raise Exception( # TODO be more explicit with exception.
f"Expected status code 200 (or 410 with invalid opgaver), but got {request.status_code} for entity {ENTITY}. Error: {request.text} - {request.reason} - {request.url}"
)
I can get an exception to throw using pytest.raises(Exception) in a different test (see below), and the test passes, so I'm on the right track:
def test_handle_non_200_response():
# Arrange
r = MockResponse()
# Act
with pytest.raises(Exception) as exc_info:
handle_non_200_response(r)
# Assert
assert (
exc_info.value.args[0]
== "Expected status code 200, but got 504. Error: Gateway Timeout - Exceeded 30 seconds - https://testdriven.io"
)
class MockResponse:
def __init__(self):
self.status_code = 504
self.text = "Gateway Timeout"
self.reason = "Exceeded 30 seconds"
self.url = "https://testdriven.io"
def json(self):
return {"id": 1}
def handle_non_200_response(request):
"""
Handles a non-200 response from the API.
"""
if request.status_code != 200:
print(f"Status code for request on {id}: {request.status_code}")
raise Exception(
f"Expected status code 200, but got {request.status_code}. Error: {request.text} - {request.reason} - {request.url}"
)
Can you see where I have gone astray?

Is there a way to avoid Rate Limit Error in Twitter API in my python program?

I am trying to create a Twitter user graph and for that I have written the following code :
import operator
import sys
import time
from urllib.error import URLError
from http.client import BadStatusLine
import json
import twitter
from functools import partial
from sys import maxsize as maxint
import itertools
import networkx
import matplotlib.pyplot as plt
G = networkx.Graph()
# Code and function taken from the twitter cookbook
def oauth_login():
CONSUMER_KEY = 'xxxx'
CONSUMER_SECRET = 'xxZD6r'
OAUTH_TOKEN = 'xxNRYl'
OAUTH_TOKEN_SECRET = 'xxHYJl'
auth = twitter.oauth.OAuth(OAUTH_TOKEN, OAUTH_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET)
twitter_api = twitter.Twitter(auth=auth)
return twitter_api
# Code and function taken from the twitter cookbook
def make_twitter_request(twitter_api_func, max_errors=10, *args, **kw):
# A nested helper function that handles common HTTPErrors. Return an updated
# value for wait_period if the problem is a 500 level error. Block until the
# rate limit is reset if it's a rate limiting issue (429 error). Returns None
# for 401 and 404 errors, which requires special handling by the caller.
def handle_twitter_http_error(e, wait_period=2, sleep_when_rate_limited=True):
if wait_period > 3600: # Seconds
print('Too many retries. Quitting.', file=sys.stderr)
raise e
if e.e.code == 401:
print('Encountered 401 Error (Not Authorized)', file=sys.stderr)
return None
elif e.e.code == 404:
print('Encountered 404 Error (Not Found)', file=sys.stderr)
return None
elif e.e.code == 429:
print('Encountered 429 Error (Rate Limit Exceeded)', file=sys.stderr)
if sleep_when_rate_limited:
print("Retrying in 15 minutes...ZzZ...", file=sys.stderr)
sys.stderr.flush()
time.sleep(60 * 15 + 5)
print('...ZzZ...Awake now and trying again.', file=sys.stderr)
return 2
else:
raise e # Caller must handle the rate limiting issue
elif e.e.code in (500, 502, 503, 504):
print('Encountered {0} Error. Retrying in {1} seconds'.format(e.e.code, wait_period), file=sys.stderr)
time.sleep(wait_period)
wait_period *= 1.5
return wait_period
else:
raise e
wait_period = 2
error_count = 0
while True:
try:
return twitter_api_func(*args, **kw)
except twitter.api.TwitterHTTPError as e:
error_count = 0
wait_period = handle_twitter_http_error(e, wait_period)
if wait_period is None:
return
except URLError as e:
error_count += 1
time.sleep(wait_period)
wait_period *= 1.5
print("URLError encountered. Continuing.", file=sys.stderr)
if error_count > max_errors:
print("Too many consecutive errors...bailing out.", file=sys.stderr)
raise
except BadStatusLine as e:
error_count += 1
time.sleep(wait_period)
wait_period *= 1.5
print("BadStatusLine encountered. Continuing.", file=sys.stderr)
if error_count > max_errors:
print("Too many consecutive errors...bailing out.", file=sys.stderr)
raise
# Code and function taken from the twitter cookbook
def get_friends_followers_ids(twitter_api, screen_name=None, user_id=None,
friends_limit=maxint, followers_limit=maxint):
# Must have either screen_name or user_id (logical xor)
assert (screen_name is not None) != (user_id is not None), "Must have screen_name or user_id, but not both"
# See https://developer.twitter.com/en/docs/twitter-api/v1/accounts-and-users/follow-search-get-
#users/api-reference/get-friends-ids for details
# on API parameters
get_friends_ids = partial(make_twitter_request, twitter_api.friends.ids, count=5000)
get_followers_ids = partial(make_twitter_request, twitter_api.followers.ids, count=5000)
friends_ids, followers_ids = [], []
for twitter_api_func, limit, ids, label in [
[get_friends_ids, friends_limit, friends_ids, "friends"],
[get_followers_ids, followers_limit, followers_ids, "followers"]
]:
if limit == 0: continue
cursor = -1
while cursor != 0:
# Use make_twitter_request via the partially bound callable...
if screen_name:
response = twitter_api_func(screen_name=screen_name, cursor=cursor)
else: # user_id
response = twitter_api_func(user_id=user_id, cursor=cursor)
if response is not None:
ids += response['ids']
cursor = response['next_cursor']
print('Fetched {0} total {1} ids for {2}'.format(len(ids), label, (user_id or screen_name)),
file=sys.stderr)
# XXX: You may want to store data during each iteration to provide an
# an additional layer of protection from exceptional circumstances
if len(ids) >= limit or response is None:
break
# Do something useful with the IDs, like store them to disk...
return friends_ids[:friends_limit], followers_ids[:followers_limit]
# Code and function taken from the twitter cookbook
def get_user_profile(twitter_api, screen_names=None, user_ids=None):
# Must have either screen_name or user_id (logical xor)
assert (screen_names is not None) != (user_ids is not None)
items_to_info = {}
items = screen_names or user_ids
while len(items) > 0:
items_str = ','.join([str(item) for item in items[:100]])
items = items[100:]
if screen_names:
response = make_twitter_request(twitter_api.users.lookup, screen_name=items_str)
else: # user_ids
response = make_twitter_request(twitter_api.users.lookup, user_id=items_str)
for user_info in response:
if screen_names:
items_to_info[user_info['screen_name']] = user_info
else: # user_ids
items_to_info[user_info['id']] = user_info
return items_to_info
# Function to find reciprocal friends and sort them such that we get the top 5 friends
def reciprocal_friends(twitter_api, screen_name=None, user_id=None):
friends_list_ids, followers_list_ids = get_friends_followers_ids(twitter_api, screen_name=screen_name,
user_id=user_id,
friends_limit=5000, followers_limit=5000)
friends_reciprocal = list(set(friends_list_ids) & set(followers_list_ids))
list_followers_count = []
user_profiles = {}
for each in friends_reciprocal:
user_profiles[each] = get_user_profile(twitter_api, user_ids=[each])[each]
list_followers_count.append(user_profiles[each]['followers_count'])
res = sorted(list_followers_count, reverse=True)
friends_count = {user_profiles[fr]['followers_count']: fr for fr in friends_reciprocal}
list_resciprocal = []
if len(res) < 6:
list_resciprocal = friends_reciprocal
else:
for i in range(5):
list_resciprocal.append(friends_count[res[i]])
return list_resciprocal
# This function finds reciprocal friends again and again till we achieve at least 100 nodes
def crawler(twitter_api, screen_name=None, user_id=None):
rec_friends = reciprocal_friends(twitter_api, screen_name=screen_name, user_id=user_id)
edges = [(screen_name, x) for x in rec_friends]
G.add_edges_from(edges)
nodes = nxt_qu = rec_friends
if len(nodes) == 0:
print("No reciprocal friends")
return rec_friends
while G.number_of_nodes() < 101:
print("Queue Items : ", nxt_qu)
(queue, nxt_qu) = (nxt_qu, [])
for q in queue:
if G.number_of_nodes() >= 101:
break
print("ID Entered:", q)
res = reciprocal_friends(twitter_api, screen_name=None, user_id=q)
edges = [(q, z) for z in res]
G.add_edges_from(edges)
nxt_qu += res
nodes += res
print(nodes)
# To Plot the graph
networkx.draw(G)
plt.savefig("graphresult.png")
plt.show()
# Printing the Output
print("No. of Edges: ", G.number_of_edges())
print("No. of Nodes: ", G.number_of_nodes())
print("Diameter : ", networkx.diameter(G))
print("Average Distance: ", networkx.average_shortest_path_length(G))
# To write the output into a file
f = open("output.txt", "w")
f.write("No. of Nodes: " + str(G.number_of_nodes()))
f.write("\nNo. of Edges: " + str(G.number_of_edges()))
f.write("\nDiameter: " + str(networkx.diameter(G)))
f.write("\nAverage Distance: " + str(networkx.average_shortest_path_length(G)))
twitter_api = oauth_login()
crawler(twitter_api, screen_name="POTUS")
However I am getting this error often and this is making my program run very slow
ID Entered: 60784269
Fetched 5000 total friends ids for 60784269
Fetched 5000 total followers ids for 60784269
Encountered 429 Error (Rate Limit Exceeded)
Retrying in 15 minutes...ZzZ...
Is there a way to get around this ? Make the code run faster ?
I have read a few documents but I still dont have any clear picture. Any help is appreciated.

There is no way to go around the rate limits restrictions with the Public API.
Though there is an API v2 now which also allow you to get users and do not work against the same rate limits.
https://developer.twitter.com/en/docs/twitter-api/users/lookup/introduction
Notice that this solution would be temporary as Twitter will at some point remove access to API v1.
You can request twitter to have access to premium/enterprise level of the API but you will have to pay for that.
You can see rate limits documentation here :
API v1
API v2

Mocking a 500 response when an operation is performed

This is my test so far:
test_500(self):
client = ClientConfiguration(token=token, url=url)
client.url = 'https://localhost:1234/v1/' + bucket
keys = None
try:
get_bucket = json.loads(str(client.get_bucket(bucket)))
result = get_bucket['result']
except Exception as e:
expected_status_code = 500
failure_message = "Expected status code %s but got status code %s" % (expected_status_code, e)
self.assertEquals(e, expected_status_code, failure_message)
I need to write a mock that will return a 500 response when the 'https://localhost:1234/v1/' + bucket url is used. Can this be done with unittest and if so, how or where can I find some documentation on this? I've been through this site, the unittest documentation and Youtube and can't find anythingspecific to what I want to do.

I ended up using this to create my test.
The end result is:
#responses.activate
test_500(self):
responses.add(responses.GET, 'https://localhost:1234/v1/' + bucket,
json={'error': 'server error'}, status=500)
client = ClientConfiguration(token=token, url=url)
client.url = 'https://localhost:1234/v1/'
keys = None
try:
get_bucket = json.loads(str(client.get_bucket(bucket)))
result = get_bucket['result']
except Exception as e:
expected_status_code = 500
failure_message = "Expected status code %s but got status code %s" % (expected_status_code, e)
self.assertEquals(e, expected_status_code, failure_message)

Exception not raised assertRaises()?

I am giving a wrong input and I want an exception to be raised. Somehow this is not happening. This is my unit test code:
def test_invalid_bag_json_conversion_1(self):
file_name = "../test/test_files/wrong_bag.bag"
ru = RosbagUploader(file_name, self.json_output, "", "", "", "")
status = ru.start()
self.assertRaises(Exception, RosbagUploader, file_name, self.json_output, "", "", "", "")
self.assertEquals(ReturnCodes.FAIL, status)
and my code that I am testing:
class RosbagUploader(object):
"""
#brief Uploads deserialized input Rosbag file to ElasticSearch or
stores it locally
"""
def __init__(self, _rosbag_filepath, _json_filename, _es_addr, _es_index,
_es_type, _buffer_size):
self.overall_status = ReturnCodes.SUCCESS
self._rosbag_filepath = _rosbag_filepath
self._json_filename = _json_filename
self._buffer_size = _buffer_size if _buffer_size > 0 else 5000
self._es_type = _es_type
self._es_addr = _es_addr
self._es_index = _es_index
self._es_buff = []
try:
self._rosbag = rosbag.Bag(_rosbag_filepath, "r")
if self._es_addr:
self._es = Elasticsearch() if _es_addr == "" else \
Elasticsearch([_es_addr])
self._total_num_record = self._rosbag.get_message_count()
except:
print("[ERROR] {}".format(sys.exc_info()))
self.overall_status = ReturnCodes.FAIL
It shows the output that the exception is raised as below:
[ERROR] (<class 'rosbag.bag.ROSBagException'>, ROSBagException(), <traceback object at 0x7fdcb463e8c0>)
EException AttributeError: "'RosbagUploader' object has no attribute '_rosbag'" in <bound method RosbagUploader.__del__ of <rosbag_deserializer_core.RosbagUploader object at 0x7fdcb4899ad0>> ignored
Which is what it should do. But why doesnt it raise the exception?

You are essentially ignoring the exception since you are not re-raising after the print statement:
try:
[...]
except:
print("[ERROR] {}".format(sys.exc_info()))
self.overall_status = ReturnCodes.FAIL
You need to re-raise for callers to receive the exception:
try:
[...]
except:
print("[ERROR] {}".format(sys.exc_info()))
self.overall_status = ReturnCodes.FAIL
raise

Do you have a __del__ destruction method? Seems to be failing there.