My main function :
def get_data():
try:
response = send_request_to_get_data()
// will get one dict output looks like :
{
"data": ['some datas.....'],
"next": "api/data?top=100&skip=200",
}
if response.status_code == 200:
if response.json().get("next"):
first_paginated_response = get_paginated_data(response.json().get("next"))
if response.status_code == 200:
if first_paginated_response.json().get("next"):
second_paginated_response = get_paginated_data(response.json().get("next"))
if response.status_code == 200:
if second_paginated_response.json().get("next"):
print('again...again....again....again...again)
def send_request_to_get_data():
return rq.get('https://example.com')
def get_paginated_data(paginated):
url = "https://example.com/{next}".format(next=paginated)
return rq.get(url)
If "next" key is in response, i need to send another request for pagination api, but my if statement looks weird.
What is the good approach for this?
You could use while loop and save the data like this:
response = send_request_to_get_data()
data = response['data']
while response.status_code == 200 and response.json().get("next"):
response = get_paginated_data(response.json().get("next"))
data.extend(response['data'])
Related
So in this unit test file I have tried testing that "WikiVoyage" has a response code of 200 and when I tested that it was fine, it was then when I tried creating a false test it started saying that the response code of 404 was now passing the test, is there any way of testing just the response code of 200
############## python code #############
import requests
import json
used_country_list = [
"Nigeria",
# "South_West_Nigeria",
# "Lagos_State",
# "Lagos_City",
# "Amuwo_odofin",
# # California Detail
# "California",
# "Southern_California",
"Los_Angeles",
# "Los_Angeles/Eastside",
# "Los_Angeles/Downtown",
# "Bay_Area",
# German etc.
"Germany",
"Frankfurt",
]
def wikivoyage_extract(used_country_list):
print(f"[Extractor - WikiVoyage] pre loop")
current_dict = {}
for index, country_name in enumerate(used_country_list):
url = f"http://en.wikivoyage.org/w/api.php?action=query&prop=extracts&titles={country_name}&format=json"
print(f"[Extractor - WikiVoyage] url is {url}")
response = requests.get(url)
content = json.loads(response.content)
print(f"[Extractor - WikiVoyage] response is {response.status_code}")
pages = content['query']['pages']
print(f"{index} : {url}")
for count, key in enumerate(pages):
if count>0:
print(country_name, count)
extract = pages[key]['extract']
current_dict[country_name] = extract
return current_dict
wiki_dict = wikivoyage_extract(used_country_list)
############## unitest code ##############
import unittest
from unit_test import wikivoyage_extract
class ResponseCodeTestCase(unittest.TestCase):
def test_is_wikivoyage_200(self):
self.assertEqual(wikivoyage_extract) == 200
def test_is_wikivoyage_404(self):
self.assertFalse(wikivoyage_extract) == 404
if __name__ == "__main__":
unittest.main()
What they're asking for is SOMETHING like this:
import requests
import json
used_country_list = [
"Nigeria",
"Los_Angeles",
"Germany",
"Frankfurt",
]
def wikivoyage_extract(used_country_list):
print(f"[Extractor - WikiVoyage] pre loop")
current_dict = {}
for index, country_name in enumerate(used_country_list):
url = f"http://en.wikivoyage.org/w/api.php?action=query&prop=extracts&titles={country_name}&format=json"
print(f"[Extractor - WikiVoyage] url is {url}")
response = requests.get(url)
content = json.loads(response.content)
print(f"[Extractor - WikiVoyage] response is {response.status_code}")
assert response.status_code == 200
pages = content['query']['pages']
print(f"{index} : {url}")
for count, key in enumerate(pages):
if count>0:
print(country_name, count)
extract = pages[key]['extract']
current_dict[country_name] = extract
return current_dict
wiki_dict = wikivoyage_extract(used_country_list)
Now, in reality you wouldn't use that exact line, because that will crash the app if the status code is not 200, but you'll need to know what you SHOULD do. Should you return None? Should you log the problem?
...
if response.status_code != 200:
print( "*** Bad status code received" )
return None
I have an issue with my post request. This is my code:
def add_review(request, dealer_id):
if request.method == "GET":
context = {
"cars": CarModel.objects.all().filter(dealerId = dealer_id),
"dealerId": dealer_id
}
return render(request, 'djangoapp/add_review.html', context)
if request.method == "POST":
if request.user.is_authenticated:
form = request.POST
review = {
"dealership": int(dealer_id),
"name": request.user.username,
"review": form["review"],
"purchase": form.get("purchasecheck") == 'on',
}
if form.get("purchasecheck"):
review["purchase_date"] = datetime.strptime(form.get("purchasedate"), "%m/%d/%Y").isoformat()
car = CarModel.objects.get(pk=form["car"])
review["car_make"] = car.make.name
review["car_model"] = car.name
review["car_year"]= int(car.year.strftime("%Y"))
json_payload = {"review": review}
url = "https://4fbfebf7.us-south.apigw.appdomain.cloud/api/review"
post_request(url=url, json_payload=json_payload, dealer_id=dealer_id)
return redirect("djangoapp:dealer_details", dealer_id=dealer_id)
else:
return redirect("/djangoapp/login")
And this:
def post_request(url, json_payload, **kwargs):
json_data = json.dumps(json_payload, indent=4)
print(f"{json_data}")
try:
# Call get method of requests library with URL and parameters
response = requests.post(url, params=kwargs, json=json_data)
except Exception as e:
# If any error occurs
print("Network exception occurred")
print(f"Exception: {e}")
print(f"With status {response.status_code}")
print(f"Response: {response.text}")
I am receiving Response is not valid 'application/json' error as you can see here.
Meanwhile, when I copy the exact same JSON to IBM Cloud to test my APIs, all is working fine and the record is created as you can see here.
I guess it's a very silly mistake, but where?
When you pass json to request.post it should be a serializable object (not already serialized)
def post_request(url, json_payload, **kwargs):
# json_data = json.dumps(json_payload, indent=4). << delete
print(f"{json_payload}")
try:
# Call get method of requests library with URL and parameters
response = requests.post(url, params=kwargs, json=json_payload)
except Exception as e:
# If any error occurs
print("Network exception occurred")
print(f"Exception: {e}")
print(f"With status {response.status_code}")
print(f"Response: {response.text}")
I grab the site-key using a some-what hardcoded method, but since the len is always the same it's ok.
After that I use the 2captcha API Documentation so that I can POST the key and GET the token for the captcha back. I face two major problems:
1) I always get the site-key wrong error, but the sitekey is correct according to their example (their sitekey is 40chars long and my sitekeys are 40 too)
2) I tried creating a POST function externally and trying it out seeing if it is a bug, but using the Py2Captcha documentation I always get the following error:
This is captcha key grabbing.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Captcha Key~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
url=driver.find_element_by_css_selector("iframe[role='presentation']").get_attribute('src')
print(url)
keygoogle = url[53:93]
print('Site Key = ', keygoogle)
This is the captcha key solving block:
answer = ""
answer_id = 0
api_key = '--------------------------------'
data_post = {'key': api_key, 'method': 'userrecaptcha', 'googlekey': keygoogle, "pageurl": mainurl}
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
print(response)
print("Waiting for server response.")
for x in range(15):
time.sleep(1)
if x == 5:
print('Downloading info..')
elif x == 10:
print('Processing info..')
elif x == 14:
print('Solving captcha..')
data_request = {
'key': api_key,
'action': answer,
'id': answer_id,
}
requests.get(url ='https://2captcha.com/res.php', data=data_request)
print(answer)
def captcha():
google_key = keygoogle
url = mainurl
client = TwoCaptchaClient(client_key=api_key)
task = GoogleReCaptchaV2Task(googlekey=google_key, pageurl=mainurl)
job = client.create_task(task)
token = job.get_solution_response()
return token
print(captcha())
What I haven't included is the part where the token gets posted into the answer field, I am not sure how to do that yet but I will find a way for sure!
EDIT:
This is the value I get from printing this:
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
And this is the value I get from print('Site Key = ', keygoogle)
Site Key = Lc3HAsUAAAAACsN7CgY9MMVxo2M09n_e4heJEiZ&
This is my way of grabbing the correct key:
url=driver.find_element_by_css_selector("iframe[role='presentation']").get_attribute('src')
keygoogle = url[52:92]
And this is my POST function:
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~Captcha Solve~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
answer = ""
answer_id = 0
data_post = {'key': api_key, 'method': 'userrecaptcha', 'googlekey': keygoogle, "pageurl": mainurl}
response = requests.post(url = 'https://2captcha.com/in.php', data = data_post )
response = response.text[3:]
print("Waiting for server response.")
for x in range(30):
time.sleep(1)
if x == 8:
print('Downloading info..')
elif x == 15:
print('Processing info..')
data_request = {'key': api_key,'id': int(response),'action': 'get'}
response = requests.get(url='https://2captcha.com/res.php', params=data_request)
token = response.text.split('|')[0]
while response.text == 'CAPCHA_NOT_READY':
print('Waiting for Capcha..')
time.sleep(5)
response = requests.get(url='https://2captcha.com/res.php', params=data_request)
token = response
print(token)
The & on the end of that sitekey definitely isn't supposed to be there. Use a regex instead of what you're doing there with the indexes.
I am trying to create and populate a playlist using the Spotify Web API. I am following this official reference, and I am using Python 3 with the requests module. Here is my code:
def spotify_write_playlist(auth, name, tracks, public=True):
ids = []
for track in tracks:
track_id = track.services['spotify']
if track_id: ids.append(track_id)
headers = {
"authorization":"Bearer " + auth.token,
"content-type":"application/json"
}
data = {
"name":name,
"public":public
}
r = makeRequest("https://api.spotify.com/v1/users/" + auth.username + "/playlists", "post", 201, json=data, headers=headers)
playlist_id = json.loads(r.content)['id']
data = {"uris":ids}
r = makeRequest("https://api.spotify.com/v1/users/" + auth.username + "/playlists/" + playlist_id + "/tracks", "post", 201, json=data, headers=headers)
return playlist_id
def makeRequest(url, method="get", expectedCode=200, *args, **kwargs):
while True:
r = requests.request(method, url, **kwargs)
if r.status_code == 429:
time.sleep(TMR_DELAY)
continue
elif r.status_code == expectedCode:
return r
else:
if "spotify.com" in url:
raise spotify.ApiError(r.status_code, expectedCode, r.content)
else:
raise youtube.ApiError(r.status_code, expectedCode, r.content)
The makeRequest function is a wrapper around requests.request that handles ratelimiting.
The above code when run with a bunch of sample tracks returns an error 400 at the first call of makeRequest, so my sample tracks can't be the issue as only the name and public variables are involved with that call.
The error response has no body, so no description of the error. This suggests I am probably missing something very obvious. Can anyone help?
The playlist creation request stopped erroring, and I have no idea why. I must of changed whatever was wrong when trying to diagnose the problem. I thought it was trying to create a playlist of the same name but spotify allows duplicate playlists. Maybe something was wrong with the token or token object I was using. Sorry.
However, I also found another issue when the previous one was cleared: the when adding to tracks you specify the track uri (spotify:track:abcdef123) not just the track id (abcdef123). I have revised spotify_write_playlists below:
def spotify_write_playlist(auth, name, tracks, public=True):
ids = []
for track in tracks:
track_id = track.services['spotify']
if track_id: ids.append("spotify:track:"+track_id)
headers = {
"authorization":"Bearer " + auth.token,
"content-type":"application/json"
}
data = {
"name":name,
"public":public
}
r = makeRequest("https://api.spotify.com/v1/users/" + auth.username + "/playlists", "post", 201, json=data, headers=headers)
playlist_id = json.loads(r.content)['id']
data = {"uris":ids}
r = makeRequest("https://api.spotify.com/v1/users/" + auth.username + "/playlists/" + playlist_id + "/tracks", "post", 201, json=data, headers=headers)
return playlist_id
I am getting an error while using Request object in Python.
Below is my code.
class APIDOC(Document):
def request_api(self):
method_type = self.method_type
api = self.api
parameters = self.parameters
session_object = requests.session()
self.get_login(session_object)
if method_type == "POST":
data = {}
for param in parameters:
data[param.key] = param.value
response = session_object.post(api,data)
if response.status_code == 200:
return response.text
else:
return "Error while getting response error code:{0}".format(response.status_code)
elif method_type == "GET":
data = {}
for param in parameters:
data[param.key] = param.value
print("____________________________",data)
response = session_object.get(api,data)
if response.status_code == 200:
return response.text
else:
return "Error while getting response error code:{0}".format(response.status_code)
After reffering one document on requests in python I found below things for "GET" method
r = requests.get('http://httpbin.org/get', params=payload)
But on executing the same I got an error
response = session_object.get(api,data)
TypeError: get() takes exactly 2 arguments (3 given)
To send parameters with a GET, you'll need to specify them by keyword:
session_object.get(api, params=data)