keyerror when adding key in dict - python

I cleaned some keys of a dictionary and tried to add them into a new dict, so i can only work with them. But when i try to encode and decode keys such as Radaufhängung or Zündanlage and add them into the new dict i get an error. My question is if there is a way to go around thir or if there is a better solution to handle this (line: 49)?
my code:
import requests
import json
import time
from requests.exceptions import HTTPError
attempts = 0
def get_data_from_url(url):
try:
response = requests.get(url)
# If the response was successful, no Exception will be raised
response.raise_for_status()
except HTTPError:
return "HTTPError"
else:
response_dict = json.loads(response.text)
return response_dict
url = get_data_from_url("http://160.85.252.148/")
#(1) upon no/invalid response, the query is resubmitted with exponential backoff waiting time in between requests to avoid server overload;
while url == "HTTPError":
attempts += 1
time.sleep(attempts * 1.5)
url = get_data_from_url("http://160.85.252.148/")
print(url)
#(2) material records with missing or invalid cost are ignored;
valid_values = {}
for key in url:
if type(url[key]) == int or type(url[key]) == float and url[key].isdigit() == True:
#(3) wrongly encoded umlauts are repaired.
key = key.encode('latin1').decode('utf8')
#key = key.replace('é', 'Oe').replace('ä', 'ae').replace('ü', 'ue')
valid_values[key]=abs(url[key])
print(valid_values)


You're trying to access the dictionary with a modified key. Store the original key and use it when accessing the dictionary:
okey = key
key = key.encode('latin1').decode('utf8')
...
valid_values[key]=abs(url[okey])

Related

Python API script

I am making a python script using API of a free test automation website called TestProject.
Link to their API: https://api.testproject.io/docs/v2/
Basically what i want to do is grab pdf of reports of all tests and save them somewhere.
But to make the GET request to do that i first need projectID and jobID which i already wrote functions getting them and saving them in the array.
But now i have a problem where its looping through both lists and not using correct projectID and jobID and its throwing errors because it does not exist.
So what i need is something to check if jobID is in projectID so that way i can make a GET request to get all the executionID's to get the PDF of the report.
I am kinda new to programming so i would love any help i can get. If anyone has any better solutions please feel free to let me know.
My script:
import requests
import json
import csv
from datetime import datetime
from jsonpath_ng import jsonpath, parse
API_key = 'api_key'
headers = {'Authorization':'{}'.format(API_key)}
list_projectId = []
list_jobId = []
list_executionId = []
ParseData_projectId = parse('$..id')
ParseData_jobId = parse('$..id')
ParseData_executionId = parse('$..id')
def parsing (response,ParseData,list_data):
# parses data and appends it to the list
Data = json.loads(response)
Parsaj = ParseData
Podatki = Parsaj.find(Data)
for i in range(0, len(Podatki)):
vrednost = Podatki[i].value
list_data.append(vrednost)
def projectId():
# gets all projectId's and saves them in list_projectId
url = 'https://api.testproject.io/v2/projects?_start=0'
response = requests.get(url,headers=headers)
response_json = response.json()
converted = json.dumps(response_json)
parsing(converted,ParseData_projectId,list_projectId)
def jobId():
# gets all jobId's and saves them in list_jobId
for i in range(0, len(list_projectId)):
id = list_projectId[i]
url = 'https://api.testproject.io/v2/projects/{}'.format(id) + '/jobs?onlyScheduled=false&_start=0'
response = requests.get(url,headers=headers)
response_json = response.json()
converted = json.dumps(response_json)
parsing(converted,ParseData_jobId,list_jobId)
def executionId():
# Their API link:
# https://api.testproject.io/v2/projects/{projectId}/jobs/{jobId}/reports?_start=0
# the for loop below does not work here is where i need the help:
for i in range(0, len(list_projectId)):
project_id = list_projectId[i]
job_id = list_jobId[i]
url = 'https://api.testproject.io/v2/projects/{}'.format(project_id) + '/jobs/{}'.format(job_id) + '/reports?_start=0'
response = requests.get(url,headers=headers)
response_json = response.json()
converted = json.dumps(response_json)
parsing(converted,ParseData_executionId,list_executionId)
projectId()
print("----------LIST PROJECT ID: ----------")
print(list_projectId)
print("")
jobId()
print("----------LIST JOB ID: ----------")
print(list_jobId)
executionId()
print("----------LIST EXECUTION ID: ----------")
print(list_executionId)

you have to use 'in' operator to check the value exist in the list data structure.

REST API requests - Use Concurrent.futures in the right way

The code below is a sample from my complete program, I tried it to make understandable.
It sends requests to a REST API. It starts with an URL and the number of pages for this specific search and tries to catch the content for each page.
Each page has several results. Each result becomes a FinalObject.
Because there are as many API requests as there are pages, I decided to use multi-threading and the concurrent.futures module.
=> It works but, as I'm new in coding and Python, I still have these 2 questions:
How to use ThreadPoolExecutor sequentially in this case,
Is there a better way to handle multi-threading in this case?
from concurrent.futures import ThreadPoolExecutor
from requests import get as re_get
def main_function(global_page_number, headers, url_request):
# create a list of pages number
pages_numbers_list = [i for i in range(global_page_number)]
# for each page, call the page_handler (MultiThreading)
with ThreadPoolExecutor(max_workers=10) as executor:
for item in pages_numbers_list:
executor.submit(
page_handler,
item,
url_request,
headers
)
def page_handler(page_number, url_request, headers):
# we change the page number in the url request
url_request = change_page(url_request, page_number)
# new request with the new url
result = re_get(url_request, headers=headers)
result = result.json()
# in the result, with found the list of dict in order to create the
# final object
final_object_creation(result['results_list'])
def change_page(url_request, new_page_number):
"to increment the value of the 'page=' attribute in the url"
current_nb_page = ''
start_nb = url_request.find("page=") + len('page=')
while 1:
if url_request[start_nb].isdigit():
current_nb_page = url_request[start_nb]
else:
break
new_url_request = url_request.replace("page=" + current_nb_page,
"page=" + str(new_page_number))
return new_url_request
def final_object_creation(results_list):
'thanks to the object from requests.get(), it builts the final object'
global current_id_decision, dict_decisions
# each item in the results lis should be an instance of the final object
for item in results_list:
# On définit l'identifiant du nouvel objet Decision
current_id_decision += 1
new_id = current_id_decision
# On crée l'objet Décision et on l'ajoute au dico des décisions
dict_decisions[new_id] = FinalObject(item)
class FinalObject:
def __init__(self, content):
self.content = content
current_id_decision = 0
dict_decisions = {}
main_function(1000, "headers", "https://api/v1.0/search?page=0&query=test")

How do I filter out error responses from API call in python?

I'm using python to retrieve some data back from a rest API. I'm looping through a list to get data back for each record that is passed in like so:
safety_codes = ['M516RHJ', 'M16AJAR', 'Z49EJ57', 'Z1035TH', 'S0X6DJU9', 'S9099LP', 'S912AZSD', 'S72AEFH', 'S61ABKJ', 'W4XXATF']
#Building API variables:
api_data = dict()
rest_url = "https://www.gencodedemosite.com/restws/empcodes="
response_type = '&format=json'
header_details = {"KEY": "1101079000335WAXMEMU14532"}
#Making API call:
for i in generic_emp_codes:
response = requests.get(rest_url+'{}'.format(i)+response_format, headers = header_details)
data = json.loads(response.text)
api_data.update({i: data})
So long as the emp_code is valid I will return some data. But the moment one of those codes are bad, I get the following response and all of the other data is lost.
{
"error": "No record found for given employee safety code."
}
How do I filter out the these responses so that my api call does not fall over? Essentially, if codes M516RHJ and M16AJAR return that data. But if Z49EJ57 returns an error message, move that aside and keep the the data for M516RHJ and M16AJAR, then move on to the next code and repeat the process.
json_rdd = sc.parallelize(api_data)
json_df = spark.read.json(json_rdd)
json_df.printSchema()
root
|-- _corrupt_record: string (nullable = true)
Once I get that error message all of my data becomes corrupted.

results = []
bad_codes = []
#Making API call:
for i in generic_emp_codes:
response = requests.get(rest_url+'{}'.format(i)+response_format, headers = header_details)
data = json.loads(response.text)
if type(data)==dict and 'error' in data.keys():
bad_codes.append(data)
else:
results.append(data)

KeyError: records when trying to return record count from API

I am trying to check how many records a player has using the Hypixel API friends endpoint (api.hypixel.net/friends)
It keeps giving me a key error when trying to count the records. Here is what the API gives me:
{
"success": true,
"records": [{"_id":"5806841c0cf247f13be18b9d","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"976129e438b54a839944b1c0703d4da3","started":1476822044856},{"_id":"59589dde0cf250df95af825e","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"2c5bfce120c04ef1bfb3b798fe0d650e","started":1498979806692},{"_id":"5a444d820cf2604c12e0f6bd","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"ea703151981a409f8d0ff7cb782ab1c1","started":1514425730370},{"_id":"5aa595800cf24bd1104381cc","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"27af346e5bde40f0a665e808a331576f","started":1520801152354},{"_id":"5e2511f90cf289be2d0ea273","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"24a90aca074c4293a656f5fda047f816","started":1579487737061},{"_id":"5e36448c0cf2174287f94af7","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","started":1580614796544},{"_id":"5e3f54190cf2ab010c5a21ce","uuidSender":"71ef88df5f7e482fb472f344965beba8","uuidReceiver":"63817f05823945a2b58f4ba1de5589a3","started":1581208601957},{"_id":"55bec5e0c8f2e017bca39176","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1438565856136},{"_id":"55e21268c8f21846db2f2566","uuidSender":"8fca5ebf02f74a369b13f3407ad4a9bc","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1440879208434},{"_id":"56c26b190cf2d1a91ec25e83","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1455581977070},{"_id":"5755edc20cf2db67507e3a2e","uuidSender":"0ce4597de5484c0e82a067fa0bf171df","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1465249218466},{"_id":"5851ed8d0cf2b9563974034d","uuidSender":"d3dd0059775e46b1b1a63b94a10d2450","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1481764237412},{"_id":"5d087a1f0cf2d7aebbf96fd6","uuidSender":"d6695d1ea7ae480bb2129a3b7d0269ad","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1560836639346},{"_id":"5d703a880cf299e651b71cbb","uuidSender":"cbd9e9009ee94d159d52dff284ad7bf8","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1567636104118},{"_id":"5deeff7f0cf2d87bd75df1a0","uuidSender":"c53c4524ad174fd78134223fddedc484","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1575944063334},{"_id":"5e0638f20cf24f983d2cb02c","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1577466098097},{"_id":"5e1e70e70cf2795e4f1322be","uuidSender":"85090a8b495d4856817fd6df1d4da0bd","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579053287837},{"_id":"5e1fbe4b0cf2795e4f14a36f","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579138635859},{"_id":"5e25910d0cf2a892e569db39","uuidSender":"63ac73044aca4b2f908baff858cf34b9","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1579520269103},{"_id":"5e712f890cf2d292e1148ba6","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1584476041946},{"_id":"5e819f550cf2675e4372109b","uuidSender":"8cd38d8f97a24090a43e2e0ce898c521","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1585553237302},{"_id":"5e82c5ad0cf2675e437308aa","uuidSender":"d618457dd6044256bdb287b0df8137d4","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1585628589448},{"_id":"5eae3e420cf26efdbd55b592","uuidSender":"a936c3468bec4a2685199a398212b62d","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1588477506714},{"_id":"5ebd9da40cf22f431e9164d8","uuidSender":"1nmHNU1mERJEG452cx1FazPx3RpAuZ9vW","uuidReceiver":"71ef88df5f7e482fb472f344965beba8","started":1589484964296}]
}
Here is my code:
def get_friend_count(name):
getUUID = f"https://api.mojang.com/users/profiles/minecraft/{name}"
res = requests.get(getUUID)
data = res.json()
if data["id"] is None:
return None
returnUuid = (data["id"])
url1 = f"https://api.hypixel.net/friends?key={API_KEY}&uuid=" + returnUuid
res2 = requests.get(url1)
data2 = res2.json()
if data2["records"] is None:
return None
friend_count = len(data["records"])
return "Friends: " + friend_count
getUUID gets the UUID from the requested username and then uses the UUID to get the players Hypixel stats.
Any help is appreciated, thanks!
Mojang API: https://wiki.vg/Mojang_API
Hypixel API: https://github.com/HypixelDev/PublicAPI/tree/master/Documentation

Did you just forget to request.get(url1)?
In any case, from the mojang API doc, it seems the endpoint you query (https://api.mojang.com/users/profiles/minecraft/) never returns a "records" key ... thus the error when you do if data["records"] is None

http get python query key - value

Write a function named "query_string" that doesn't take any parameters. The function will make an HTTPS GET request to the url "https://fury.cse.buffalo.edu/ps-api/a" with a query string containing the key-value pairs x=5, y=4, and z=5. The response from the server will be a JSON string representing an object in the format "{"answer": }" where is a floating point Number. Return the value at the key "answer" as a float
import urllib.request
import json
def query_string():
response = urllib.request.urlopen("https://fury.cse.buffalo.edu/ps-api/a")
content_string = response.read().decode()
content=json.loads(content_string)
return float(content['answer'])
output: function query_string incorrect on input []
returned: -1.0
expected: 119.99
any idea how i can fix this issue?

You can do something like this using the requests package. It's super helpful. You can add the query parameters in a dict that gets passed to the params keyword argument
def query_string():
import requests
url=r'https://fury.cse.buffalo.edu/ps-api/a'
payload={
'x':5,
'y':4,
'z':5}
r=requests.get(url,params=payload)
j=r.json()
print(j)
EDIT for urllib
def query_string():
url=r'https://fury.cse.buffalo.edu/ps-api/a'
payload={
'x':5,
'y':4,
'z':5}
url=url+'?'+urllib.parse.urlencode(payload)
r=urllib.request.urlopen(url).read().decode()
r=json.loads(r)
return float(r['answer'])

import urllib.request
import json
def query_string():
url = "https://fury.cse.buffalo.edu/ps-api/a"
url = url + "?x=5&y=4&z=5"
response = urllib.request.urlopen(url)
content_string = response.read().decode()
content=json.loads(content_string)
return float(content['answer'])
ended up fixing it after a little research

Categories