I am reading an excel file which contain multiple unique numbers, which i use to perform request activity on a REST API. At the end of the program, I am trying to write another excel file where i am writing the status as 'Success' or 'Fail'.
The issue is, I am reading a file which contain data over 100k numbers. So if my program stop at any reason, or even i stop it intentionally, the output excel file never create. How do i make sure to get the output file till my script run.
here is my code.
from openpyxl import Workbook
from openpyxl import load_workbook
import requests
from datetime import datetime
def api_status():
wk = Workbook()
ws = wk.active
start_row = 2
start_column = 1
status_column = 2
wk = load_workbook("Data-File.xlsx")
source = wk["Sheet"]
global IDNO
for id_list in source['A']:
IDNO = id_list.value
url = "someURL"
payload = {'id_no': str(IDNO)}
headers = {}
response = requests.request("POST", json_url, data=json.dumps(payload), headers=headers)
json_obj = response.json()
ws.cell(row=start_row, column=start_column).value = IDNO
json_message = (json_obj.get('message'))
if json_message == "Success":
ws.cell(row=start_row, column=status_column).value = "Success"
start_row += 1
else:
print("NO")
ws.cell(row=start_row, column=status_column).value = "FAIL"
start_row += 1
wb.save("STATUS-FILE-%s.xlsx" % datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p"))
api_status()
You just fallback on some code by using a try... except. Replace this line:
api_status()
With this block of code:
try:
api_status()
except:
# CODE TO WRITE YOUR "FAIL" STATUS
Or we can do this for the loop inside the function. Of course there's a little more to it than that. You can specify specific actions to be taken for different error types, or you might want to put that try...except block inside of your function to control for specific lines failing and carrying out the rest.
I'm assuming the most likely error would come from your web request. In that case:
try:
response = requests.request("POST", json_url, data=json.dumps(payload), headers=headers)
json_obj = response.json()
except:
json.obj = {}
By making json_obj an empty dict if the request doesn't work, I guarantee that the next lines would write FAIL in your excel instead of Success
Combining both ideas to make sure your code reaches the save as well would look like this (using a finally to make sure it runs in any possible case):
from openpyxl import Workbook
from openpyxl import load_workbook
import requests
from datetime import datetime
def api_status():
wk = Workbook()
ws = wk.active
start_row = 2
start_column = 1
status_column = 2
wk = load_workbook("Data-File.xlsx")
source = wk["Sheet"]
try:
global IDNO
for id_list in source['A']:
IDNO = id_list.value
url = "someURL"
payload = {'id_no': str(IDNO)}
headers = {}
try:
response = requests.request("POST", json_url, data=json.dumps(payload), headers=headers)
json_obj = response.json()
except:
json.obj = {}
ws.cell(row=start_row, column=start_column).value = IDNO
json_message = (json_obj.get('message'))
if json_message == "Success":
ws.cell(row=start_row, column=status_column).value = "Success"
start_row += 1
else:
print("NO")
ws.cell(row=start_row, column=status_column).value = "FAIL"
start_row += 1
finally:
wb.save("STATUS-FILE-%s.xlsx" % datetime.now().strftime("%d-%m-%Y_%I-%M-%S_%p"))
api_status()
Related
I am trying to executing rest api get using python script.Below the code I am using.But api response doesn't have proper Json format I am getting error.
import requests
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in json format
data = r.json()
print(data)
My api response below
[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]
I need to change response as below
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
How to achieve this in python? I am new to python.
test = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
res = to_json(test)
print(res)
result:
[{"mac_address":"10:55","Parameter":"Device.Info","status":"Success","response_code":"200","value":"2.4Ghz"}]
in your case:
import requests
import json
def to_json(val: str):
val = val.replace("{:", '{"').replace(" :", '"').replace("=>", '":')
return val
# api-endpoint
URL = "end point url"
# sending get request and saving the response as response object
r = requests.get(url = URL)
# extracting data in string format
data = r.text
# converting string to json object
res = json.loads(to_json(data))
print(res)
Assuming you get a plain text response, you can manually re-format the response using String.replace():
response = '[{:mac_address=>"10:55", :Parameter=>"Device.Info", :status=>"Success", :response_code=>"200", :value=>"2.4Ghz"}]'
desired = {}
response = response[2:-3].replace('"', '').split(', ')
for r in response:
key, value = r.split('=>')
key = key.replace(':','')
desired[key] = value
I'm new to python and I want this code to run only once and stops, not every 30 seconds
because I want to run multiple codes like this with different access tokens every 5 seconds using the command line.
and when I tried this code it never jumps to the second one because it's a while true:
import requests
import time
api_url = "https://graph.facebook.com/v2.9/"
access_token = "access token"
graph_url = "site url"
post_data = { 'id':graph_url, 'scrape':True, 'access_token':access_token }
# Beware of rate limiting if trying to increase frequency.
refresh_rate = 30 # refresh rate in second
while True:
try:
resp = requests.post(api_url, data = post_data)
if resp.status_code == 200:
contents = resp.json()
print(contents['title'])
else:
error = "Warning: Status Code {}\n{}\n".format(
resp.status_code, resp.content)
print(error)
raise RuntimeWarning(error)
except Exception as e:
f = open ("open_graph_refresher.log", "a")
f.write("{} : {}".format(type(e), e))
f.close()
print(e)
time.sleep(refresh_rate)
From what I understood you're trying to execute the piece of code for multiple access tokens. To make your job simple, have all your access_tokens as lists and use the following code. It assumes that you know all your access_tokens in advance.
import requests
import time
def scrape_facebook(api_url, access_token, graph_url):
""" Scrapes the given access token"""
post_data = { 'id':graph_url, 'scrape':True, 'access_token':access_token }
try:
resp = requests.post(api_url, data = post_data)
if resp.status_code == 200:
contents = resp.json()
print(contents['title'])
else:
error = "Warning: Status Code {}\n{}\n".format(
resp.status_code, resp.content)
print(error)
raise RuntimeWarning(error)
except Exception as e:
f = open (access_token+"_"+"open_graph_refresher.log", "a")
f.write("{} : {}".format(type(e), e))
f.close()
print(e)
access_token = ['a','b','c']
graph_url = ['sss','xxx','ppp']
api_url = "https://graph.facebook.com/v2.9/"
for n in range(len(graph_url)):
scrape_facebook(api_url, access_token[n], graph_url[n])
time.sleep(5)
so im trying to write this JSON from the Kik smiley site, and im trying to do this so I wont have to write it manually, anyways I need to parse the JSON so only some of the existing JSON shows up in the file (basically cleaning it) what I need from the site is... (name, id, type) how would I do this?
I have written this in python but it seems to fail, and im not 100% sure as to why. I am new to Python, so sorry if this is an obvious question! I did find something earlier but it just confused me even more :) Thank you!
import requests, json, sys
from colorama import init
from termcolor import colored
#colorama
init()
class SmileyGrabber():
def __init__(self):
# requests vars
self.smileysFound = 0
self.smileysLost = 0
self.url="https://sticker-service.appspot.com/v2/collection/smiley"
self.session = requests.Session()
self.grabSmiley()
def grabSmiley(self):
while True:
try:
r = self.session.get(self.url)
j = r.json()
try:
if j["IsSuccess"] == True:
meta = j["smileys"]
sID = meta["id"]
sType = meta["type"]
sName = meta["name"]
FormatSmileyData(sID, sType, sName)
print "Smiley Found:", colored("({0})".format(sName), "cyan")
self.smileysFound += 1
else:
print(colored("Could not grab smiley"), "red")
self.smileysLost += 1
except:
sys.exit()
except KeyboardInterrupt:
sys.exit()
except:
print r.text
sys.exit()
class FormatSmileyData(object):
def __init__(self, sID, sType, sName):
smileyData = {}
data = []
data.append({"SMILEY_ID":sID, "SMILEY_TYPE":sType, "SMILEY_NAME":sName})
dataFile = open("smileys.json", "a+")
dataFile.write(json.dumps(smileyData)+"\n")
dataFile.close()
if __name__ == "__main__":
SmileyGrabber()
There are a number of problems with your code.
It will be more efficient to read from the network all at once
rather than making a call to session.get for each smiley.
j does not have an "IsSuccess" element, so that will never be true
j["smileys"] is a list, so to get the dictionaries (which represent each smiley) you will need to iterate through that list.
You are appending data into data but you are writing from
smileyData, which never has any data entered into it.
Each time you call the FormatSmileyData constructor, you are
resetting the data.
Take a look at a tool like Postman to prettify the JSON so you can see the structure. This can help figure out how to parse it.
Here's an updated version of your script that appears to work:
I removed the colorization and made it work with Python 3.
import requests, json, sys
class SmileyGrabber():
def __init__(self):
# requests vars
self.smileysFound = 0
self.smileysLost = 0
self.url="https://sticker-service.appspot.com/v2/collection/smiley"
self.session = requests.Session()
self.data = []
self.grabSmiley()
self.writeSmileyData()
def grabSmiley(self):
r = self.session.get(self.url)
j = r.json()
print ("got json")
print (str(len(j)))
for element in j:
for meta in element["smileys"]:
print ("---------------")
print (str(meta))
sID = meta["id"]
sType = meta["type"]
sName = meta["name"]
self.addSmileyData(sID, sType, sName)
print ("Smiley Found:" + "({0})".format(sName))
self.smileysFound += 1
print ("found " + str(self.smileysFound))
else:
print("Could not grab smiley")
self.smileysLost += 1
def addSmileyData(self, sID, sType, sName):
self.data.append({"SMILEY_ID":sID, "SMILEY_TYPE":sType, "SMILEY_NAME":sName})
def writeSmileyData(self):
dataFile = open("smileys.json", "a+")
dataFile.write(json.dumps(self.data)+"\n")
dataFile.close()
if __name__ == "__main__":
SmileyGrabber()
I have the following code
import requests
import sys
import json
arg_count = 0
#check that correct number of arguments are passed to utility
for arg in sys.argv:
arg_count = arg_count + 1
assert arg_count == 4
#parse the input arguments to the utility
connectplus_group_title = sys.argv[1]
connectplus_uname = sys.argv[2]
connectplus_passwords = sys.argv[3]
#find the group id from the input group name i.e connectplus_group_title
response = requests.get("https://connectplus.geometricglobal.com/_vti_bin/beezy/v2/api.svc//Groups/Availability?title={title})", auth=(connectplus_uname, connectplus_passwords))
#assert response.status_code == response.codes.ok
group_data = json.loads(response)
group_id = group_data[connectplus_group_title]
connectplus_group_contributors = "https://connectplus.geometricglobal.com/_vti_bin/beezy/v2/api.svc//Groups/{id}/Contributors?skip={skip}&take={take}"
response = requests.get(connectplus_group_contributors,auth=(connectplus_uname, connectplus_passwords))
data = response.json()
I need to get the json data which is returned by the URL , I have tried using json.loads(response.text) or simple json.loads(response) with no luck I keep getting the above error.
What is the correct way to get the data from a REST url in the from of dictionary.
You have to pass response.content, response is a Response type not a simple string (try type(response)) and json.loads needs (as the error highlights) a string or buffer.
Replace your call with:
json.loads(response.content)
I’m trying to make a program that will do the following:
check if auth_file exists
if yes -> read file and try to login using data from that file
- if data is wrong -> request new data
if no -> request some data and then create the file and fill it with requested data
So far:
import json
import getpass
import os
import requests
filename = ".auth_data"
auth_file = os.path.realpath(filename)
url = 'http://example.com/api'
headers = {'content-type': 'application/json'}
def load_auth_file():
try:
f = open(auth_file, "r")
auth_data = f.read()
r = requests.get(url, auth=auth_data, headers=headers)
if r.reason == 'OK':
return auth_data
else:
print "Incorrect login..."
req_auth()
except IOError:
f = file(auth_file, "w")
f.write(req_auth())
f.close()
def req_auth():
user = str(raw_input('Username: '))
password = getpass.getpass('Password: ')
auth_data = (user, password)
r = requests.get(url, auth=auth_data, headers=headers)
if r.reason == 'OK':
return user, password
elif r.reason == "FORBIDDEN":
print "Incorrect login information..."
req_auth()
return False
I have the following problems(understanding and applying the correct way):
I can't find a correct way of storing the returned data from req_auth() to auth_file in a format that can be read and used in load_auth file
PS: Of course I'm a beginner in Python and I'm sure I have missed some key elements here :(
To read and write data, you can use json:
>>> with open('login.json','w') as f:
f.write(json.dumps({'user': 'abc', 'pass': '123'}))
>>> with open('login.json','r') as f:
data=json.loads(f.read())
>>> print data
{u'user': u'abc', u'pass': u'123'}
A few improvements I'd suggest:
Have a function that tests login (arguments: user,pwd) and returns True/False
Save data inside req_data, because req_data is called only when you have incorrect/missing data
Add an optional argument tries=0 to req_data, and test against it for a maximum number of tries
(1):
def check_login(user,pwd):
r = requests.get(url, auth=(user, pwd), headers=headers)
return r.reason == 'OK':
for (2), you can use json (as described above), csv, etc. Both of those are extremely easy, though json might make more sense since you're already using it.
for (3):
def req_auth(tries = 0) #accept an optional argument for no. of tries
#your existing code here
if check_login(user, password):
#Save data here
else:
if tries<3: #an exit condition and an error message:
req_auth(tries+1) #increment no. of tries on every failed attempt
else:
print "You have exceeded the number of failed attempts. Exiting..."
There are a couple of things I would approach differently, but you're off to a good start.
Instead of trying to open the file initially I'd check for it's existence:
if not os.path.isfile(auth_file):
Next, when you're working with writing the output you should use context managers:
with open(auth_file, 'w') as fh:
fh.write(data)
And finally, as a storage open (not terribly secure), it might work well to put the information you're saving in json format:
userdata = dict()
userdata['username'] = raw_input('Username: ')
userdata['password'] = getpass.getpass('Password: ')
# saving
with open(auth_file, 'w') as fho:
fho.write(josn.dumps(userdata))
# loading
with open(auth_file) as fhi:
userdata = json.loads(fhi.read())