I am trying to run this code but it creates error.
import json
import requests
import pprint
data = []
with open('data.txt') as o1:
for line in o1:
data.append(json.loads(line))
print(data)
print(" \n")
print(data)
url = 'http://xyz.abcdfx.in/devicedata'
body_json=json.dumps(data)
headers = {'Content-Type':'application/json'}
d = requests.post(url, data = body_json, headers=headers)
pprint.pprint(d.json())
it shows
Value Error: No json object could be Decoded
I am new to programming and not able to figure out what is the problem.
It seems like you are trying to parse the json file line by line, but the json objects may (and usually are) span more than one line. You need to have the entire file in order to parse it:
with open('data.txt') as o1:
data = json.loads(o1.read()) # read ALL the file and parse. no loops
print(data)
i solved my problem using this:
data =[]
with open('data.txt') as f:
for line in f:
data = json.loads(line)
print(data)
url = 'http://xyz.abcdfx.cn/devicedata'
body_json=json.dumps(data)
headers = {'Content-Type':'application/json'}
d = requests.post(url, data = body_json, headers=headers)
pprint.pprint(d.json())
Related
I'm getting the following error for my python scraper:
import requests
import json
symbol_id = 'COINBASE_SPOT_BTC_USDT'
time_start = '2022-11-20T17:00:00'
time_end = '2022-11-21T05:00:00'
limit_levels = 100000000
limit = 100000000
url = 'https://rest.coinapi.io/v1/orderbooks/{symbol_id}/history?time_start={time_start}limit={limit}&limit_levels={limit_levels}'
headers = {'X-CoinAPI-Key' : 'XXXXXXXXXXXXXXXXXXXXXXX'}
response = requests.get(url, headers=headers)
print(response)
with open('raw_coinbase_ob_history.json', 'w') as json_file:
json.dump(response.json(), json_file)
with open('raw_coinbase_ob_history.json', 'r') as handle:
parsed = json.load(handle)
with open('coinbase_ob_history.json', 'w') as coinbase_ob:
json.dump(parsed, coinbase_ob, indent = 4)
<Response [400]>
And in my written json file, I'm outputted
{"error": "Wrong format of 'time_start' parameter."}
I assume a string goes into a url, so I flattened the timestring to a string. I don't understand why this doesn't work. This is the documentation for the coinAPI call I'm trying to make with 'timestring'. https://docs.coinapi.io/?python#historical-data-get-4
Incorrect syntax for python. To concatenate strings, stick them together like such:
a = 'a' + 'b' + 'c'
string formatting is invalid, and also need use & in between different url params
# python3
url = f"https://rest.coinapi.io/v1/orderbooks/{symbol_id}/history?time_start={time_start}&limit={limit}&limit_levels={limit_levels}"
# python 2
url = "https://rest.coinapi.io/v1/orderbooks/{symbol_id}/history?time_start={time_start}&limit={limit}&limit_levels={limit_levels}".format(symbol_id=symbol_id, time_start=time_start, limit=limit, limit_levels=limit_levels)
https://docs.python.org/3/tutorial/inputoutput.html
https://docs.python.org/2/tutorial/inputoutput.html
I have a json file, and I need to read all of that json file content as String data. How can I read all the data and set a variable as a String for all of that content? Json file has blanks, new lines, special characters etc if it's neccesarry.
Thanks for your help!
import json
from ast import literal_eval
with open('<path_to_json_data>/json_data.txt') as f:
json_data = json.load(f) # dict object
print(json_data, type(json_data))
json_data_as_str = str(json_data) # dict-->str object
print(json_data_as_str, type(json_data_as_str))
data = literal_eval(json_data_as_str) # str-->dict object again
print(data, type(data))
Hope it helps
Simple as this example
import json
with open("path/to/json/filename.json", "r") as json_file:
data = json.load(json_file)
print(data)
dataStr = json.dumps(data)
print(dataStr)
use json.loads
import json
with open(file_name, "r") as fp:
as_string = str(json.loads(fp.read()))
Hi I would like to save my python requests cookies into a text file as json like so:
{"Proxy": "Cookies"}
At the moment im using:
with open('data.txt') as f:
proxy_data = json.load(f)
r = requests.get(URL,proxies=proxies, cookies=proxy_data['127:000'])
proxy_data['127:000'] = dict(session.cookies)
with open('data.txt', 'w') as json_file:
json.dump(proxy_data, json_file)
My data.txt file looks like this after writing it:
{"127:000": "{\"__cf_bm\": \"628f4c77a5139148cf28961a73f4edffed52f8c1-1622925414-1800-AaLguAyS9Hfl8TXCaQYUjjZ5i3sDck15kn/vcPJCWZBxi+nM9xU0zd/GfyKJu6/3R+GQEFdk/DGHEV4mANvpG1UMvcsXA9G9l/AFNEnIknTv\", \"cf_clearance\": \"84bffd070fbafcb91edcfa82f8cc3ba5a60a618d-1622925414-0-150\", \"cf_chl_seq_b116a013f9e2c44\": \"d3ac4406fc56d30\"}"}
However I run into a number of errors here. How can I do the properly?
UPDATE
ERRORS I GET
Python\Python37\lib\site-packages\requests\cookies.py", line 524, in
cookiejar_from_dict
cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) TypeError: string indices must be integers
Currently, the value of proxy_data["127:000"] is a string which you're passing into the cookies parameter.
with open("data.txt") as f:
proxy_data = json.load(f)
print(type(proxy_data["127:000"]))
# >> <class 'str'>
Two options:
Parse the value of proxy_data["127:000"] to a dictionary using json.loads
r = requests.get(URL, proxies=proxies, cookies=json.loads(proxy_data['127:000']))
Make sure to save the nested dicts correctly.
proxy_data = {"127:000": {"cookie_test": "value_test"}}
with open("data.txt", "w") as f:
f.write(json.dumps(proxy_data))
with open("data.txt", "r") as f:
proxy_data = json.loads(f.read())
print(type(proxy_data["127:000"]))
# >> <class 'dict'>
am trying to write a loop that gets .json from an url via requests, then writes the .json to a .csv file. Then I need it to it over and over again until my list of names (.txt file) is finished(89 lines). I can't get it to go over the list, it just get the error:
AttributeError: module 'response' has no attribute 'append'
I can´t find the issue, if I change 'response' to 'responses' I get also an error
with open('listan-{}.csv'.format(pricelists), 'w') as outf:
OSError: [Errno 22] Invalid argument: "listan-['A..
I can't seem to find a loop fitting for my purpose. Since I am a total beginner of python I hope I can get some help here and learn more.
My code so far.
#Opens the file with pricelists
pricelists = []
with open('prislistor.txt', 'r') as f:
for i, line in enumerate(f):
pricelists.append(line.strip())
# build responses
responses = []
for pricelist in pricelists:
response.append(requests.get('https://api.example.com/3/prices/sublist/{}/'.format(pricelist), headers=headers))
#Format each response
fullData = []
for response in responses:
parsed = json.loads(response.text)
listan=(json.dumps(parsed, indent=4, sort_keys=True))
#Converts and creates a .csv file.
fullData.append(parsed['Prices'])
with open('listan-{}.csv'.format(pricelists), 'w') as outf:
dw.writeheader()
for data in fullData:
dw = csv.DictWriter(outf, data[0].keys())
for row in data:
dw.writerow(row)
print ("The file list-{}.csv is created!".format(pricelists))
Can you make the below changes in the place where you are making the api call(import json library as well) and see?
import json
responses = []
for pricelist in pricelists:
response = requests.get('https://api.example.com/3/prices/sublist/{}/'.format(pricelist), headers=headers)
response_json = json.loads(response.text)
responses.append(response_json)
and the below code also should be in a loop which loops through items in pricelists
for pricelist in pricelists:
with open('listan-{}.csv'.format(pricelists), 'w') as outf:
dw.writeheader()
for data in fullData:
dw = csv.DictWriter(outf, data[0].keys())
for row in data:
dw.writerow(row)
print ("The file list-{}.csv is created!".format(pricelists))
Finally got it working. Got a help from another questions I created here at the forum. #waynelpu
The misstake I did was to not put the code into a loop.
Here is the code that worked like a charm.
pricelists = []
with open('prislistor.txt', 'r') as f:
for i, line in enumerate(f): # from here on, a looping code block start with 8 spaces
pricelists = (line.strip())
# Keeps the indents
response = requests.get('https://api.example.se/3/prices/sublist/{}/'.format(pricelists), headers=headers)
#Formats it
parsed = json.loads(response.text)
listan=(json.dumps(parsed, indent=4, sort_keys=True))
#Converts and creates a .csv file.
data = parsed['Prices']
with open('listan-{}.csv'.format(pricelists), 'w') as outf:
dw = csv.DictWriter(outf, data[0].keys())
dw.writeheader()
for row in data:
dw.writerow(row)
print ("The file list-{}.csv is created!".format(pricelists))
# codes here is outside the loop but still INSIDE the 'with' block, so you can still access f here
# codes here leaves all blocks
I have a JSON file containing various objects each containing elements. With my python script, I only keep the objects I want, and then put the elements I want in a list. But the element has a prefix, which I'd like to suppress form the list.
The post-script JSON looks like that:
{
"ip_prefix": "184.72.128.0/17",
"region": "us-east-1",
"service": "EC2"
}
The "IP/mask" is what I'd like to keep. The List looks like that:
'"ip_prefix": "23.20.0.0/14",'
So what can I do to only keep "23.20.0.0/14" in the list?
Here is the code:
json_data = open(jsonsourcefile)
data = json.load(json_data)
print (destfile)
d=[]
for objects in (data['prefixes']):
if servicerequired in json.dumps(objects):
#print(json.dumps(objects, sort_keys=True, indent=4))
with open(destfile, 'a') as file:
file.write(json.dumps(objects, sort_keys=True, indent=4 ))
with open(destfile, 'r') as reads:
liste = list()
for strip in reads:
if "ip_prefix" in strip:
strip = strip.strip()
liste.append(strip)
print(liste)
Thanks,
dersoi
Ok so i've went through your JSON object
import json, urllib2
url = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
req = urllib2.Request(url)
res = urllib2.urlopen(req)
j = json.load(res)
print j['prefixes'][0]['ip_prefix']
prefixes = j['prefixes']
for i in prefixes:
print i['ip_prefix']
the result:
>>>
23.20.0.0/14
23.20.0.0/14
27.0.0.0/22
43.250.192.0/24
43.250.193.0/24
46.51.128.0/18
46.51.192.0/20
46.51.216.0/21
46.51.224.0/19
etc...
So now you want all into one txt file right?
So you do this:
import json, urllib2
url = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
req = urllib2.Request(url)
res = urllib2.urlopen(req)
j = json.load(res)
#print j['prefixes'][0]['ip_prefix']
prefixes = j['prefixes']
destfile = 'destfile.txt'
with open('destfile.txt', 'w') as f:
for i in prefixes:
#print i['ip_prefix']
f.write(i['ip_prefix'])
f.write('\n')
f.close
Best regards,
Rizzit
I've refactored your code, try this out:
import json
with open('sample.json', 'r') as data:
json_data = json.loads(data.read())
print json_data.get('ip_prefix')
# Output: "184.72.128.0/17"
You can rewrite the second open block as:
with open(destfile, 'r') as reads:
data = json.load(reads)
liste = [i['ip_prefix'] for i in data]
Although, I don't think you need to write to an intermediate file anyway, you could combine both blocks.