We Keep Coding

Python unable to parse Json file with error "raise JSONDecodeError("Extra data", s, end) json.decoder.JSONDecodeError: Extra data"

I am trying to download a Json file from an API and convert it into a csv file, but the script throws the below error while parsing the json file.
For every 100 records json file closes the "]" and starts another "[". This format is not being accepted as json format. could you please suggest me how i can parse the "]" and "[" which appears every 100 records in an efficient way.The code works fine for less than 100 records without the [] brackets.
Error message:
raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data
Json file format:
**[**
{
"A": "5",
"B": "811",
"C": [
{ "C1": 1,
"C2": "sa",
"C3": 3
}
],
"D": "HH",
"E": 0,
"F": 6
},
{
"A": "5",
"B": "811",
"C": [
{ "C1": 1,
"C2": "fa",
"C3": 3
}
],
"D": "HH",
"E": 0,
"F": 6
}
**]**
**[**
{
"A": "5",
"B": "811",
"C": [
{ "C1": 1,
"C2": "da",
"C3": 3
}
],
"D": "HH",
"E": 0,
"F": 6
}
**]**
Code:
import json
import pandas as pd
from flatten_json import flatten
def json2excel():
file_path = r"<local file path>"
json_list = json.load(open(file_path + '.json', 'r', encoding='utf-8', errors='ignore'))
key_list = ['A', 'B']
json_list = [{k: d[k] for k in key_list} for d in json_list]
# Flatten and convert to a data frame
json_list_flattened = (flatten(d, '.') for d in json_list)
df = pd.DataFrame(json_list_flattened)
# Export to CSV in the same directory with the original file name
export_csv = df.to_csv(file_path + r'.csv', sep=',', encoding='utf-8', index=None, header=True)
def main():
json2excel()

I would recommend parsing your data that you receive from your API first. This pre-processed data can be fed to a JSON parser later.
I came up with a simple python code that's just a small tweak to the solution of parenthesis matching problem. Here's my working code that you might use for pre-processing your data.
def build_json_items(custom_json):
open_tup = tuple('({[')
close_tup = tuple(')}]')
map = dict(zip(open_tup, close_tup))
queue = []
json_items = []
temp = ""
for i in custom_json:
if i in open_tup:
queue.append(map[i])
elif i in close_tup:
if not queue or i != queue.pop():
return "Unbalanced"
if len(queue) == 0:
# We have reached to a point where everything so far is balanced.
# This is the point where we can separate out the expression
temp = temp + str(i)
json_items.append(temp)
temp = "" # Re-initialize
else:
temp = temp + str(i)
if not queue:
# Provided string is balanced
return True, json_items
else:
return False, json_items
This build_json_items function will take your custom JSON payload and will parse individual valid JSON items based on the information that you provided in your question. Here's an example of how you can trigger this function. You can use the following.
input_data = "[{\"A\":\"5\",\"B\":\"811\",\"C\":[{\"C1\":1,\"C2\":\"sa\",\"C3\":3}],\"D\":\"HH\",\"E\":0,\"F\":6},{\"A\":\"5\",\"B\":\"811\",\"C\":[{\"C1\":1,\"C2\":\"fa\",\"C3\":3}],\"D\":\"HH\",\"E\":0,\"F\":6}][{\"A\":\"5\",\"B\":\"811\",\"C\":[{\"C1\":1,\"C2\":\"da\",\"C3\":3}],\"D\":\"HH\",\"E\":0,\"F\":6}]"
is_balanced, json_items = build_json_items(input_data)
print(f"Available JSON items: {len(json_items)}")
print("JSON items are the following")
for i in json_items:
print(i)
Here's the output of the print statements.
Available JSON items: 2
JSON items are the following
[{"A":"5","B":"811","C":[{"C1":1,"C2":"sa","C3":3}],"D":"HH","E":0,"F":6},{"A":"5","B":"811","C":[{"C1":1,"C2":"fa","C3":3}],"D":"HH","E":0,"F":6}]
[{"A":"5","B":"811","C":[{"C1":1,"C2":"da","C3":3}],"D":"HH","E":0,"F":6}]
You can directly run and see the output here.
Once you have those payloads separated in valid JSON structure, you can feed these to you JSON parser.

how to store JSON list data into python variables?

I have a JSON file that contains some data.
testJSON.json
{
"hourlyData": [
{
"frequency": "49.96",
"actual": " 2,240.43 ",
"schedule": " 2,223.85 ",
"acp": "325"
},
{
"frequency": "50.04",
"actual": " 1,862.88 ",
"schedule": " 1,881.09 ",
"acp": "275"
},
{
"frequency": "50.04",
"actual": " 1,882.17 ",
"schedule": " 1,885.94 ",
"acp": "275"
}
],
"storageData": [
{
"config": "ESS1",
"name": "BRPL",
"Power Rating": "20",
"Energy Rating": "20",
"socLow": "0",
"socHigh": "0.8",
"Charge Eff": "0.9273",
"Discharge Eff": "0.9273",
"Round Trip Eff": "0.922",
"Lower Limit": "57",
"Mid Limit": "76",
"High Limit": "95",
"Thrushold": "5",
"Discharging Price": "6"
}
],
I want to store these values into python variables. So what I have done is that first I created a dictionary which contain different variables of different types then I created a function which simple opens the JSON file then I try to store those json values into declared variables:
test.py
import json
#decalaring variables
storageData = {
"name": 'No name specified',
"powerRating":-1,
"energyRating":-1,
"annualMaxCycles":365,
"socLow":-1,
"socHigh":-1,
"chargeEff":-1,
"dChargeEff":-1,
"lowerLimit": -1,
"midLimit": -1,
"highLimit": -1,
"thrushold": 5,
"dischargingPrice": 6
}
marketData = {
"marketProducts": {
"dsm":{
"frequency":[],
"schedule":[],
"actual":[],
"acp": [],
}
}
inputMode = 'JSON'
JSONfileName = "testJSON.json"
def inputJSON():
if (inputMode == 'JSON'):
fileName = JSONfileName
# Import data from JSON files
with open(JSONfileName, 'r') as myfile:
dataJSON = ((myfile.read().replace('\n', '')))
inputJSON = json.loads(dataJSON)
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
marketData['marketProducts']['dsm']['frequency'] = inputJSON['hourlyData']['frequency']
marketData['marketProducts']['dsm']['acp'] = inputJSON['hourlyData']['acp']
marketData['marketProducts']['dsm']['actual'] = inputJSON['hourlyData']['actual']
marketData['marketProducts']['dsm']['schedule'] = inputJSON['hourlyData']['schedule']
inputJSON()
error that it gives me
Traceback (most recent call last):
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 113, in <module>
inputJSON()
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 80, in inputJSON
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
TypeError: list indices must be integers or slices, not str

Instead of
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
Do
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData[0]']['Power Rating']
storageData['energyRating'] = inputJSON['storageData[0]']['energyRating']
storageData['warranty'] = inputJSON['storageData[0]']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData[0]']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData[0]']['socLow']
storageData['socHigh'] = inputJSON['storageData[0]']['socHigh']
storageData['chargeEff'] = inputJSON['storageData[0]']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData[0]']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData[0]']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData[0]']['midLimit']
storageData['highLimit'] = inputJSON['storageData[0]']['highLimit']
storageData['thrushold'] = inputJSON['storageData[0]']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData[0]']['dischargingPrice']
Because it's an array, whose 1st index has the json data.
Ex. for "hourlyData"
for(data of inputJSON['hourlyData']){
marketData['marketProducts']['dsm']['frequency'] = data['frequency']
}
Hope that helps.

i want to convert sample JSON data into nested JSON using specific key-value in python

I have below sample data in JSON format :
project_cost_details is my database result set after querying.
{
"1": {
"amount": 0,
"breakdown": [
{
"amount": 169857,
"id": 4,
"name": "SampleData",
"parent_id": "1"
}
],
"id": 1,
"name": "ABC PR"
}
}
Here is full json : https://jsoneditoronline.org/?id=2ce7ab19af6f420397b07b939674f49c
Expected output :https://jsoneditoronline.org/?id=56a47e6f8e424fe8ac58c5e0732168d7
I have this sample JSON which i created using loops in code. But i am stuck at how to convert this to expected JSON format. I am getting sequential changes, need to convert to tree like or nested JSON format.
Trying in Python :
project_cost = {}
for cost in project_cost_details:
if cost.get('Parent_Cost_Type_ID'):
project_id = str(cost.get('Project_ID'))
parent_cost_type_id = str(cost.get('Parent_Cost_Type_ID'))
if project_id not in project_cost:
project_cost[project_id] = {}
if "breakdown" not in project_cost[project_id]:
project_cost[project_id]["breakdown"] = []
if 'amount' not in project_cost[project_id]:
project_cost[project_id]['amount'] = 0
project_cost[project_id]['name'] = cost.get('Title')
project_cost[project_id]['id'] = cost.get('Project_ID')
if parent_cost_type_id == cost.get('Cost_Type_ID'):
project_cost[project_id]['amount'] += int(cost.get('Amount'))
#if parent_cost_type_id is None:
project_cost[project_id]["breakdown"].append(
{
'amount': int(cost.get('Amount')),
'name': cost.get('Name'),
'parent_id': parent_cost_type_id,
'id' : cost.get('Cost_Type_ID')
}
)
from this i am getting sample JSON. It will be good if get in this code only desired format.
Also tried this solution mention here : https://adiyatmubarak.wordpress.com/2015/10/05/group-list-of-dictionary-data-by-particular-key-in-python/

I got approach to convert sample JSON to expected JSON :
data = [
{ "name" : "ABC", "parent":"DEF", },
{ "name" : "DEF", "parent":"null" },
{ "name" : "new_name", "parent":"ABC" },
{ "name" : "new_name2", "parent":"ABC" },
{ "name" : "Foo", "parent":"DEF"},
{ "name" : "Bar", "parent":"null"},
{ "name" : "Chandani", "parent":"new_name", "relation": "rel", "depth": 3 },
{ "name" : "Chandani333", "parent":"new_name", "relation": "rel", "depth": 3 }
]
result = {x.get("name"):x for x in data}
#print(result)
tree = [];
for a in data:
#print(a)
if a.get("parent") in result:
parent = result[a.get("parent")]
else:
parent = ""
if parent:
if "children" not in parent:
parent["children"] = []
parent["children"].append(a)
else:
tree.append(a)
Reference help : http://jsfiddle.net/9FqKS/ this is a JavaScript solution i converted to Python

It seems that you want to get a list of values from a dictionary.
result = [value for key, value in project_cost_details.items()]

Python - Print when a json object gets higher number

I want to create a script where I check a json file times to times using a while function. In there there is a json that looks like:
{
"names":[
{
"name":"hello",
"numbers":0
},
{
"name":"stack",
"numbers":1
},
{
"name":"over",
"numbers":2
},
{
"name":"flow",
"numbers":12
},
{
"name":"how",
"numbers":17
},
{
"name":"are",
"numbers":11
},
{
"name":"you",
"numbers":18
},
{
"name":"today",
"numbers":6
},
{
"name":"merry",
"numbers":4
},
{
"name":"x",
"numbers":1
},
{
"name":"mass",
"numbers":0
},
{
"name":"santa",
"numbers":4
},
{
"name":"hohoho",
"numbers":1
}
]
}
and what I want to do is that I want to check every number if numbers for each name has been increased than previous json look.
def script():
with open('data.json') as f:
old_data = json.load(f)
while True:
with open('data.json') as f:
new_data = json.load(f)
if old_data < new_data:
print("Bigger!!" + new_data['name'])
old_data = new_data
else:
randomtime = random.randint(5, 15)
print("Nothing increased")
old_data = new_data
time.sleep(randomtime)
Now I know that I have done it wrong and that's the reason I am here. I have no idea at this moment what I can do to make a sort of function where it checks numbers by numbers to see if its gotten bigger or not.
My question is:
How can I make it so it checks object by object to see if the numbers has gotten bigger from previous loop? and if it has not gotten bigger but lower, it should update the value of old_data and loops forever until the numbers has gotten bigger than previous loop?
EDIT:
Recommendation that I got from #Karl
{
'names': {
'hello': 0,
'stack': 0,
'over': 2,
'flow': 12,
'how': 17,
'are': 11,
'you': 18,
'today': 6,
'merry': 4,
'x': 1,
'mass': 0,
'santa': 4,
'hohoho': 1
}
}

Assuming your json is in this format:
{
"names": {
"hello": 0,
"stack": 1,
"over": 2,
"flow": 13,
"how": 17,
"are": 12,
"you": 18,
"today": 6,
"merry": 4,
"x": 1,
"mass": 0,
"santa": 4,
"hohoho": 1
}
}
I would do something along the following lines:
import json
import time
with open("data.json") as f:
old_data = json.load(f)["names"]
while True:
with open("data.json") as f:
new_data = json.load(f)["names"]
for name, number in new_data.items():
if number > old_data[name]:
print("Entry '{0}' has increased from {1} to {2}".format(name, old_data[name], number))
old_data = new_data
print("sleeping for 5 seconds")
time.sleep(5)
EDIT to answer question posted in comment "just curious, lets say if I want to add another value beside the numbers etc "stack": 1, yes (Yes and no to each of format), What would be needed to do in that case? (Just a script that I want to develop from this)".
In that case you should design your json input as follows:
{
"names": {
"hello": {
"number": 0,
"status": true
},
"stack": {
"number": 1,
"status": true
},
"over": {
"number": 2,
"status": false
},
...
}
}
You would need to change the lookups in the comparison script as follows:
for name, values in new_data.items():
if values["number"] > old_data[name]["number"]
(Note that for status you could also just have "yes" or "no" as inputs, but using booleans is must more useful when you have to represent a binary choice like this).
By the way, unless you aim to have objects other than names in this json, you can leave out that level and just make it:
{
"hello": {
"number": 0,
"status": true
},
"stack": {
"number": 1,
"status": true
},
"over": {
"number": 2,
"status": false
},
...
}
In that case, replace old_data = json.load(f)["names"] with old_data = json.load(f) and new_data= json.load(f)["names"] with new_data= json.load(f)

I took your original .json which you edited and presented in your question and re-factored your code to the below example. It appears to be working.
import time
import random
import json
path_to_file = r"C:\path\to\.json"
def script():
with open(path_to_file) as f:
d = json.load(f)
old_data = 0
for a_list in d.values():
for i in a_list:
print()
for d_keys, d_values in i.items():
print(d_keys, d_values)
if type(d_values) == int and d_values > old_data:
print("Bigger!!" + i['name'])
old_data = d_values
elif type(d_values) == int and d_values < old_data:
print("Nothing increased")
old_data = d_values
randomtime = random.randint(5, 15)
time.sleep(randomtime)
script()
This is the output I receive:
name hello numbers 0
name stack numbers 1 Bigger!!stack
name over numbers 2 Bigger!!over
name flow numbers 12 Bigger!!flow
name how numbers 17 Bigger!!how
name are numbers 11 Nothing increased
name you numbers 18 Bigger!!you
name today numbers 6 Nothing increased
name merry numbers 4 Nothing increased
name x numbers 1 Nothing increased
name mass numbers 0 Nothing increased
name santa numbers 4 Bigger!!santa
name hohoho numbers 1 Nothing increased

How to search in particular level in JSON tree using Python

I'm trying to learn a method of searching within a JSON tree on a particular level. I have the following JSON template example:
"Pair": {
"Instrument_A": {
"Segment A": {
"default": {
"value X": 1,
"value Z": 2,
"value Y": 3,
}
},
"Segment B": {
"default": {
"value X": 1,
"value Z": 2,
"value Y": 3,
}
}
},
"Instrument_B": {
"Segment A": {
"not-default": {
"value X": 1,
"value Z": 2,
"value Y": 3,
}
}
}
}
My goal is to count all arrays with the name that does not equal to "default", for example, you can see on the 4 level under instrument B, Segment A, there is an object named "not-default"

You can parse json using json package and iterate through the dictionaries.
Python2:
import json
json_str = "..."
json_object = json.loads(json_str)
for pair_k, pair_v in json_object.iteritems():
for inst_k, inst_v in pair_v.iteritems():
for seg_k, seg_v in inst_v.iteritems():
if not seg_v == "default"
pass # do whatever you want - print, append to list, etc.
Python3:
import json
json_str = "..."
json_object = json.loads(json_str)
for pair_k, pair_v in json_object.items():
for inst_k, inst_v in pair_v.items():
for seg_k, seg_v in inst_v.items():
if not seg_v == "default"
pass # do whatever you want - print, append to list, etc.

count = Counter()
def count_keys(data):
if isinstance(data, dict):
for key, value in data.items():
if key != 'default':
count[key]+=1
count_keys(data[key])
count_keys(data)
print(count)
prints
Counter({'value X': 3, 'value Z': 3, 'value Y': 3, 'Segment A': 2, 'Pair': 1,'Instrument_A': 1, 'Segment B': 1, 'Instrument_B': 1, 'not-default': 1})