We Keep Coding

Query Nested JSON Data

I am trying to query using the code below to get the value etoday but it does not return a result.
Result = json.loads(json_string)
# Next Line works perfect
print("The value of msg", "msg", "is: ", Result["msg"])
#Next Line does not (I have a few variations here that I tried
print(Result['data']['page']['records']['etoday'])
print(Result["data"][0]["page"][0]["etoday"])
print(Result['page']['records'][0]['etoday'])
Print("The value of", "etoday", "is: ", Result["etoday"])
File I am querying
{
"success": true,
"code": "0",
"msg": "success",
"data": {
"inverterStatusVo": {
"all": 1,
"normal": 1,
"fault": 0,
"offline": 0,
"mppt": 0
},
"page": {
"records": [
{
"id": "zzz",
"sn": "xxx",
"collectorSn": "vvv",
"userId": "ttt",
"productModel": "3105",
"nationalStandards": "68",
"inverterSoftwareVersion": "3d0037",
"dcInputType": 2,
"acOutputType": 0,
"stationType": 0,
"stationId": "1298491919448828419",
"rs485ComAddr": "101",
"simFlowState": -1,
"power": 6.000,
"powerStr": "kW",
"pac": 0.001,
"pac1": 0,
"pacStr": "kW",
"state": 1,
"stateExceptionFlag": 0,
"fullHour": 1.82,
"totalFullHour": 191.83,
"maxDcBusTime": "1675460767377",
"maxUac": 251.5,
"maxUacTime": "1664022228160",
"maxUpv": 366.3,
"maxUpvTime": "1663309960961",
"timeZone": 0.00,
"timeZoneStr": "(UTC+00:00)",
"timeZoneName": "(UTC+00:00)Europe/Dublin",
"dataTimestamp": "1675460767377",
"dataTimestampStr": "2023-02-03 21:46:07 (UTC+00:00)",
"fisTime": "1663189116371",
"inverterMeterModel": 5,
"updateShelfBeginTime": 1671292800000,
"updateShelfEndTime": 1829059200000,
"updateShelfEndTimeStr": "2027-12-18",
"updateShelfTime": "5",
"collectorId": "1306858901387600551",
"dispersionRate": 0.0,
"currentState": "3",
"pow1": 0.0,
"pow2": 0.0,
"pow3": 0.0,
"pow4": 0.0,
"pow5": 0.0,
"pow6": 0.0,
"pow7": 0.0,
"pow8": 0.0,
"pow9": 0.0,
"pow10": 0.0,
"pow11": 0.0,
"pow12": 0.0,
"pow13": 0.0,
"pow14": 0.0,
"pow15": 0.0,
"pow16": 0.0,
"pow17": 0.0,
"pow18": 0.0,
"pow19": 0.0,
"pow20": 0.0,
"pow21": 0.0,
"pow22": 0.0,
"pow23": 0.0,
"pow24": 0.0,
"pow25": 0.0,
"pow26": 0.0,
"pow27": 0.0,
"pow28": 0.0,
"pow29": 0.0,
"pow30": 0.0,
"pow31": 0.0,
"pow32": 0.0,
"gridPurchasedTodayEnergy": 3.800,
"gridPurchasedTodayEnergyStr": "kWh",
"gridSellTodayEnergy": 2.400,
"gridSellTodayEnergyStr": "kWh",
"psumCalPec": "1",
"batteryPower": 0.099,
"batteryPowerStr": "kW",
"batteryPowerPec": "1",
"batteryCapacitySoc": 20.000,
"parallelStatus": 0,
"parallelAddr": 0,
"parallelPhase": 0,
"parallelBattery": 0,
"batteryTodayChargeEnergy": 4.800,
"batteryTodayChargeEnergyStr": "kWh",
"batteryTotalChargeEnergy": 449.000,
"batteryTotalChargeEnergyStr": "kWh",
"batteryTodayDischargeEnergy": 5.500,
"batteryTodayDischargeEnergyStr": "kWh",
"batteryTotalDischargeEnergy": 627.000,
"batteryTotalDischargeEnergyStr": "kWh",
"bypassLoadPower": 0.000,
"bypassLoadPowerStr": "kW",
"backupTodayEnergy": 0.000,
"backupTodayEnergyStr": "kWh",
"backupTotalEnergy": 0.000,
"backupTotalEnergyStr": "kWh",
"etotal": 1.153,
"etoday": 10.900,
"psum": -1.756,
"psumCal": -1.756,
"etotal1": 1153.000,
"etoday1": 10.900000,
"offlineLongStr": "--",
"etotalStr": "MWh",
"etodayStr": "kWh",
"psumStr": "kW",
"psumCalStr": "kW"
}
],
"total": 1,
"size": 20,
"current": 1,
"orders": [
],
"optimizeCountSql": false,
"searchCount": true,
"pages": 1
},
"mpptSwitch": 0
}
}

The records key is the only array in your path, that's the only one where you need to select with an index as below:
Result['data']['page']['records'][0]['etoday']

The issue might be that the key 'etoday' is not present at the location you are trying to access it from. Try using the following code to see if the key exists in the JSON object:
if 'etoday' in Result['data']['page']['records']:
print("The value of etoday is: ", Result['data']['page']['records']['etoday'])
else:
print("The key 'etoday' was not found in the Result['data']['page']['records'] dictionary.")

Unable to avoid list items in JSON file from being printed in new lines

edit
problem solved thanks to Oyono and chepner!
I'm trying to save a dictionary which includes long lists as a JSON file, without having each new item in the lists in a new line.
to save the dictionary, I'm using the command:
with open('file.json', 'w') as fp:
json.dump(coco, fp, indent=2)
The file is really big and I can not open it in Colab. so when I open it using VSC it's shown each item in the list in a separate line.
when I try to print only a small part of the dictionary in the Colab I'm getting everything in a single line.
Any ideas why could it happen or how to avoid it?
this is how it's look like in VSC:
"annotation": [
{
"segmentation": [
[
75.0,
74.5,
...(many more lines like this),
435.0,
435.5
]
],
"iscrowd": 0,
"category_id": 1,
"image_id": 43,
"id": 430,
"bbox": [
11.0,
280.0,
117.0,
156.0
],
"area": 9897,
}
],
]
}
And this is how I want it to look (and can't tell if there is actual difference between the files)
{
"segmentation": [ [ 316.0, 171.5, 320.5, 168.0, 332.5, 153.0, 332.5, 149.0, 330.0, 146.5, 305.0, 134.5, 292.0, 125.5, 280.0, 120.5, 275.0, 116.5, 270.0, 115.5, 261.5, 130.0, 258.0, 133.5, 251.5, 136.0, 255.0, 140.5, 282.0, 153.5, 285.0, 156.5, 289.0, 156.5, 296.0, 159.5, 310.0, 170.5, 316.0, 171.5 ] ],
"iscrowd": 0,
"image_id": 5,
"category_id": 1,
"id": 5,
"bbox": [ 251.5, 115.5, 81.0, 56.0 ],
"area": 2075.0
},

You have to remove indent=2 in your call to dump:
with open('file.json', 'w') as fp:
json.dump(coco, fp)
The indent keyword will save your dictionary object as a pretty printed json object, which means printing each list item in individual lines.
You can try this in a python console to see how indent influences your output json:
json.dumps({1: "a", "b": [1, 2]}, indent=2)
# will output: '{\n "1": "a",\n "b": [\n 1,\n 2\n ]\n}'
json.dumps({1: "a", "b": [1, 2]})
# will output: '{"1": "a", "b": [1, 2]}'

How to calculate some data with python?

I have JSON file. I have parsed it and I have extracted some data which are classes and code smells. Now I should calculate the number of smells on each class. I tried this with an example of code smells and it return for me the number of this smell in all the json file.
this is a part of the Json file beacause it's too long
{
"methods": [
{
"parametersTypes": [
"Bundle"
],
"sourceFile": {
"file": {
"path": "/mnt/c/shortrain-master/app/src/main/java/com/nirhart/shortrain/MainActivity.java"
}
},
"metricsValues": {
"ParameterCount": 1.0,
"NumberOfAccessedVariables": 9.0,
"ChangingClasses": 0.0,
"CouplingDispersion": 0.5,
"MethodLinesOfCode": 21.0,
"MaxNesting": 0.0,
"CyclomaticComplexity": 1.0,
"MaxCallChain": 2.0,
"ChangingMethods": 0.0,
"CouplingIntensity": 4.0
},
"fullyQualifiedName": "com.nirhart.shortrain.MainActivity.onCreate",
"smells": []
},
{
"parametersTypes": [],
"sourceFile": {
"file": {
"path": "/mnt/c/shortrain-master/app/src/main/java/com/nirhart/shortrain/MainActivity.java"
}
},
"metricsValues": {
"ParameterCount": 0.0,
"NumberOfAccessedVariables": 2.0,
"ChangingClasses": 1.0,
"CouplingDispersion": 0.0,
"MethodLinesOfCode": 6.0,
"MaxNesting": 0.0,
"CyclomaticComplexity": 1.0,
"MaxCallChain": 6.0,
"ChangingMethods": 3.0,
"CouplingIntensity": 0.0
},
"fullyQualifiedName": "com.nirhart.shortrain.MainActivity.finishActivity",
"smells": [
{
"name": "MessageChain",
"reason": "MAX_CALL_CHAIN = 6.0",
"startingLine": 54,
"endingLine": 66
}
]
},
{
"parametersTypes": [
"View"
],
"sourceFile": {
"file": {
"path": "/mnt/c/shortrain-master/app/src/main/java/com/nirhart/shortrain/MainActivity.java"
}
},
"metricsValues": {
"ParameterCount": 1.0,
"NumberOfAccessedVariables": 4.0,
"ChangingClasses": 0.0,
"CouplingDispersion": 1.0,
"MethodLinesOfCode": 6.0,
"MaxNesting": 1.0,
"CyclomaticComplexity": 3.0,
"MaxCallChain": 1.0,
"ChangingMethods": 0.0,
"CouplingIntensity": 2.0
},
"fullyQualifiedName": "com.nirhart.shortrain.MainActivity.onClick",
"smells": []
},
{
"parametersTypes": [],
"sourceFile": {
"file": {
"path": "/mnt/c/shortrain-master/app/src/main/java/com/nirhart/shortrain/MainActivity.java"
}
},
"metricsValues": {
"ParameterCount": 0.0,
"NumberOfAccessedVariables": 0.0,
"ChangingClasses": 0.0,
"CouplingDispersion": 1.0,
"MethodLinesOfCode": 3.0,
"MaxNesting": 0.0,
"CyclomaticComplexity": 1.0,
"MaxCallChain": 1.0,
"ChangingMethods": 0.0,
"CouplingIntensity": 1.0
},
"fullyQualifiedName": "com.nirhart.shortrain.MainActivity.onBackPressed",
"smells": []
}
],
"sourceFile": {
"file": {
"path": "/mnt/c/shortrain-master/app/src/main/java/com/nirhart/shortrain/MainActivity.java"
}
},
"metricsValues": {
"ClassLinesOfCode": 40.0,
"OverrideRatio": null,
"WeighOfClass": 1.0,
"LCOM2": 0.5,
"TightClassCohesion": 0.0,
"LCOM3": 0.6666666666666666,
"NumberOfAccessorMethods": 0.0,
"WeightedMethodCount": 6.0,
"IsAbstract": 0.0,
"PublicFieldCount": 0.0
},
"fullyQualifiedName": "com.nirhart.shortrain.MainActivity",
"smells": []
},
]
This is my code:
import pandas as pd
import json
all_smells=['LazyClass','ComplexClass','LongParameterList','FeatureEnvy','LongMethod','GodClass','MessageChain']
with open('/content/result_smells.json') as handle:
dictdump = json.loads(handle.read())
my_map = {}
for elem in dictdump :
my_map[elem["fullyQualifiedName"]] = []
#adding all class smells
for class_smell in elem["smells"] :
my_map[elem["fullyQualifiedName"]].append(class_smell)
#adding all methods smells
for method in elem["methods"] :
for method_smell in method["smells"] :
my_map[elem["fullyQualifiedName"]].append(method_smell)
for elem in my_map :
print(elem)
for smell in my_map[elem] :
print(smell["name"])
This is the result : the name of class and the smells on it
com.nirhart.shortrain.MainActivity
MessageChain
com.nirhart.shortrain.path.PathParser
ComplexClass
FeatureEnvy
LongParameterList
LongParameterList
LongMethod
com.nirhart.shortrain.path.PathPoint
LazyClass
LongParameterList
com.nirhart.shortrain.path.TrainPath
FeatureEnvy
com.nirhart.shortrain.rail.RailActionActivity
FeatureEnvy
LongMethod
com.nirhart.shortrain.rail.RailInfo
com.nirhart.shortrain.train.TrainActionActivity
ComplexClass
SpaghettiCode
LongMethod
LongMethod
IntensiveCoupling
I try to calculate the number of the MessageChain smell in com.nirhart.shortrain.MainActivity class which is one it return for me 5 which is the MessageChain smell in all the json file
this is my code:
x=0
for elem in my_map :
print(elem)
for smell in my_map[elem] :
if smell["name"]=='MessageChain':
x+=1
Then I need to put all results on a CSV to analyse it.
this is an exemple of csv file with one smell

A Python Counter() can be used to simplyfy counting the smells, and a csv.DictWriter() can be used to then write the resulting dictionary holding all of the counts. For example:
from collections import Counter
import csv
import json
all_smells = ['LazyClass', 'ComplexClass', 'LongParameterList', 'FeatureEnvy', 'LongMethod', 'GodClass', 'MessageChain']
my_map = {}
with open('result_smells.json') as f_json:
json_data = json.load(f_json)
for entry in json_data:
my_map[entry["fullyQualifiedName"]] = []
#adding all class smells
for class_smell in entry["smells"] :
my_map[entry["fullyQualifiedName"]].append(class_smell)
#adding all methods smells
for method in entry["methods"] :
for method_smell in method["smells"] :
my_map[entry["fullyQualifiedName"]].append(method_smell)
with open('output.csv', 'w', newline='') as f_output:
csv_output = csv.DictWriter(f_output, fieldnames=["NameOfClass", *all_smells], extrasaction='ignore', restval='NaN')
csv_output.writeheader()
for elem in my_map :
smell_counts = Counter()
for smell in my_map[elem] :
smell_counts[smell["name"]] += 1
smell_counts['NameOfClass'] = elem
csv_output.writerow(smell_counts)
Giving you an output CSV file looking like:
NameOfClass,LazyClass,ComplexClass,LongParameterList,FeatureEnvy,LongMethod,GodClass,MessageChain
com.nirhart.shortrain.MainActivity,NaN,NaN,NaN,NaN,NaN,NaN,1
com.nirhart.shortrain.path.PathParser,NaN,1,2,1,1,NaN,NaN
com.nirhart.shortrain.path.PathPoint,1,NaN,1,NaN,NaN,NaN,NaN
com.nirhart.shortrain.path.TrainPath,NaN,NaN,NaN,1,NaN,NaN,NaN
com.nirhart.shortrain.rail.RailActionActivity,NaN,NaN,NaN,1,1,NaN,NaN
com.nirhart.shortrain.rail.RailInfo,NaN,NaN,NaN,NaN,NaN,NaN,NaN
com.nirhart.shortrain.train.TrainActionActivity,NaN,1,1,1,2,NaN,2
com.nirhart.shortrain.train.TrainDirection,NaN,NaN,NaN,NaN,NaN,NaN,NaN
com.nirhart.shortrain.train.TrainView,NaN,NaN,NaN,NaN,NaN,NaN,NaN
com.nirhart.shortrain.tutorial.TutorialFragment,NaN,NaN,NaN,NaN,NaN,NaN,NaN
com.nirhart.shortrain.tutorial.TutorialFragment.OnNextSlideClicked,1,NaN,NaN,NaN,NaN,NaN,NaN
com.nirhart.shortrain.tutorial.TutorialViewPagerAdapter,1,NaN,NaN,1,NaN,NaN,NaN
com.nirhart.shortrain.utils.ShortcutsUtils,NaN,NaN,1,NaN,NaN,NaN,2

JSON file to dataframe conversion-ValueError: Unexpected character found when decoding array value (2)

I have a huge json file having the format of the form:
[{"faceId": "2cb5a26a-1acc-4eb2-8c39-8e05e604f057", "faceRectangle": {"top":
54, "left": 125, "width": 78, "height": 78}, "faceAttributes": {"smile":
0.584, "headPose": {"pitch": 0.0, "roll": -2.4, "yaw": 1.4}, "gender":
"male", "age": 34.4, "facialHair": {"moustache": 0.5, "beard": 0.6,
"sideburns": 0.3}, "glasses": "NoGlasses", "emotion": {"anger": 0.0,
"contempt": 0.003, "disgust": 0.0, "fear": 0.0, "happiness": 0.584,
"neutral": 0.413, "sadness": 0.0, "surprise": 0.0}, "blur": {"blurLevel":
"high", "value": 1.0}, "exposure": {"exposureLevel": "goodExposure",
"value": 0.61}, "noise": {"noiseLevel": "low", "value": 0.0}, "makeup":
{"eyeMakeup": false, "lipMakeup": false}, "accessories": [{"type":
"headwear", "confidence": 1.0}], "occlusion": {"foreheadOccluded": true,
"eyeOccluded": false, "mouthOccluded": false}, "hair": {"bald": 0.0,
"invisible": true, "hairColor": []}}}]
I am trying to convert this to a dataframe by using the following code:
import pandas as pd
ent_json_file = 'ent.json'
reading_json = pd.read_json(ent_json_file,convert_dates=True,lines=True)
I am getting the following error:
ValueError: Unexpected character found when decoding array value (2)
Any help is greatly appreciated!
Thanks!

Your error seems to be coming from an invalid JSON format.
I've found that online JSON lints are a quick and easy way to figure out what's going on with a JSON file:
https://jsonlint.com/

In Python, matching the first occurrence of a word AFTER the occurrence of another word

I am attempting to replace the occurrence of a word that occurs only after another word in a JSON text string. I have been struggling to use regular expressions to do this but using just Python functions will be fine with me.
So what I want is to find the first occurrence of "LEVEL1": (with quotes), then find first occurrence of "session_transition":, then find whatever string is in quotes after "session_transition":, and then replace it with another string. Here is the string I am working with:
"BASELINE": {
"audio_volume": 150,
"cry_threshold": 70,
"cry_transition": "LEVEL1",
"expected_volume": 63,
"led_color": "BLUE",
"led_blink_speed": "NONE",
"motor_amplitude": 0.97,
"motor_frequency": 0.5,
"power_transition": "SUSPENDED",
"seconds_to_ignore_cry": 10.0,
"seconds_in_state": -1.0,
"session_transition": "ONLINE",
"track": "RoR",
"timer_transition": null,
"active_session" : 1
},
"LEVEL1": {
"audio_volume": 300,
"cry_threshold": 75,
"expected_volume": 63,
"cry_transition": "LEVEL2",
"led_color": "PURPLE",
"led_blink_speed": "NONE",
"motor_amplitude": 0.76,
"motor_frequency": 1.20,
"power_transition": "SUSPENDED",
"seconds_to_ignore_cry": 10.0,
"seconds_in_state": 480.0,
"session_transition": "ONLINE",
"track": "RoR",
"timer_transition": "BASELINE",
"active_session" : 1
}
}
For instance, below I want to find and replace "ONLINE" under "LEVEL1": --> "session_transition": to "OFFLINE" so it'll look like this:
"LEVEL1": {
"audio_volume": 300,
"cry_threshold": 75,
"expected_volume": 63,
"cry_transition": "LEVEL2",
"led_color": "PURPLE",
"led_blink_speed": "NONE",
"motor_amplitude": 0.76,
"motor_frequency": 1.20,
"power_transition": "SUSPENDED",
"seconds_to_ignore_cry": 10.0,
"seconds_in_state": 480.0,
"session_transition": "OFFLINE",
"track": "RoR",
"timer_transition": "BASELINE",
"active_session" : 1
}
So far I have r"(?<=\"LEVEL1\"\:).* to match the first occurrence but don't know how to proceed further.

I would suggest you to use the JSON library which is inbuilt in Python. You can easily convert the JSON to a Python Dict Object. This would prevent you from complex regex as well. It's better to reduce complexity in readability as well.
Documentation:
Python 3.4
Python 2.7
In Python 2
import json
jsonDict = json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]')
print(jsonDict)
Result:
['foo', {'bar': ['baz', None, 1.0, 2]}]
You can then easily do manipulations as you would do to a Python Dictionary. Seems more intuitive this way.
Once you are done you can convert it using
jsonStr = json.dumps(jsonDict)
print(jsonStr)
Result:
["foo", {"bar":["baz", null, 1.0, 2]}]

Ok so I was able to solve this in the following way:
#1)Find the unique string "LEVEL1": and save its index
after_index = configuration_in_text_format.index('"LEVEL1":')
#2)Starting from previous index, find the "session_transition": string and save its index
after_index = configuration_in_text_format.find('"session_transition":', after_index)
#3)Create a new string of bottom part with "session_transition": as first line
new_config_in_text_format = configuration_in_text_format[after_index:]
#4)Remove the first line with "session_transition": in it
new_config_in_text_format = new_config_in_text_format[new_config_in_text_format.find('\n')+1:]
#5)Create a new string to replace the deleted new line
new_line_str = '"session_transition": "OFFLINE",\n'
#6)Put new string on top of remaining text, effectively replacing old line
new_config_in_text_format = new_line_str + new_config_in_text_format
#7)Take the top part of original text and append the newly modified bottom part
new_config_in_text_format = configuration_in_text_format[:after_index] + new_config_in_text_format
print new_config_in_text_format
And the proper output:
"BASELINE": {
"audio_volume": 150,
"cry_threshold": 70,
"cry_transition": "LEVEL1",
"expected_volume": 63,
"led_color": "BLUE",
"led_blink_speed": "NONE",
"motor_amplitude": 0.97,
"motor_frequency": 0.5,
"power_transition": "SUSPENDED",
"seconds_to_ignore_cry": 10.0,
"seconds_in_state": -1.0,
"session_transition": "ONLINE",
"track": "RoR",
"timer_transition": null,
"active_session" : 1
},
"LEVEL1": {
"audio_volume": 300,
"cry_threshold": 75,
"expected_volume": 63,
"cry_transition": "LEVEL2",
"led_color": "PURPLE",
"led_blink_speed": "NONE",
"motor_amplitude": 0.76,
"motor_frequency": 1.20,
"power_transition": "SUSPENDED",
"seconds_to_ignore_cry": 10.0,
"seconds_in_state": 480.0,
"session_transition": "OFFLINE",
"track": "RoR",
"timer_transition": "BASELINE",
"active_session" : 1
},

I think you can do this somewhat easily using string.index()
first_index = some_string.index('"Level1"')
second_index = some_string[first_index:].index('"Online"')
after that I leave it up to you to replace the string. You should be able to do it using some_string[second_index:].split('"') and then use splicing and join to put it back together.