I have a JSON file that contains some data.
testJSON.json
{
"hourlyData": [
{
"frequency": "49.96",
"actual": " 2,240.43 ",
"schedule": " 2,223.85 ",
"acp": "325"
},
{
"frequency": "50.04",
"actual": " 1,862.88 ",
"schedule": " 1,881.09 ",
"acp": "275"
},
{
"frequency": "50.04",
"actual": " 1,882.17 ",
"schedule": " 1,885.94 ",
"acp": "275"
}
],
"storageData": [
{
"config": "ESS1",
"name": "BRPL",
"Power Rating": "20",
"Energy Rating": "20",
"socLow": "0",
"socHigh": "0.8",
"Charge Eff": "0.9273",
"Discharge Eff": "0.9273",
"Round Trip Eff": "0.922",
"Lower Limit": "57",
"Mid Limit": "76",
"High Limit": "95",
"Thrushold": "5",
"Discharging Price": "6"
}
],
I want to store these values into python variables. So what I have done is that first I created a dictionary which contain different variables of different types then I created a function which simple opens the JSON file then I try to store those json values into declared variables:
test.py
import json
#decalaring variables
storageData = {
"name": 'No name specified',
"powerRating":-1,
"energyRating":-1,
"annualMaxCycles":365,
"socLow":-1,
"socHigh":-1,
"chargeEff":-1,
"dChargeEff":-1,
"lowerLimit": -1,
"midLimit": -1,
"highLimit": -1,
"thrushold": 5,
"dischargingPrice": 6
}
marketData = {
"marketProducts": {
"dsm":{
"frequency":[],
"schedule":[],
"actual":[],
"acp": [],
}
}
inputMode = 'JSON'
JSONfileName = "testJSON.json"
def inputJSON():
if (inputMode == 'JSON'):
fileName = JSONfileName
# Import data from JSON files
with open(JSONfileName, 'r') as myfile:
dataJSON = ((myfile.read().replace('\n', '')))
inputJSON = json.loads(dataJSON)
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
marketData['marketProducts']['dsm']['frequency'] = inputJSON['hourlyData']['frequency']
marketData['marketProducts']['dsm']['acp'] = inputJSON['hourlyData']['acp']
marketData['marketProducts']['dsm']['actual'] = inputJSON['hourlyData']['actual']
marketData['marketProducts']['dsm']['schedule'] = inputJSON['hourlyData']['schedule']
inputJSON()
error that it gives me
Traceback (most recent call last):
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 113, in <module>
inputJSON()
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 80, in inputJSON
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
TypeError: list indices must be integers or slices, not str
Instead of
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
Do
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData[0]']['Power Rating']
storageData['energyRating'] = inputJSON['storageData[0]']['energyRating']
storageData['warranty'] = inputJSON['storageData[0]']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData[0]']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData[0]']['socLow']
storageData['socHigh'] = inputJSON['storageData[0]']['socHigh']
storageData['chargeEff'] = inputJSON['storageData[0]']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData[0]']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData[0]']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData[0]']['midLimit']
storageData['highLimit'] = inputJSON['storageData[0]']['highLimit']
storageData['thrushold'] = inputJSON['storageData[0]']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData[0]']['dischargingPrice']
Because it's an array, whose 1st index has the json data.
Ex. for "hourlyData"
for(data of inputJSON['hourlyData']){
marketData['marketProducts']['dsm']['frequency'] = data['frequency']
}
Hope that helps.
Related
I have 3 csv files, loan, customer and security, each of these files are quite large (800k+ rows). Each file is linked by a single column uniqueid. My aim is to create a single JSON file, the code below achieves this, however, it is very slow.
My question is how can i achieve this result faster?
import csv
import json
def multicsvtojson():
loanscsvfile = open('C:\\***\\loan.csv', 'r')
custcsvfile = open('C:\\***\\customer.csv', 'r')
securcsvfile = open('C:\\***\\security.csv', 'r')
loansreader = csv.DictReader(loanscsvfile, delimiter=',')
custreader = csv.DictReader(custcsvfile, delimiter=',')
securreader = csv.DictReader(securcsvfile, delimiter=',')
jsonfile = open('test.json', 'w')
#ready json file
output = []
loanscount = 0
#total loan count
for loansrow in loansreader:
loansrow['customers'] = []
loansrow['securities'] = []
output.append(loansrow)
custcsvfile.seek(0)
securcsvfile.seek(0)
for custrow in custreader:
if (loansrow["UniqueID"] == custrow["UniqueID"]):
loansrow['customers'].append(custrow)
for securrow in securreader:
if (loansrow["UniqueID"] == securrow["UniqueID"]):
loansrow['securities'].append(securrow)
loanscount = loanscount + 1 #increment the loan counter
print(loanscount)
total = {}
total['total'] = loanscount
output.insert(0, total)
json.dump(output, jsonfile, indent=4)
The current output is as follow
[{
"total": 2
},
{
"uniqueID": "",
"uniqueID2": "",
"colA": "",
"colB": "",
"colC": "",
"colD": "",
"customers": [
{
"uniqueID": "",
"custID": "",
"colA": "",
"colB": "",
}
],
"securities": [
{
"uniqueID": "",
"secuID": "",
"colA": "",
"colB": ""
}
]
},
{
"uniqueID": "",
"uniqueID2": "",
"colA": "",
"colB": "",
"colC": "",
"colD": "",
"customers": [
{
"uniqueID": "",
"custID": "",
"colA": "",
"colB": "",
},
{
"uniqueID": "",
"secuID": "",
"colA": "",
"colB": ""
}
],
"securities": [
{
"uniqueID": "",
"secuID": "",
"colA": "",
"colB": ""
},
{
"uniqueID": "",
"secuID": "",
"colA": "",
"colB": ""
}
]
}
}]
What probably costs you performance are the multiple reads of the customer and securities files. Because you reread the whole file at every loan row.
Maybe try to regroup customers and securties by ID before getting to the loans file, so you don't have to re-read every time.
import csv
import json
def multicsvtojson():
loanscsvfile = open('C:\\***\\loan.csv', 'r')
custcsvfile = open('C:\\***\\customer.csv', 'r')
securcsvfile = open('C:\\***\\security.csv', 'r')
loansreader = csv.DictReader(loanscsvfile, delimiter=',')
custreader = csv.DictReader(custcsvfile, delimiter=',')
securreader = csv.DictReader(securcsvfile, delimiter=',')
jsonfile = open('test.json', 'w')
#ready json file
output = []
loanscount = 0
# regroup customers by ID
customers = {}
for custrow in custreader:
id = custrow["UniqueID"]
if id not in customers:
customers[id] = []
customers[id].append(custrow)
# regroup securities by ID
securities = {}
for securrow in securreader:
id = securrow["UniqueID"]
if id not in securities:
securities[id] = []
securities[id].append(securrow)
#total loan count
for loansrow in loansreader:
loansrow['customers'] = customers.get("UniqueID", [])
loansrow['securities'] = securities.get("UniqueID", [])
output.append(loansrow)
loanscount = loanscount + 1 #increment the loan counter
print(loanscount)
total = {}
total['total'] = loanscount
output.insert(0, total)
json.dump(output, jsonfile, indent=4)
If the CSV files are too big to handle in memory, you can also try to use a temporary database such as tinydb.
I am trying to create one nested Python dictionary called Results.
I am using AWS Rekognition to get an image and output the results.
The results_dict only contains one result after it's complated, and I wish to have all the results in one nested loop
I'm trying to get:
{
"Results": [
{
"Name": "Human",
"Confidence": 98.87621307373047,
},
{
"Name": "Face",
"Confidence": 98.87621307373047,
},
{
"Name": "Person",
"Confidence": 98.87621307373047,
},
]
}
But I'm getting:
{
'Results':
{
'Name': 'Paper',
'Confidence': 57.299766540527344
}
}
The code is replacing the text, and I want to add another set of Name and Confidence.
My code is:
import boto3
import json
BUCKET = "*****"
FOLDER = 'testing/'
JOEY = FOLDER + "Joey_30_Sept.png"
BEYONCE = FOLDER + "beyonce_rekognition_moderation_testing.jpg"
MANBEARD = FOLDER + "man_beard.jpg"
MEN = FOLDER + "men_group.jpg"
client = boto3.client('rekognition')
response = client.detect_labels(Image=
{'S3Object': {
'Bucket': BUCKET,
'Name': JOEY
}},
MaxLabels = 10,
MinConfidence=0)
results_dict = {}
results_dict['Results'] = {}
results_dict['Results']['Name'] = ""
results_dict['Results']['Confidence'] = ""
for label in response['Labels']:
name = label['Name'] #to get the whole bounding box.
confidence = label['Confidence']
name_str = str(name)
conf_str = str(confidence)
results_dict["Results"]["Name"] = label['Name']
results_dict["Results"]["Confidence"] = label['Confidence']
print(results_dict)
You defined results_dict['Results'] as dictionary as dict not list:
...
results_dict = {}
results_dict['Results'] = []
results_dict['Results']['Name'] = ""
results_dict['Results']['Confidence'] = ""
for label in response['Labels']:
name = label['Name'] #to get the whole bounding box.
confidence = label['Confidence']
name_str = str(name)
conf_str = str(confidence)
results_dict['Results'].append({["Name": name_str, "Confidence": conf_str })
print(results_dict)
I used this answer Create array of json objects from for loops and works really well for "plain" JSON objects, but if I have a nested element is not working correctly, this is my code:
story_project=open('json_jira/stories/stories_to_jira_TESTEST.json', 'w+'
#######Projects############
json_projects = []
p_name_a, p_key_a, p_type_a = [], [], []
#######Issues##############
json_issues = []
i_summary_a, i_created_a, i_reporter_a, i_status_a, i_issue_type_a = [], [], [], [], []
#######Custom Fields########
json_custom_field_values = []
cf_field_name_a, cf_field_type_a, cf_value_a = [], [], []
#########The Values################
p_name_a.append("ClubHouseDEV")
p_key_a.append("CLUB")
p_type_a.append("software")
i_summary_a.append("This summary doesn not exist")
i_created_a.append("2017-07-17T02:35:16Z")
i_reporter_a.append("5a02285487c3eb1913c44a80")
i_status_a.append("Open")
i_issue_type_a.append("Milestones")
cf_field_name_a.append("external_id")
cf_field_type_a.append("com.atlassian.jira.plugin.system.customfieldtypes:float")
cf_value_a.append(3)
cf_field_name_a.append("Story Points")
cf_field_type_a.append("com.atlassian.jira.plugin.system.customfieldtypes:float")
cf_value_a.append(5)
###########Build The JSON##############
json_custom_field_values = [{"fieldName": cf_field_name, "fieldType": cf_field_type, "value": cf_value} for cf_field_name, cf_field_type, cf_value in zip(cf_field_name_a, cf_field_type_a, cf_value_a)]
json_issues = [{"sumamry": i_summary, "created": i_created, "reporter": i_reporter, "status": i_status, "issueType": i_issue_type, "customFieldValues" : json_custom_field_value} for i_summary, i_created, i_reporter, i_status, i_issue_type, json_custom_field_value in zip(i_summary_a, i_created_a, i_reporter_a, i_status_a, i_issue_type_a, json_custom_field_values)]
json_projects = [{"name": p_name, "key": p_key, "type": p_type, "issues": json_issue} for p_name, p_key, p_type,json_issue in zip(p_name_a, p_key_a, p_type_a,json_issues)]
json_file = [{"projects": json_project} for json_project in zip(json_projects)]
json.dump(json_file, story_project)
The output should be:
{ "projects": [
{
"name": "ClubHouseDEV",
"key": "CLUB",
"type":"software",
"issues":
[
{
"summary":"This summary doesn not exist",
"created":"2017-07-17T02:35:16Z",
"reporter":"5a02285487c3eb1913c44a80",
"status":"Open",
"issueType":"Milestones",
"customFieldValues":
[
{
"fieldName": "external_id",
"fieldType": "com.atlassian.jira.plugin.system.customfieldtypes:float",
"value": 3
},
{
"fieldName": "Story Points",
"fieldType": "com.atlassian.jira.plugin.system.customfieldtypes:float",
"value": 5
}
],
"labels" : ["ch_epics"],
"updated": "2017-07-17T02:35:16Z"
}
]
}
]
}
But it is:
[{"projects": [{"name": "ClubHouseDEV", "key": "CLUB", "type": "software", "issues": {"sumamry": "This summary doesn not exist", "created": "2017-07-17T02:35:16Z", "reporter": "5a02285487c3eb1913c44a80", "status": "Open", "issueType": "Milestones", "customFieldValues": {"fieldName": "external_id", "fieldType": "com.atlassian.jira.plugin.system.customfieldtypes:float", "value": 3}}}]}]
As you can see it only added one value on the nested "Custom Field Values", how can I add all the values.
This is how I solve id: Build the deepest level, then integrated it with the level up and so on, the output was the expected.
###########Build The JSON##############
for cf_field_name, cf_field_type, cf_value in zip(cf_field_name_a, cf_field_type_a, cf_value_a):
json_custom_field_values = {}
json_custom_field_values["fieldName"] = cf_field_name
json_custom_field_values["fieldType"] = cf_field_type
json_custom_field_values["value"] = cf_value
cf_data.append(json_custom_field_values)
for i_summary, i_created, i_reporter, i_status, i_issue_type in zip(i_summary_a, i_created_a, i_reporter_a, i_status_a, i_issue_type_a):
json_issues = {}
json_issues["summary"] = i_summary
json_issues["created"] = i_created
json_issues["reporter"] =i_reporter
json_issues["status"] = i_status
json_issues["issueType"] = i_issue_type
# json_issues["customFieldValues"] = [{"fieldName": cf_field_name, "fieldType": cf_field_type, "value": cf_value} for cf_field_name, cf_field_type, cf_value in zip(cf_field_name_a, cf_field_type_a, cf_value_a)]
json_issues["customFieldValues"] = cf_data
issues_data.append(json_issues)
for p_name, p_key, p_type in zip(p_name_a, p_key_a, p_type_a):
json_projects = {}
json_projects["name"] = p_name
json_projects["key"] = p_key
json_projects["type"] = p_type
json_projects["issues"] = issues_data
projects_data.append(json_projects)
json_dict["projects"] = projects_data
json.dump(json_dict, story_project)
I would like to iterate through a list of dictionaries and save values of certain keys (in my case "consumer Key" and "consumer Secret") as many times they are present into another dictionary.
Problem: I'm able to iterate through the list but my code is not saving the second consumer key and consumer secret, instead it is saving the first consumer key and consumer secret twice.
Input:
{
"accessType": "",
"apiProducts": [],
"appFamily": "default",
"appId": "ac56c8b2-6ac1-4971-a1d3-4bf97893c067",
"attributes": [
{
"name": "DisplayName",
"value": "quotaapp"
},
{
"name": "Notes",
"value": ""
}
],
"callbackUrl": "",
"createdAt": 1549274952045,
"createdBy": "suraj.pai.airody#sap.com",
"credentials": [
{
"apiProducts": [
{
"apiproduct": "apiprod",
"status": "approved"
}
],
"attributes": [],
"consumerKey": "xyz",
"consumerSecret": "abc",
"expiresAt": -1,
"issuedAt": 1549274952051,
"scopes": [],
"status": "approved"
},
{
"apiProducts": [
{
"apiproduct": "ouathTest-Product",
"status": "approved"
}
],
"attributes": [],
"consumerKey": "pqr",
"consumerSecret": "wmn",
"expiresAt": -1,
"issuedAt": 1554802431452,
"scopes": [],
"status": "approved"
}
],
"developerId": "xyz",
"lastModifiedAt": 1554802431662,
"lastModifiedBy": "suraj.pai.airody#sap.com",
"name": "quotaapp",
"scopes": [],
"status": "approved"
}
Code:
import requests
import json
from requests.auth import HTTPBasicAuth
import csv
def get_v2details():
a = 'orgID1'
b = 'appID1'
c = 'ConKey1'
d = 'ConSecret1'
e = 'appName1'
org_lst = []
some_dict = {}
con_blst = [] # variable to append the dictionary app level
n = int(input("Enter number of orgs from Landscape 1: "))
for i in range(0, n):
ele = str(input())
org_lst.append(ele)
cmp_orglst = list(org_lst)
print(cmp_orglst)
for j in cmp_orglst:
url = "https://canarydevmgmtsrv.dmzmo.sap.corp/v1/o/" + str(j) + "/apps/"
headers = {'Content-Type': 'application/json'}
response = requests.get(url, auth=HTTPBasicAuth('xyz', 'xyz'), headers=headers, verify=False)
app_data = json.loads(response.text)
print(app_data)
for k in app_data:
url1 = "https://canarydevmgmtsrv.dmzmo.sap.corp/v1/o/" + str(j) + "/apps/" + str(k)
headers = {'Content-Type': 'application/json'}
response1 = requests.get(url1, auth=HTTPBasicAuth('xyz', 'xyz'), headers=headers, verify=False)
consumer_data = json.loads(response1.text)
print(" Consumer Data is ", consumer_data)
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][0]['consumerKey']
some_dict[d] = consumer_data['credentials'][0]['consumerSecret']
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
print(con_blst)
csv_columns = ['orgID1', 'appName1', 'appID1', 'ConKey1', 'ConSecret1']
csv_file = "Names1.csv"
try:
with open(csv_file, 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
writer.writeheader()
for data in con_blst:
writer.writerow(data)
except IOError:
print("I/O error")
Expected result:
orgID1 appName1 appID1 ConKey1 ConSecret1
VALIDATE quotaapp 4bf97893c067 xyz abc
VALIDATE quotaapp 4bf97893c067 pqr wmn
Actual result:
orgID1 appName1 appID1 ConKey1 ConSecret1
VALIDATE quotaapp 4bf97893c067 xyz abc
VALIDATE quotaapp 4bf97893c067 xyz abc
It seems you just made a small error.
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][0]['consumerKey'] #this line
some_dict[d] = consumer_data['credentials'][0]['consumerSecret'] #and this line
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
Should be
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][l]['consumerKey'] # Here
some_dict[d] = consumer_data['credentials'][l]['consumerSecret'] # Here
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
You weren't looping through consumer_data['credentials'], you were just storing consumer_data['credentials'][0] twice
#current forecast
current_api = 'api.openweathermap.org/data/2.5/weather?zip='
current_url_zip = current_api + urllib.parse.urlencode({'Zip': zip})
#current_url_key =
json_data = requests.get(future_url_key).json()
#print (json_data)
future_temp_day_0 = json_data['list'][0]['main']['temp'] #current day
future_temp_day_1 = json_data['list'][1]['main']['temp'] #tomorrow
future_description_day_0 = json_data['list'][0]['weather']['description'] #current description
future_description_day_1 = json_data['list'][1]['weather']['description'] #current description
#Kelvin to F conversion
fTemp_0 = int((future_temp_day_0 - 273.15) * (9/5) + (32))
fTemp_1 = int((future_temp_day_1 - 273.15) * (9/5) + (32))
So I am using the openweathermap api. I want to be able to pull the current day [temperature][weather description] and tomorrows [temperature][weather description]. The problem is when I try to reference the [weather description] it pulls it from json_data['list'][3] and not json_data['list'][1]. It iterates to the next spot even though I am referencing the [1] item.
{
"cod":"200",
"message":0.0122,
"cnt":40,
"list":[
{
"dt":1519074000,
"main":{
"temp":283.99,
"temp_min":281.801,
"temp_max":283.99,
"pressure":989.94,
"sea_level":1029.29,
"grnd_level":989.94,
"humidity":52,
"temp_kf":2.19
},
"weather":[
{
"id":801,
"main":"Clouds",
"description":"few clouds",
"icon":"02d"
}
],
"clouds":{
"all":20
},
"wind":{
"speed":3.36,
"deg":325.001
},
"rain":{
},
"sys":{
"pod":"d"
},
"dt_txt":"2018-02-19 21:00:00"
},
{
"dt":1519084800,
"main":{
"temp":282.64,
"temp_min":281.177,
"temp_max":282.64,
"pressure":990.6,
"sea_level":1029.94,
"grnd_level":990.6,
"humidity":47,
"temp_kf":1.46
},
"weather":[
{
"id":802,
"main":"Clouds",
"description":"scattered clouds",
"icon":"03n"
}
],
"clouds":{
"all":36
},
"wind":{
"speed":3.17,
"deg":319.502
},
"rain":{
},
"sys":{
"pod":"n"
},
"dt_txt":"2018-02-20 00:00:00"
}
The weather key contains a list of dicts, so you should use [0] if you want the description of the first entry of the list:
future_description_day_0 = json_data['list'][0]['weather'][0]['description']
future_description_day_1 = json_data['list'][1]['weather'][0]['description']