We Keep Coding

Unable to append data to Json array object with desired output

My code
import json
import re
from http.client import responses
import vt
import requests
with open('/home/asad/Downloads/ssh-log-parser/ok.txt', 'r') as file:
file = file.read()
pattern = re.compile(r'\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3}')
ips = pattern.findall(file)
unique_ips = list(set(ips))
# print(unique_ips)
# print(len(unique_ips))
headers = {
"accept": "application/json",
"x-apikey": "9765ba5d9fd52f5747dde5240606019f83d32758cb664abc63a43488aa42812d"
}
i = 0
url = "https://www.virustotal.com/api/v3/ip_addresses/"
# messages = \[\]
f = open('formater.json')
# returns JSON object as
# a dictionary
data = json.load(f)
f.close()
no = 0
while i \< len(unique_ips):
furl = url + str(unique_ips[i])
# response = requests.get(furl, headers=headers)
# data_ = response.json()
# print(data_)
# messages = [data_['data']['attributes']['last_analysis_results']]
messages = [data['data']['attributes']['last_analysis_results']]
y = json.dumps(messages)
y1 = json.loads(y)
# print(y1)
a = []
r = []
v = []
cnt = 0
store = len(y1[0])
out_json_new = []
out_json1 ={}
for o in y1:
for k, vv in o.items():
a_ = vv['result']
a.append(a_)
r_ = vv['engine_name']
r.append(r_)
v_ = vv['category']
v.append(v_)
out_json = {
"indicators": [{
"value": str(unique_ips[i]),
"type": 'ip',
}]
}
out_json1 ={
"providers":[{
"provider": str(r),
"verdict": str(a),
"score": str(v)
}] }
out_json1['providers'].append(out_json1)
i += 1
print(out_json,out_json1)
\#for aaa in a:
\#print(a\[0\])\`
Outputs as
{'indicators': [{'value': '192.91.72.201', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
I want to change the output to this format.
{
"providers":\[
{
"provider":"['Bkav']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC Threat Intelligence']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC sarah']",
"verdict":"['hate']",
"score":"['harmless']"
}
]
}
My current code, groups under one key e.g provider instead it should be appended one after another like in output above. I tried to use append logic but its not working as i attended. It output as
`out_json1['providers'].append(out_json1)`
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
RELEVANT FILES
In order to run the code these files are required.
ok.txt
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.201
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.101
formater.json
{
"data": {
"attributes": {
"country": "US",
"last_analysis_stats": {
"harmless": 86,
"malicious": 0,
"suspicious": 0,
"undetected": 0,
"timeout": 0
},
"last_analysis_results": {
"Bkav": {
"category": "harmless",
"result": "clean",
"method": "blacklist",
"engine_name": "Bkav"
},
"CMC Threat Intelligence": {
"category": "harmless",
"result": "legs",
"method": "blacklist",
"engine_name": "CMC Threat Intelligence"
},
"CMC sarah ": {
"category": "sarah",
"result": "hate",
"method": "you",
"engine_name": "CMC sarah "
}
}
}
}
}

How to read a .json file in pandas to export it as a readable .csv file

I have created a .json file by appending a number of json strings using a get request. My aim is to convert the appended .json file into a readable .csv file. The .json file has the following format:
[{
"trades":[
{
"id": 20995465,
"unique_identifier": null,
"transaction_type": "BUY",
"transaction_date": "2016-11-08",
"symbol": "RDSA",
"market": "LSE",
"quantity": 10,
"price": 20.84,
"exchange_rate": 0.5525,
"brokerage": 3.619909502,
"brokerage_currency_code": "GBP",
"value": 380.81,
"comments": null,
"portfolio_id": 293304,
"holding_id": 6258682,
"instrument_id": 32021,
"confirmed": true,
"links": {
"portfolio": "https://api.sharesight.com/api/v3/portfolios/293304"
}
}
],
"links":{
"self":"https://api.sharesight.com/api/v3/portfolios/2/trades"
}
},
{
"trades":[
{
"id": 20995425,
"unique_identifier": null,
"transaction_type": "BUY",
"transaction_date": "2018-11-08",
"symbol": "PDSA",
"market": "LSE",
"quantity": 1,
"price": 2.84,
"exchange_rate": 0.25,
"brokerage": 7.619909502,
"brokerage_currency_code": "GBP",
"value": 80.81,
"comments": null,
"portfolio_id": 293604,
"holding_id": 6258635,
"instrument_id": 32023,
"confirmed": true,
"links": {
"portfolio": "https://api.sharesight.com/api/v3/portfolios/293604"
}
}
],
"links":{
"self":"https://api.sharesight.com/api/v3/portfolios/2/trades"
}
}
]
My attempt
client_id = 'ClientID'
client_secret = 'ClientSecret'
access_token_url='https://api.sharesight.com/oauth2/token'
client = BackendApplicationClient(client_id=client_id)
oauth = OAuth2Session(client=client)
token = oauth.fetch_token(token_url=access_token_url, client_id=client_id, client_secret=client_secret)
access_token = token['access_token']
head = {'Authorization': f'Bearer {access_token}'}
# Get the portfolios
r = requests.get('https://api.sharesight.com/api/v2/portfolios.json', headers=head)
# print(r)
j = r.json()
# print(j)
rjs = []
for p in j['portfolios']:
# print(p)
name = p['name']
pid = p['id']
print(f'Retrieving {name} - {pid}')
vurl = f'https://api.sharesight.com/api/v2/portfolios/{pid}/trades.json'
r = requests.get(vurl, headers=head)
rj = r.json()
rjs.append(rj)
with open('/Users/Filename.json', 'w') as json_file:
json.dump(rjs, json_file)
# Opening JSON file and loading the data
# into the variable data
with open('/Users/Filename.json') as json_file:
data = json.load(json_file)
trades_data = data['trades']
# now we will open a file for writing - create a blank .csv file
data_file = open('/Users/Filename.csv', 'w')
# create the csv writer object
csv_writer = csv.writer(data_file)
# Counter variable used for writing
# headers to the CSV file
count = 0
for emp in trades_data:
if count == 0:
# Writing headers of CSV file
header = emp.keys()
csv_writer.writerow(header)
count += 1
# Writing data of CSV file
csv_writer.writerow(emp.values())
data_file.close()
Error Code
trades_data = data['trades']
TypeError: list indices must be integers or slices, not str
I think I get this error because 'trades' is replicated twice in my .json string and thus might be viewed as a string. Is there a workaround around this issue? I'm new to python so would greatly appreciate your help!
Desired Output
A .csv file with the following structure:

Answer by #dsillman2000 for entry in data: trades_data = entry['trades'] ... etc

How to access different keys within a json file?

I'm calling an API service which returns JSON (with Czech language values) that looks like:
{
"model": "czech-morfflex-pdt-161115",
"acknowledgements": [
"http://ufal.mff.cuni.cz/morphodita#morphodita_acknowledgements",
"http://ufal.mff.cuni.cz/morphodita/users-manual#czech-morfflex-pdt_acknowledgements"
],
"result": [
[
{
"token": "Děti",
"analyses": [
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=1|Neg=A"
},
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=4|Neg=A"
},
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=5|Neg=A"
}
],
"space": " "
},
...
I want to return "lemma" value where "tag" values Cas=3
I tried:
import json
import os
import httpx
service_url = "http://lindat.mff.cuni.cz/services/morphodita/api"
output_format = "json"
model = "czech-morfflex"
text = "Děti pojedou k babičce Martě. Už se těší."
anal_service_url = "/".join([service_url, "analyze"])
params = {"output": output_format, "model": model, "data": text}
response = httpx.request("GET", anal_service_url, params=params)
response.raise_for_status()
response_dict = response.json()
result = response_dict.get("result")
print(type(result))
for res in result:
for a in res:
for b in a['analyses']:
for case in b['tag'][4]:
for i in [i for i,x in enumerate(case) if x == '3']:
print(i) # print position
But I don't know how to access "lemma" if case=3.
Help would be appreciated.

You can use an if statement to find the tag in the string:
case_tag = 'Cas=3'
for res_list in result:
for res_list_elem in res_list:
for item in res_list_elem['analyses']:
if case_tag in item['tag']:
print(item['lemma'])

python why in my json array each json element have the same value

So I have a json template and I am reading from a csv to update the some of the value of the json properties. I then put all the json in a array to write to a file. but in my file, all the json elements have the same value.
The issue is the old values are being overwritten some how. How should I fix that?
def main():
df = pd.read_csv("Daily_EXRATE.csv")
df = df.loc[df['Field1'] == '04']
opdb = {
"sell_rate": 1.2676,
"type": "currency_exchange",
"version": "1"
}
opdbarray = []
for index, rowsr in df.iterrows():
data = {}
data = rowsr.to_json()
data = json.loads(data)
opdb["sell_rate"] = data["Field11"]
opdbarray.append(opdb)
print(json.dumps(opdb, indent = 4 ))
# now write output to a file
jsonDataFile = open("ccData_1.json", "w")
jsonDataFile.write(json.dumps(opdbarray, indent=4, sort_keys=True))
jsonDataFile.close()
outputs are all the same
[
{
"sell_rate": "2.1058000000",
"type": "currency_exchange",
"version": "1"
},
{
"sell_rate": "2.1058000000",
"type": "currency_exchange",
"version": "1"
},
{
"sell_rate": "2.1058000000",
"type": "currency_exchange",
"version": "1"
},

You're appending the same obdb dictionary to apdbarray each time through the loop, just replacing its sell_rate element. You need to create a new dictionary each time.
def main():
df = pd.read_csv("Daily_EXRATE.csv")
df = df.loc[df['Field1'] == '04']
opdbarray = []
for index, rowsr in df.iterrows():
data = {}
data = rowsr.to_json()
data = json.loads(data)
opdb = {
"sell_rate": 1.2676,
"type": "currency_exchange",
"version": "1",
"sell_rate": data["Field11"]
}
opdbarray.append(opdb)
print(json.dumps(opdb, indent = 4 ))
# now write output to a file
jsonDataFile = open("ccData_1.json", "w")
jsonDataFile.write(json.dumps(opdbarray, indent=4, sort_keys=True))
jsonDataFile.close()

how to store JSON list data into python variables?

I have a JSON file that contains some data.
testJSON.json
{
"hourlyData": [
{
"frequency": "49.96",
"actual": " 2,240.43 ",
"schedule": " 2,223.85 ",
"acp": "325"
},
{
"frequency": "50.04",
"actual": " 1,862.88 ",
"schedule": " 1,881.09 ",
"acp": "275"
},
{
"frequency": "50.04",
"actual": " 1,882.17 ",
"schedule": " 1,885.94 ",
"acp": "275"
}
],
"storageData": [
{
"config": "ESS1",
"name": "BRPL",
"Power Rating": "20",
"Energy Rating": "20",
"socLow": "0",
"socHigh": "0.8",
"Charge Eff": "0.9273",
"Discharge Eff": "0.9273",
"Round Trip Eff": "0.922",
"Lower Limit": "57",
"Mid Limit": "76",
"High Limit": "95",
"Thrushold": "5",
"Discharging Price": "6"
}
],
I want to store these values into python variables. So what I have done is that first I created a dictionary which contain different variables of different types then I created a function which simple opens the JSON file then I try to store those json values into declared variables:
test.py
import json
#decalaring variables
storageData = {
"name": 'No name specified',
"powerRating":-1,
"energyRating":-1,
"annualMaxCycles":365,
"socLow":-1,
"socHigh":-1,
"chargeEff":-1,
"dChargeEff":-1,
"lowerLimit": -1,
"midLimit": -1,
"highLimit": -1,
"thrushold": 5,
"dischargingPrice": 6
}
marketData = {
"marketProducts": {
"dsm":{
"frequency":[],
"schedule":[],
"actual":[],
"acp": [],
}
}
inputMode = 'JSON'
JSONfileName = "testJSON.json"
def inputJSON():
if (inputMode == 'JSON'):
fileName = JSONfileName
# Import data from JSON files
with open(JSONfileName, 'r') as myfile:
dataJSON = ((myfile.read().replace('\n', '')))
inputJSON = json.loads(dataJSON)
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
marketData['marketProducts']['dsm']['frequency'] = inputJSON['hourlyData']['frequency']
marketData['marketProducts']['dsm']['acp'] = inputJSON['hourlyData']['acp']
marketData['marketProducts']['dsm']['actual'] = inputJSON['hourlyData']['actual']
marketData['marketProducts']['dsm']['schedule'] = inputJSON['hourlyData']['schedule']
inputJSON()
error that it gives me
Traceback (most recent call last):
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 113, in <module>
inputJSON()
File "C:/Users/nvats/PycharmProjects/dsm-final/test2.py", line 80, in inputJSON
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
TypeError: list indices must be integers or slices, not str

Instead of
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData']['Power Rating']
storageData['energyRating'] = inputJSON['storageData']['energyRating']
storageData['warranty'] = inputJSON['storageData']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData']['socLow']
storageData['socHigh'] = inputJSON['storageData']['socHigh']
storageData['chargeEff'] = inputJSON['storageData']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData']['midLimit']
storageData['highLimit'] = inputJSON['storageData']['highLimit']
storageData['thrushold'] = inputJSON['storageData']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData']['dischargingPrice']
Do
# Assigning the storageData data
storageData['powerRating'] = inputJSON['storageData[0]']['Power Rating']
storageData['energyRating'] = inputJSON['storageData[0]']['energyRating']
storageData['warranty'] = inputJSON['storageData[0]']['powerRating']
storageData['annualMaxCycles'] = inputJSON['storageData[0]']['maxAnnualCycles']
storageData['socLow'] = inputJSON['storageData[0]']['socLow']
storageData['socHigh'] = inputJSON['storageData[0]']['socHigh']
storageData['chargeEff'] = inputJSON['storageData[0]']['chargeEff']
storageData['dChargeEff'] = inputJSON['storageData[0]']['dChargeEff']
storageData['lowerLimit'] = inputJSON['storageData[0]']['lowerLimit']
storageData['midLimit'] = inputJSON['storageData[0]']['midLimit']
storageData['highLimit'] = inputJSON['storageData[0]']['highLimit']
storageData['thrushold'] = inputJSON['storageData[0]']['thrushold']
storageData['dischargingPrice'] = inputJSON['storageData[0]']['dischargingPrice']
Because it's an array, whose 1st index has the json data.
Ex. for "hourlyData"
for(data of inputJSON['hourlyData']){
marketData['marketProducts']['dsm']['frequency'] = data['frequency']
}
Hope that helps.