Creating a visual tree from JSON file - python

There is a lot of info how to create a tree inside JSON file, but is there any way to visualise dependencies from JSON file to any visual interface, picture or terminal as tree ( like on pictures ) ?
Stole this pictures from another post
There is my code to generate JSON from database ( Im generating parentID with Radom module ) :
emp = []
for item in root.iter('empl'):
name_tag = {'name': '', 'dep': '', 'Title': '', 'email': '', 'id': int(item.attrib['id'])}
for data in item.iter('field'):
if data.attrib['id'] == 'displayName':
name_tag['name'] = data.text
elif data.attrib['id'] == 'dep':
name_tag['dep'] = data.text
elif data.attrib['id'] == 'Title':
name_tag['Title'] = data.text
elif data.attrib['id'] == 'Email':
name_tag['email'] = data.text
elif data.attrib['id'] == 'id':
name_tag['id'] = data.text
else:
continue
emp.append(name_tag)
session_factory = sessionmaker(engine)
session = session_factory()
emp_list = [TableName(name=item['name'], department=item['dep'], Title=item['Title'], email=item['email'], id=item['id']) for item in employees]
avoid_duplicates = list(connection.execute('select * from TableName'))
for i in emp_list:
if i.name not in [j[1] for j in avoid_duplicates]:
session.add(i)
session.commit()
write_list = [{'id': i[0], 'name': i[1], 'dep': i[2], 'Title': i[3], 'email': i[4]} for i in list(connection.execute('select * from TableName'))]
session.close()
connection.close()
for randomID in write_list:
randomID['parentId'] = random.randrange(0, 487, 2)
with open('data.json', 'w') as file:
json.dump(write_list, file)
Afther this JSONs looks like :
{
"id": 9,
"name": "Name1",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 408
},
{
"id": 10,
"name": "Name2",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 316
},
{
"id": 12,
"name": "Name3",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 160
}
Is there any way how I can visualize 'id - parentID' relations to a visual tree ?

Related

Unable to append data to Json array object with desired output

My code
import json
import re
from http.client import responses
import vt
import requests
with open('/home/asad/Downloads/ssh-log-parser/ok.txt', 'r') as file:
file = file.read()
pattern = re.compile(r'\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3}')
ips = pattern.findall(file)
unique_ips = list(set(ips))
# print(unique_ips)
# print(len(unique_ips))
headers = {
"accept": "application/json",
"x-apikey": "9765ba5d9fd52f5747dde5240606019f83d32758cb664abc63a43488aa42812d"
}
i = 0
url = "https://www.virustotal.com/api/v3/ip_addresses/"
# messages = \[\]
f = open('formater.json')
# returns JSON object as
# a dictionary
data = json.load(f)
f.close()
no = 0
while i \< len(unique_ips):
furl = url + str(unique_ips[i])
# response = requests.get(furl, headers=headers)
# data_ = response.json()
# print(data_)
# messages = [data_['data']['attributes']['last_analysis_results']]
messages = [data['data']['attributes']['last_analysis_results']]
y = json.dumps(messages)
y1 = json.loads(y)
# print(y1)
a = []
r = []
v = []
cnt = 0
store = len(y1[0])
out_json_new = []
out_json1 ={}
for o in y1:
for k, vv in o.items():
a_ = vv['result']
a.append(a_)
r_ = vv['engine_name']
r.append(r_)
v_ = vv['category']
v.append(v_)
out_json = {
"indicators": [{
"value": str(unique_ips[i]),
"type": 'ip',
}]
}
out_json1 ={
"providers":[{
"provider": str(r),
"verdict": str(a),
"score": str(v)
}] }
out_json1['providers'].append(out_json1)
i += 1
print(out_json,out_json1)
\#for aaa in a:
\#print(a\[0\])\`
Outputs as
{'indicators': [{'value': '192.91.72.201', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
I want to change the output to this format.
{
"providers":\[
{
"provider":"['Bkav']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC Threat Intelligence']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC sarah']",
"verdict":"['hate']",
"score":"['harmless']"
}
]
}
My current code, groups under one key e.g provider instead it should be appended one after another like in output above. I tried to use append logic but its not working as i attended. It output as
`out_json1['providers'].append(out_json1)`
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
RELEVANT FILES
In order to run the code these files are required.
ok.txt
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.201
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.101
formater.json
{
"data": {
"attributes": {
"country": "US",
"last_analysis_stats": {
"harmless": 86,
"malicious": 0,
"suspicious": 0,
"undetected": 0,
"timeout": 0
},
"last_analysis_results": {
"Bkav": {
"category": "harmless",
"result": "clean",
"method": "blacklist",
"engine_name": "Bkav"
},
"CMC Threat Intelligence": {
"category": "harmless",
"result": "legs",
"method": "blacklist",
"engine_name": "CMC Threat Intelligence"
},
"CMC sarah ": {
"category": "sarah",
"result": "hate",
"method": "you",
"engine_name": "CMC sarah "
}
}
}
}
}

add element in json object working with 2 different lists

i am trying to add an email object to the testing accounts json. finding myself a little stuck on this.
ideally i would like to add an email field to the testing_accounts objects. 1 Per.
jsondata
testing_accounts = [{'tenantName': 'hsmd1012159', 'studentName': 'stu1012159', 'studentPassword': 'asdf123!'}, {'tenantName': 'hsmd1012716', 'studentName': 'stu1012716', 'studentPassword': 'asdf123!'}, {'tenantName': 'hsmd1011234', 'studentName': 'stu1011234', 'studentPassword': 'asdf123!'}]
item i would like to add to each json object above.
test = "asdf#gmail.com\nasdf123#gmail.com\ntest#gmail.com"
The other catch is if there are not enough emails to match the tenants then add a "null#null.com"
Desired output
desiredOutput = [{
'tenantName': 'hsmd1012159', 'studentName': 'stu1012159', 'studentPassword': 'asdf123!', 'email': 'asdf#gmail.com'
},
{
'tenantName': 'hsmd1012716', 'studentName': 'stu1012716', 'studentPassword': 'asdf123!', 'email': 'asdf123#gmail.com'
},
{
'tenantName': 'hsmd1011234', 'studentName': 'stu1011234', 'studentPassword': 'asdf123!' , 'email': 'test#gmail.com'
},
{
'tenantName': 'hsmd1011s2234', 'studentName': 'stu122011234', 'studentPassword': 'asdf123!' , 'email': 'null#null'
}
]
Where i am at so far....
print(len(testing_accounts))
print(testing_accounts)
def newline(string):
listRes = list(string.split("\n"))
return listRes
def comma(string):
listRes = list(string.split(","))
return listRes
def comma_and_space(string):
listRes = list(string.split(", "))
return listRes
if '\n' in test:
test1 = newline(test)
test1 = json.dumps(test1)
print(test1)
elif ',' in test:
test1 = comma(test)
test1 = json.dumps(test1)
print(test1)
elif ', ' in test:
test1 = comma_and_space(test)
test1 = json.dumps(test1)
print(test1)
else:
print("didn't hit any of those")
print(len(test1))
print(test1)
for item in testing_accounts:
for email in test1:
item.update({'email': email})
#temp = json.loads(testing_accounts)
#temp.update({'email': test})
print(item)
any guidance would be greatly appreciated.

Try itertools.zip_longest:
from itertools import zip_longest
test = "asdf#gmail.com\nasdf123#gmail.com\ntest#gmail.com"
for email, account in zip_longest(
test.split(), testing_accounts, fillvalue="null#null.com"
):
account["email"] = email
print(testing_accounts)
Prints:
[
{
"tenantName": "hsmd1012159",
"studentName": "stu1012159",
"studentPassword": "asdf123!",
"email": "asdf#gmail.com",
},
{
"tenantName": "hsmd1012716",
"studentName": "stu1012716",
"studentPassword": "asdf123!",
"email": "asdf123#gmail.com",
},
{
"tenantName": "hsmd1011234",
"studentName": "stu1011234",
"studentPassword": "asdf123!",
"email": "test#gmail.com",
},
{
"tenantName": "hsmd1011s2234",
"studentName": "stu122011234",
"studentPassword": "asdf123!",
"email": "null#null.com",
},
]

Twint to Kafka -> Dont receive any events in my topic

i'm trying to load some tweets via TWINT into a kafka topic with the confluent-kafka[avro]
Producer. I don't get any errors but my topic wont receive any events from twint. I even get succes msg, when debugging(with try and except).
My Code:
import twint
import sys
import json
from time import sleep
from confluent_kafka import avro
from confluent_kafka.avro import AvroProducer
# Define Avro schema
value_schema_str = """
{
"namespace": "my.test",
"name": "value",
"type": "record",
"fields" : [
{ "name": "id", "type": "long" },
{ "name": "tweet", "type": "string" },
{ "name": "datetime", "type": "string" },
{ "name": "username", "type": "string" },
{ "name": "user_id", "type": "long" },
{ "name": "name", "type": "string" }
]
}
"""
key_schema_str = """
{
"namespace": "my.test",
"name": "key",
"type": "record",
"fields" : [
{
"name" : "name",
"type" : "string"
}
]
}
"""
kafka_broker = 'host.docker.internal:9092'
schema_registry = 'http://host.docker.internal:8081'
value_schema = avro.loads(value_schema_str)
key_schema = avro.loads(key_schema_str)
producer = AvroProducer({
'bootstrap.servers': kafka_broker,
'schema.registry.url': schema_registry
}, default_key_schema=key_schema, default_value_schema=value_schema)
module = sys.modules["twint.storage.write"]
def Json(obj, config):
tweet = obj.__dict__
tweet_new = {}
tweet_new['id'] = tweet['id']
tweet_new['tweet'] = tweet['tweet']
tweet_new['datetime'] = tweet['datetime']
tweet_new['username'] = tweet['username']
tweet_new['user_id'] = tweet['user_id']
tweet_new['name'] = tweet['name']
print(tweet_new)
try:
producer.produce(topic='tweets_test', key={"name": "Key"}, value=tweet_new)
except Exception as e:
print(f"Exception while producing record value - {tweet_new} to topic - tweets_test: {e}")
else:
print(f"Successfully producing record value - {tweet_new} to topic - tweets_test")
try:
producer.flush()
except Exception as e:
print(f"Exception while flush record value - {tweet_new} to topic - tweets_test: {e}")
else:
print(f"Successfully flushed record value - {tweet_new} to topic - tweets_test")
module.Json = Json
c2 = twint.Config()
c2.Search = "corona OR regen OR \"stark regen\" OR \"sturm\" OR überschwemmung OR landunter OR #hagel OR #regen OR #sturm OR flut"
c2.Store_json = True
c2.Custom["user"] = ["id", "tweet", "user_id", "username", "hashtags"]
c2.User_full = True
c2.Output = "tweets.json"
c2.Since = '2019-05-20'
c2.Hide_output = True
twint.run.Search(c2)
When i run it i get the following output:
{'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'}
Successfully producing record value - {'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'} to topic - tweets_test
Successfully flushed record value - {'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'} to topic - tweets_test
Any help how i can debug it better or any advice would be awesome.

CSV to nested (hierarchical) JSON - labelling parents using Python

I have a CSV file named data.csv. I am converting CSV file in to nested JSON and inserting it in to mongodb using python.Below is the code, but here i wanna label FirstName and LastName under the parent called 'Name'. Can somebody help?
import json
import pandas as pd
from pymongo import MongoClient
try:
conn = MongoClient()
print("Connected successfully!!!")
except:
print("Could not connect to MongoDB")
# database
db = conn.database
collection = db.collection3
df = pd.read_csv(r'C:\Users\swetha1\Desktop\data.csv')
def get_nested_rec(key, grp):
rec = {}
rec['PrimaryId'] = key[0]
rec['FirstName'] = key[1]
rec['LastName'] = key[2]
rec['City'] = key[3]
for field in ['CarName','DogName']:
rec[field] = list(grp[field].unique())
return rec
records = []
for key, grp in df.groupby(['PrimaryId','FirstName','LastName','City']):
rec = get_nested_rec(key, grp)
records.append(rec)
records = dict(data = records)
r=json.dumps(records,default=int,indent=4)
my_dict = json.loads(r)
print(my_dict)
collection.insert(my_dict)
print('inserted')
the above code converts CSV to nested JSON
output converting CSV to nested JSON is as follows:
Connected successfully!!!
{
"data": [
{
"PrimaryId": 100,
"FirstName": "John",
"LastName": "Smith",
"City": "NewYork",
"CarName": [
"Toyota",
"BMW"
],
"DogName": [
"Spike",
"Rusty"
]
},
{
"PrimaryId": 101,
"FirstName": "Ben",
"LastName": "Swan",
"City": "Sydney",
"CarName": [
"Volkswagen",
"Ford",
"Audi"
],
"DogName": [
"Buddy",
"Max"
]
},
{
"PrimaryId": 102,
"FirstName": "Julia",
"LastName": "Brown",
"City": "London",
"CarName": [
"Mini"
],
"DogName": [
"Lucy"
]
}
]
}
desired output structure should be something like this:
PS: i have given blank values here just for sample.
"info":[
{ "primaryId":" "
"City":" "
"Name":
{ "FirstName":" "
"LastName" :" "
}
"CarName":
{ "car1": " "
"car2": " "
}
"DogName":
{ "Dog1": " "
"Dog2": " "
}
}]
data.CSV
PrimaryId,FirstName,LastName,City,CarName,DogName
100,John,Smith,NewYork,Toyota,Spike
100,John,Smith,NewYork,BMW,Spike
100,John,Smith,NewYork,Toyota,Rusty
100,John,Smith,NewYork,BMW,Rusty
101,Ben,Swan,Sydney,Volkswagen,Buddy
101,Ben,Swan,Sydney,Ford,Buddy
101,Ben,Swan,Sydney,Audi,Buddy
101,Ben,Swan,Sydney,Volkswagen,Max
101,Ben,Swan,Sydney,Ford,Max
101,Ben,Swan,Sydney,Audi,Max
102,Julia,Brown,London,Mini,Lucy

This is one approach.
Demo:
import pandas as pd
df = pd.read_csv(filename)
def get_nested_rec(key, grp):
rec = {}
rec['PrimaryId'] = key[0]
rec['City'] = key[3]
rec["Name"] = [{'FirstName': key[1], 'LastName': key[2]}] #Update.
for field in ['CarName','DogName']:
rec[field] = list(grp[field].unique())
return rec
records = []
for key, grp in df.groupby(['PrimaryId','FirstName','LastName','City']):
rec = get_nested_rec(key, grp)
records.append(rec)
records = dict(data = records)
print(records)
Output:
{'data': [{'CarName': ['Toyota', 'BMW'],
'City': 'NewYork',
'DogName': ['Spike', 'Rusty'],
'Name': [{'FirstName': 'John', 'LastName': 'Smith'}],
'PrimaryId': 100},
{'CarName': ['Volkswagen', 'Ford', 'Audi'],
'City': 'Sydney',
'DogName': ['Buddy', 'Max'],
'Name': [{'FirstName': 'Ben', 'LastName': 'Swan'}],
'PrimaryId': 101},
{'CarName': ['Mini'],
'City': 'London',
'DogName': ['Lucy'],
'Name': [{'FirstName': 'Julia', 'LastName': 'Brown'}],
'PrimaryId': 102}]}

Creating a jSON Array using Django

I am trying to create a jSON object with sample output such as
{
"pickups": [
{
"id": " ",
"name": " ",
"number": " ",
"time": " ",
"status": " "
},
{
"id": " ",
"name": " ",
"time": " ",
"number": " ",
"status": " "
}
]
}
I am getting a sample response like
{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}
What I have been tried
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
#pickup_records = json.dumps(pickup_records)
pickup_records = json.dumps(pickup_records, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
EDIT 1
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
pickup_response={"records":pickup_records}
print "before pickup+records",pickup_response
#pickup_records = json.dumps( pickup_response, sort_keys=True, indent=4)
print "after pickup+records"
#pickup_response={"pickup":pickup_records}
print "after pickup+response"
return HttpResponse(pickup_response, content_type="application/json")
LOG RESPONSE
before pickup+records {'records': [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr Ayurveda Delhi', 'id': 83L}, {'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Callmate India', 'id': 84L}]}
I suppose I am making mistakes on return HttpResponse(pickup_response, content_type="application/json") Please correct me

Here is the final working code
pickup_dict = {}
pickup_records=[]
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"number":pickup_number,"status":"1","time":"time"}
print record
pickup_records.append(record)
pickup_dict["pickup"]=pickup_records
return JsonResponse(pickup_dict)

I think you need to make sure you're declaring pickup_records as a list, and then check the way you're calling json.dumps.
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
pickup_records.append(record)
pickup_records = json.dumps({'pickups': pickup_records}, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
UPDATE
I've run the following in a console - (I think the error must be with your TmpPickUp items) -
>>> import json
>>> records = []
>>> for i in ["","",""]:
... record = {"name":i, "id":i,"time":i,"number":i,"status":i}
... records.append(record)
...
>>> print json.dumps({'pickups': records}, indent=4)
{
"pickups": [
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
}
]
}

from django.utils import simplejson
pickup_records = []
for tmpPickUp in pickup:
pickup_records.append({ "id": tmpPickUp.id })
pickup_records.append({ "name": tmpPickUp.customer_name })
pickup_records.append({ "number": tmpPickUp.pieces })
pickup_records.append({ "time": tmpPickUp.pickup_time })
pickup_records.append({ "status": "1" })
return HttpResponse(simplejson.dumps(pickup_records, indent=4),
mimetype="application/json")
Or maybe this would help you: https://github.com/praekelt/django-generate

At the first you should write your own serializer for datetime.date objects:
import datetime
class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.date):
return obj.strftime('%m-%d-%Y')
return json.JSONEncoder.default(self, obj)
after that you can use it:
json.dumps(d, cls=CustomEncoder)
'{"status": "1", "id": 83, "number": 4, "name": "Dr John", "time": "02-27-2013"}'
so you finale code is:
resp = [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}]
finale_struct = {'products':[]}
for res in resp:
finale_struct['products'].append(json.dumps(res, cls=CustomEncoder))

Categories