I am trying to create a jSON object with sample output such as
{
"pickups": [
{
"id": " ",
"name": " ",
"number": " ",
"time": " ",
"status": " "
},
{
"id": " ",
"name": " ",
"time": " ",
"number": " ",
"status": " "
}
]
}
I am getting a sample response like
{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}
What I have been tried
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
#pickup_records = json.dumps(pickup_records)
pickup_records = json.dumps(pickup_records, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
EDIT 1
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
pickup_response={"records":pickup_records}
print "before pickup+records",pickup_response
#pickup_records = json.dumps( pickup_response, sort_keys=True, indent=4)
print "after pickup+records"
#pickup_response={"pickup":pickup_records}
print "after pickup+response"
return HttpResponse(pickup_response, content_type="application/json")
LOG RESPONSE
before pickup+records {'records': [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr Ayurveda Delhi', 'id': 83L}, {'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Callmate India', 'id': 84L}]}
I suppose I am making mistakes on return HttpResponse(pickup_response, content_type="application/json") Please correct me
Here is the final working code
pickup_dict = {}
pickup_records=[]
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"number":pickup_number,"status":"1","time":"time"}
print record
pickup_records.append(record)
pickup_dict["pickup"]=pickup_records
return JsonResponse(pickup_dict)
I think you need to make sure you're declaring pickup_records as a list, and then check the way you're calling json.dumps.
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
pickup_records.append(record)
pickup_records = json.dumps({'pickups': pickup_records}, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
UPDATE
I've run the following in a console - (I think the error must be with your TmpPickUp items) -
>>> import json
>>> records = []
>>> for i in ["","",""]:
... record = {"name":i, "id":i,"time":i,"number":i,"status":i}
... records.append(record)
...
>>> print json.dumps({'pickups': records}, indent=4)
{
"pickups": [
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
}
]
}
from django.utils import simplejson
pickup_records = []
for tmpPickUp in pickup:
pickup_records.append({ "id": tmpPickUp.id })
pickup_records.append({ "name": tmpPickUp.customer_name })
pickup_records.append({ "number": tmpPickUp.pieces })
pickup_records.append({ "time": tmpPickUp.pickup_time })
pickup_records.append({ "status": "1" })
return HttpResponse(simplejson.dumps(pickup_records, indent=4),
mimetype="application/json")
Or maybe this would help you: https://github.com/praekelt/django-generate
At the first you should write your own serializer for datetime.date objects:
import datetime
class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.date):
return obj.strftime('%m-%d-%Y')
return json.JSONEncoder.default(self, obj)
after that you can use it:
json.dumps(d, cls=CustomEncoder)
'{"status": "1", "id": 83, "number": 4, "name": "Dr John", "time": "02-27-2013"}'
so you finale code is:
resp = [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}]
finale_struct = {'products':[]}
for res in resp:
finale_struct['products'].append(json.dumps(res, cls=CustomEncoder))
Related
My code
import json
import re
from http.client import responses
import vt
import requests
with open('/home/asad/Downloads/ssh-log-parser/ok.txt', 'r') as file:
file = file.read()
pattern = re.compile(r'\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3}')
ips = pattern.findall(file)
unique_ips = list(set(ips))
# print(unique_ips)
# print(len(unique_ips))
headers = {
"accept": "application/json",
"x-apikey": "9765ba5d9fd52f5747dde5240606019f83d32758cb664abc63a43488aa42812d"
}
i = 0
url = "https://www.virustotal.com/api/v3/ip_addresses/"
# messages = \[\]
f = open('formater.json')
# returns JSON object as
# a dictionary
data = json.load(f)
f.close()
no = 0
while i \< len(unique_ips):
furl = url + str(unique_ips[i])
# response = requests.get(furl, headers=headers)
# data_ = response.json()
# print(data_)
# messages = [data_['data']['attributes']['last_analysis_results']]
messages = [data['data']['attributes']['last_analysis_results']]
y = json.dumps(messages)
y1 = json.loads(y)
# print(y1)
a = []
r = []
v = []
cnt = 0
store = len(y1[0])
out_json_new = []
out_json1 ={}
for o in y1:
for k, vv in o.items():
a_ = vv['result']
a.append(a_)
r_ = vv['engine_name']
r.append(r_)
v_ = vv['category']
v.append(v_)
out_json = {
"indicators": [{
"value": str(unique_ips[i]),
"type": 'ip',
}]
}
out_json1 ={
"providers":[{
"provider": str(r),
"verdict": str(a),
"score": str(v)
}] }
out_json1['providers'].append(out_json1)
i += 1
print(out_json,out_json1)
\#for aaa in a:
\#print(a\[0\])\`
Outputs as
{'indicators': [{'value': '192.91.72.201', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
I want to change the output to this format.
{
"providers":\[
{
"provider":"['Bkav']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC Threat Intelligence']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC sarah']",
"verdict":"['hate']",
"score":"['harmless']"
}
]
}
My current code, groups under one key e.g provider instead it should be appended one after another like in output above. I tried to use append logic but its not working as i attended. It output as
`out_json1['providers'].append(out_json1)`
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
RELEVANT FILES
In order to run the code these files are required.
ok.txt
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.201
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.101
formater.json
{
"data": {
"attributes": {
"country": "US",
"last_analysis_stats": {
"harmless": 86,
"malicious": 0,
"suspicious": 0,
"undetected": 0,
"timeout": 0
},
"last_analysis_results": {
"Bkav": {
"category": "harmless",
"result": "clean",
"method": "blacklist",
"engine_name": "Bkav"
},
"CMC Threat Intelligence": {
"category": "harmless",
"result": "legs",
"method": "blacklist",
"engine_name": "CMC Threat Intelligence"
},
"CMC sarah ": {
"category": "sarah",
"result": "hate",
"method": "you",
"engine_name": "CMC sarah "
}
}
}
}
}
i am trying to add an email object to the testing accounts json. finding myself a little stuck on this.
ideally i would like to add an email field to the testing_accounts objects. 1 Per.
jsondata
testing_accounts = [{'tenantName': 'hsmd1012159', 'studentName': 'stu1012159', 'studentPassword': 'asdf123!'}, {'tenantName': 'hsmd1012716', 'studentName': 'stu1012716', 'studentPassword': 'asdf123!'}, {'tenantName': 'hsmd1011234', 'studentName': 'stu1011234', 'studentPassword': 'asdf123!'}]
item i would like to add to each json object above.
test = "asdf#gmail.com\nasdf123#gmail.com\ntest#gmail.com"
The other catch is if there are not enough emails to match the tenants then add a "null#null.com"
Desired output
desiredOutput = [{
'tenantName': 'hsmd1012159', 'studentName': 'stu1012159', 'studentPassword': 'asdf123!', 'email': 'asdf#gmail.com'
},
{
'tenantName': 'hsmd1012716', 'studentName': 'stu1012716', 'studentPassword': 'asdf123!', 'email': 'asdf123#gmail.com'
},
{
'tenantName': 'hsmd1011234', 'studentName': 'stu1011234', 'studentPassword': 'asdf123!' , 'email': 'test#gmail.com'
},
{
'tenantName': 'hsmd1011s2234', 'studentName': 'stu122011234', 'studentPassword': 'asdf123!' , 'email': 'null#null'
}
]
Where i am at so far....
print(len(testing_accounts))
print(testing_accounts)
def newline(string):
listRes = list(string.split("\n"))
return listRes
def comma(string):
listRes = list(string.split(","))
return listRes
def comma_and_space(string):
listRes = list(string.split(", "))
return listRes
if '\n' in test:
test1 = newline(test)
test1 = json.dumps(test1)
print(test1)
elif ',' in test:
test1 = comma(test)
test1 = json.dumps(test1)
print(test1)
elif ', ' in test:
test1 = comma_and_space(test)
test1 = json.dumps(test1)
print(test1)
else:
print("didn't hit any of those")
print(len(test1))
print(test1)
for item in testing_accounts:
for email in test1:
item.update({'email': email})
#temp = json.loads(testing_accounts)
#temp.update({'email': test})
print(item)
any guidance would be greatly appreciated.
Try itertools.zip_longest:
from itertools import zip_longest
test = "asdf#gmail.com\nasdf123#gmail.com\ntest#gmail.com"
for email, account in zip_longest(
test.split(), testing_accounts, fillvalue="null#null.com"
):
account["email"] = email
print(testing_accounts)
Prints:
[
{
"tenantName": "hsmd1012159",
"studentName": "stu1012159",
"studentPassword": "asdf123!",
"email": "asdf#gmail.com",
},
{
"tenantName": "hsmd1012716",
"studentName": "stu1012716",
"studentPassword": "asdf123!",
"email": "asdf123#gmail.com",
},
{
"tenantName": "hsmd1011234",
"studentName": "stu1011234",
"studentPassword": "asdf123!",
"email": "test#gmail.com",
},
{
"tenantName": "hsmd1011s2234",
"studentName": "stu122011234",
"studentPassword": "asdf123!",
"email": "null#null.com",
},
]
I want to extract certain object when going through a response i'm getting from an API CALL.
Response =
[
{'id': '2a15947c-8cdb-4f1d-a1cc-a8d76fd97d61', 'name': 'Human', 'i18nNameKey': 'Blank space Blueprint', 'pluginClone': True
},
{'id': '99accff8-9e24-4c76-b21a-f12ef6572369', 'name': 'Robot', 'i18nNameKey': 'Personal space Blueprint', 'pluginClone': True
},
{'id': 'bf40b0e5-f151-4df6-b305-4a91b4b7c1da', 'name': 'Dog', 'i18nNameKey': 'Game.blueprints.space.kb.name', 'pluginClone': True
},
{'id': '42868b38-b9f8-4540-ba26-0988e8a2e1f7', 'name': 'Bug', 'i18nNameKey': 'Game.blueprints.space.team.name', 'pluginClone': True
},
{'id': 'b23eb9fd-0106-452a-8cab-551ce3b45eb0', 'name': 'Cat', 'i18nNameKey': 'Game.blueprints.space.documentation.name', 'pluginClone': True
},
{'id': '67668d17-6c08-4c85-a6b6-3c1d6fb23000', 'name': 'Cat', 'i18nNameKey': 'Game.blueprints.space.sp.name', 'pluginClone': True,
}
]
I need to regroup, the id and the name, at the moment i'm able to retrieve them but the response is not what i want t achieve
I'm getting a list of string , instead of a list of object, what i'm doing wrong ?
def listallCategories(self):
self.intentsResponseDict=[]
url = self.url
response = requests.get(url,auth=(self.user,self.password))
json_data = response.json()
#print(json_data)
for resp in json_data:
self.intentsResponseDict.append(resp['id'])
self.intentsResponseDict.append(resp['name'])
return self.intentsResponseDict
this is what i'm getting
['2a15947c-8cdb-4f1d-a1cc-a8d76fd97d61', 'Human', '99accff8-9e24-4c76-b21a-f12ef6572369', 'Robot', 'bf40b0e5-f151-4df6-b305-4a91b4b7c1da', 'Dog', '42868b38-b9f8-4540-ba26-0988e8a2e1f7', 'Bug', 'b23eb9fd-0106-452a-8cab-551ce3b45eb0', 'Cat', '67668d17-6c08-4c85-a6b6-3c1d6fb23000', 'Bird']
This is what i want id , name
[
{'2a15947c-8cdb-4f1d-a1cc-a8d76fd97d61', 'Human'
},
{'99accff8-9e24-4c76-b21a-f12ef6572369', 'Robot'
},
{'bf40b0e5-f151-4df6-b305-4a91b4b7c1da', 'Dog'
},
{'42868b38-b9f8-4540-ba26-0988e8a2e1f7', 'Bug'
},
{'b23eb9fd-0106-452a-8cab-551ce3b45eb0', 'Cat'
},
{'67668d17-6c08-4c85-a6b6-3c1d6fb23000', 'Bird'
}
]
The resp is probably like:
[
{
"id": "2a15947c-8cdb-4f1d-a1cc-a8d76fd97d61",
"name": "Human",
... more fields ...
},
{
"id": "99accff8-9e24-4c76-b21a-f12ef6572369",
"name": "Robot",
... more fields ...
},
... more objects ...
]
The format that you want the data isn't a valid JSON, a better schema for that JSON may be:
[
{
"id": "2a15947c-8cdb-4f1d-a1cc-a8d76fd97d61",
"name": "Human",
},
{
"id": "99accff8-9e24-4c76-b21a-f12ef6572369",
"name": "Robot",
},
... more objects ...
]
So, you are just picking id and name in the response
for resp in json_data:
self.intentsResponseDict.append({
"id": resp["id"],
"name": resp["name"]
})
What you show is a list of sets. Just build it:
...
for resp in json_data:
self.intentsResponseDict.append({resp['id'], resp['name']})
...
You're adding both the ID and name to the end of a list
for resp in json_data:
self.intentsResponseDict.append(resp['id']) # here is your issue
self.intentsResponseDict.append(resp['name']) # and here
return self.intentsResponseDict
If you want these values to stay together you would need to add them to the list together. e.g.
for resp in json_data:
self.intentsResponseDict.append([resp['id'], resp['name']]) # to create list of lists
# OR
self.intentsResponseDict.append({resp['id'], resp['name']}) # to create list of dicts
return self.intentsResponseDict
I have a CSV file named data.csv. I am converting CSV file in to nested JSON and inserting it in to mongodb using python.Below is the code, but here i wanna label FirstName and LastName under the parent called 'Name'. Can somebody help?
import json
import pandas as pd
from pymongo import MongoClient
try:
conn = MongoClient()
print("Connected successfully!!!")
except:
print("Could not connect to MongoDB")
# database
db = conn.database
collection = db.collection3
df = pd.read_csv(r'C:\Users\swetha1\Desktop\data.csv')
def get_nested_rec(key, grp):
rec = {}
rec['PrimaryId'] = key[0]
rec['FirstName'] = key[1]
rec['LastName'] = key[2]
rec['City'] = key[3]
for field in ['CarName','DogName']:
rec[field] = list(grp[field].unique())
return rec
records = []
for key, grp in df.groupby(['PrimaryId','FirstName','LastName','City']):
rec = get_nested_rec(key, grp)
records.append(rec)
records = dict(data = records)
r=json.dumps(records,default=int,indent=4)
my_dict = json.loads(r)
print(my_dict)
collection.insert(my_dict)
print('inserted')
the above code converts CSV to nested JSON
output converting CSV to nested JSON is as follows:
Connected successfully!!!
{
"data": [
{
"PrimaryId": 100,
"FirstName": "John",
"LastName": "Smith",
"City": "NewYork",
"CarName": [
"Toyota",
"BMW"
],
"DogName": [
"Spike",
"Rusty"
]
},
{
"PrimaryId": 101,
"FirstName": "Ben",
"LastName": "Swan",
"City": "Sydney",
"CarName": [
"Volkswagen",
"Ford",
"Audi"
],
"DogName": [
"Buddy",
"Max"
]
},
{
"PrimaryId": 102,
"FirstName": "Julia",
"LastName": "Brown",
"City": "London",
"CarName": [
"Mini"
],
"DogName": [
"Lucy"
]
}
]
}
desired output structure should be something like this:
PS: i have given blank values here just for sample.
"info":[
{ "primaryId":" "
"City":" "
"Name":
{ "FirstName":" "
"LastName" :" "
}
"CarName":
{ "car1": " "
"car2": " "
}
"DogName":
{ "Dog1": " "
"Dog2": " "
}
}]
data.CSV
PrimaryId,FirstName,LastName,City,CarName,DogName
100,John,Smith,NewYork,Toyota,Spike
100,John,Smith,NewYork,BMW,Spike
100,John,Smith,NewYork,Toyota,Rusty
100,John,Smith,NewYork,BMW,Rusty
101,Ben,Swan,Sydney,Volkswagen,Buddy
101,Ben,Swan,Sydney,Ford,Buddy
101,Ben,Swan,Sydney,Audi,Buddy
101,Ben,Swan,Sydney,Volkswagen,Max
101,Ben,Swan,Sydney,Ford,Max
101,Ben,Swan,Sydney,Audi,Max
102,Julia,Brown,London,Mini,Lucy
This is one approach.
Demo:
import pandas as pd
df = pd.read_csv(filename)
def get_nested_rec(key, grp):
rec = {}
rec['PrimaryId'] = key[0]
rec['City'] = key[3]
rec["Name"] = [{'FirstName': key[1], 'LastName': key[2]}] #Update.
for field in ['CarName','DogName']:
rec[field] = list(grp[field].unique())
return rec
records = []
for key, grp in df.groupby(['PrimaryId','FirstName','LastName','City']):
rec = get_nested_rec(key, grp)
records.append(rec)
records = dict(data = records)
print(records)
Output:
{'data': [{'CarName': ['Toyota', 'BMW'],
'City': 'NewYork',
'DogName': ['Spike', 'Rusty'],
'Name': [{'FirstName': 'John', 'LastName': 'Smith'}],
'PrimaryId': 100},
{'CarName': ['Volkswagen', 'Ford', 'Audi'],
'City': 'Sydney',
'DogName': ['Buddy', 'Max'],
'Name': [{'FirstName': 'Ben', 'LastName': 'Swan'}],
'PrimaryId': 101},
{'CarName': ['Mini'],
'City': 'London',
'DogName': ['Lucy'],
'Name': [{'FirstName': 'Julia', 'LastName': 'Brown'}],
'PrimaryId': 102}]}
There is a lot of info how to create a tree inside JSON file, but is there any way to visualise dependencies from JSON file to any visual interface, picture or terminal as tree ( like on pictures ) ?
Stole this pictures from another post
There is my code to generate JSON from database ( Im generating parentID with Radom module ) :
emp = []
for item in root.iter('empl'):
name_tag = {'name': '', 'dep': '', 'Title': '', 'email': '', 'id': int(item.attrib['id'])}
for data in item.iter('field'):
if data.attrib['id'] == 'displayName':
name_tag['name'] = data.text
elif data.attrib['id'] == 'dep':
name_tag['dep'] = data.text
elif data.attrib['id'] == 'Title':
name_tag['Title'] = data.text
elif data.attrib['id'] == 'Email':
name_tag['email'] = data.text
elif data.attrib['id'] == 'id':
name_tag['id'] = data.text
else:
continue
emp.append(name_tag)
session_factory = sessionmaker(engine)
session = session_factory()
emp_list = [TableName(name=item['name'], department=item['dep'], Title=item['Title'], email=item['email'], id=item['id']) for item in employees]
avoid_duplicates = list(connection.execute('select * from TableName'))
for i in emp_list:
if i.name not in [j[1] for j in avoid_duplicates]:
session.add(i)
session.commit()
write_list = [{'id': i[0], 'name': i[1], 'dep': i[2], 'Title': i[3], 'email': i[4]} for i in list(connection.execute('select * from TableName'))]
session.close()
connection.close()
for randomID in write_list:
randomID['parentId'] = random.randrange(0, 487, 2)
with open('data.json', 'w') as file:
json.dump(write_list, file)
Afther this JSONs looks like :
{
"id": 9,
"name": "Name1",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 408
},
{
"id": 10,
"name": "Name2",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 316
},
{
"id": 12,
"name": "Name3",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 160
}
Is there any way how I can visualize 'id - parentID' relations to a visual tree ?