We Keep Coding

Unable to append data to Json array object with desired output

My code
import json
import re
from http.client import responses
import vt
import requests
with open('/home/asad/Downloads/ssh-log-parser/ok.txt', 'r') as file:
file = file.read()
pattern = re.compile(r'\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3}')
ips = pattern.findall(file)
unique_ips = list(set(ips))
# print(unique_ips)
# print(len(unique_ips))
headers = {
"accept": "application/json",
"x-apikey": "9765ba5d9fd52f5747dde5240606019f83d32758cb664abc63a43488aa42812d"
}
i = 0
url = "https://www.virustotal.com/api/v3/ip_addresses/"
# messages = \[\]
f = open('formater.json')
# returns JSON object as
# a dictionary
data = json.load(f)
f.close()
no = 0
while i \< len(unique_ips):
furl = url + str(unique_ips[i])
# response = requests.get(furl, headers=headers)
# data_ = response.json()
# print(data_)
# messages = [data_['data']['attributes']['last_analysis_results']]
messages = [data['data']['attributes']['last_analysis_results']]
y = json.dumps(messages)
y1 = json.loads(y)
# print(y1)
a = []
r = []
v = []
cnt = 0
store = len(y1[0])
out_json_new = []
out_json1 ={}
for o in y1:
for k, vv in o.items():
a_ = vv['result']
a.append(a_)
r_ = vv['engine_name']
r.append(r_)
v_ = vv['category']
v.append(v_)
out_json = {
"indicators": [{
"value": str(unique_ips[i]),
"type": 'ip',
}]
}
out_json1 ={
"providers":[{
"provider": str(r),
"verdict": str(a),
"score": str(v)
}] }
out_json1['providers'].append(out_json1)
i += 1
print(out_json,out_json1)
\#for aaa in a:
\#print(a\[0\])\`
Outputs as
{'indicators': [{'value': '192.91.72.201', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
I want to change the output to this format.
{
"providers":\[
{
"provider":"['Bkav']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC Threat Intelligence']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC sarah']",
"verdict":"['hate']",
"score":"['harmless']"
}
]
}
My current code, groups under one key e.g provider instead it should be appended one after another like in output above. I tried to use append logic but its not working as i attended. It output as
`out_json1['providers'].append(out_json1)`
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
RELEVANT FILES
In order to run the code these files are required.
ok.txt
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.201
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.101
formater.json
{
"data": {
"attributes": {
"country": "US",
"last_analysis_stats": {
"harmless": 86,
"malicious": 0,
"suspicious": 0,
"undetected": 0,
"timeout": 0
},
"last_analysis_results": {
"Bkav": {
"category": "harmless",
"result": "clean",
"method": "blacklist",
"engine_name": "Bkav"
},
"CMC Threat Intelligence": {
"category": "harmless",
"result": "legs",
"method": "blacklist",
"engine_name": "CMC Threat Intelligence"
},
"CMC sarah ": {
"category": "sarah",
"result": "hate",
"method": "you",
"engine_name": "CMC sarah "
}
}
}
}
}

how to create a dictionary with list of dictionaries as values from a list of dictionaries

I have this sample list of dictionaries:
[
{
"name": "like father",
"director": "Ajun kun",
"edited": "2014-12-20T21:23:49.867000Z",
"similar_movies": [
"http://movies.dev/api/films/1/",
"http://movies.dev/api/films/3/",
],
"rating": "2.0",
},
{
"name": "be like her",
"director": tuned ku",
"edited": "2014-12-20T21:23:49.870000Z",
"similar_movies": [
"http://movies.dev/api/films/1/"
]
}, .......
]
Some of the dictionaries in the list contain ratings while others do not. I want to generate a new dictionary of like the dictionary below sorted by the ratings:
{
"movies":[
{"name": "movie_4", "rating" : 0.1},
{"name": "movie_1", "rating" : 0.3},
{"name": "movies_5", "rating" : 0.5}
],
"movies_without_rating": [
{"name": "movie_8"},
{"name": "movie_3"}
]
}
Here is my sample code:
from flask import Flask, jsonify, request
import requests
from collections import ChainMap
app = Flask(__name__)
#app.route('/movies', methods=['GET'])
def returnAll():
#note: the URL is just a demo url
response = requests.get("https://movies.dev/api/movies/")
results = response.json()['results']
general_dic = {}
for result in result:
for key, val in result:
if (key == 'rating'):
general_dic['movies']
else:
general_dic['movies_with_rating']
return general_dic
return jsonify(results)
if __name__ == "__main__":
app.run(debug=True)
I got stuck and I couldn't continue, I will greatly appreciate your help.

You can use this example to integrate in your code:
lst = [
{
"movies": "like father",
"rating": "2.0",
},
{
"movies": "other movie",
"rating": "2.5",
},
{
"movies": "be like her",
},
{
"movies": "other movie 2",
"rating": "5.5",
},
{
"movies": "other movie 3",
},
]
out = {'movies':[], 'movies_without_rating':[]}
for movie in lst:
if 'rating' in movie:
out['movies'].append({'name': movie['movies'], 'rating': float(movie['rating'])})
else:
out['movies_without_rating'].append({'name': movie['movies']})
# sort it
out['movies'] = sorted(out['movies'], key=lambda k: k['rating'])
# pretty print on screen:
from pprint import pprint
pprint(out)
Prints:
{'movies': [{'name': 'like father', 'rating': 2.0},
{'name': 'other movie', 'rating': 2.5},
{'name': 'other movie 2', 'rating': 5.5}],
'movies_without_rating': [{'name': 'be like her'}, {'name': 'other movie 3'}]}

It seems something like this is what you'd like:
def separate_movies(movies_list):
movies_with_rating = []
movies_without_rating = []
for movie in movies_list:
name = movie["movies"]
if "rating" in movie:
movies_with_rating.append({
"name": name,
"rating": movie["rating"]
})
else:
movies_without_rating.append({
"name": name
})
movies_with_rating.sort(key = lambda movie: movie["rating"])
return {
"movies": movies_with_rating,
"movies_without_rating": movies_without_rating
}
The key here is using the in keyword to check whether a movie has a rating.

Creating a visual tree from JSON file

There is a lot of info how to create a tree inside JSON file, but is there any way to visualise dependencies from JSON file to any visual interface, picture or terminal as tree ( like on pictures ) ?
Stole this pictures from another post
There is my code to generate JSON from database ( Im generating parentID with Radom module ) :
emp = []
for item in root.iter('empl'):
name_tag = {'name': '', 'dep': '', 'Title': '', 'email': '', 'id': int(item.attrib['id'])}
for data in item.iter('field'):
if data.attrib['id'] == 'displayName':
name_tag['name'] = data.text
elif data.attrib['id'] == 'dep':
name_tag['dep'] = data.text
elif data.attrib['id'] == 'Title':
name_tag['Title'] = data.text
elif data.attrib['id'] == 'Email':
name_tag['email'] = data.text
elif data.attrib['id'] == 'id':
name_tag['id'] = data.text
else:
continue
emp.append(name_tag)
session_factory = sessionmaker(engine)
session = session_factory()
emp_list = [TableName(name=item['name'], department=item['dep'], Title=item['Title'], email=item['email'], id=item['id']) for item in employees]
avoid_duplicates = list(connection.execute('select * from TableName'))
for i in emp_list:
if i.name not in [j[1] for j in avoid_duplicates]:
session.add(i)
session.commit()
write_list = [{'id': i[0], 'name': i[1], 'dep': i[2], 'Title': i[3], 'email': i[4]} for i in list(connection.execute('select * from TableName'))]
session.close()
connection.close()
for randomID in write_list:
randomID['parentId'] = random.randrange(0, 487, 2)
with open('data.json', 'w') as file:
json.dump(write_list, file)
Afther this JSONs looks like :
{
"id": 9,
"name": "Name1",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 408
},
{
"id": 10,
"name": "Name2",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 316
},
{
"id": 12,
"name": "Name3",
"dep": "dep",
"Title": "title",
"email": "email",
"parentId": 160
}
Is there any way how I can visualize 'id - parentID' relations to a visual tree ?

JSON move child keys up one level

I have a Python script that parses a JSON file like below:
[
{
"_index": "bulletins",
"_type": "bulletin",
"_id": "OPENWRT-SA-000001",
"_score": null,
"_source": {
"lastseen": "2016-09-26T15:45:23",
"references": [
"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-3193",
],
"affectedPackage": [
{
"OS": "OpenWrt",
"OSVersion": "15.05",
"packageVersion": "9.9.8-P3-1",
"packageFilename": "UNKNOWN",
"arch": "all",
"packageName": "bind",
"operator": "lt"
}
],
"edition": 1,
"description": "value in here,
"reporter": "OpenWrt Project",
"published": "2016-01-24T13:33:41",
"title": "bind: Security update (4 CVEs)",
"type": "openwrt",
"bulletinFamily": "unix",
"cvelist": [
"CVE-2015-8704",
],
"modified": "2016-01-24T13:33:41",
"id": "OPENWRT-SA-000001",
"href": "https://lists.openwrt.org/pipermail/openwrt-security-announce/2016-January/000001.html",
"cvss": {
"score": 7.1,
"vector": "AV:NETWORK/AC:MEDIUM/Au:NONE/C:NONE/I:NONE/A:COMPLETE/"
}
},
"sort": [
34872
]
},
I have removed some of the values to keep the post shorter but leaving some in to try to keep the structure.
I want to take all sub keys from the _source key and move them up to the same level as _source and then delete the _source key.
My code to parse the JSON is:
import json
import logging
import logging.handlers
import os
import pymongo
from pymongo import MongoClient
def import_json(mongo_server,mongo_port, vuln_folder):
try:
logging.info('Connecting to MongoDB')
client = MongoClient(mongo_server, mongo_port)
db = client['vuln_sets']
coll = db['vulnerabilities']
logging.info('Connected to MongoDB')
basepath = os.path.dirname(__file__)
filepath = os.path.abspath(os.path.join(basepath, ".."))
archive_filepath = filepath + vuln_folder
filedir = os.chdir(archive_filepath)
file_count = 0
for item in os.listdir(filedir):
if item.endswith('.json'):
file_name = os.path.abspath(item)
with open(item, 'r') as currentfile:
vuln_counter = 0
duplicate_count = 0
logging.info('Currently processing ' + item)
file_count +=1
json_data = currentfile.read()
vuln_content = json.loads(json_data)
for vuln in vuln_content:
try:
del vuln['_type']
coll.insert(vuln, continue_on_error=True)
vuln_counter +=1
except pymongo.errors.DuplicateKeyError:
duplicate_count +=1
logging.info('Added ' + str(vuln_counter) + ' vulnerabilities for ' + item)
logging.info('Found ' + str(duplicate_count) + ' duplicate records!')
os.remove(file_name)
logging.info('Processed ' + str(file_count) + ' files')
except Exception as e:
logging.exception(e)
Which you can see already deletes one key that is not needed but that key has no needed data where as I need the sub keys from _source. I am not sure on the best way to achieve this, whether it would be programmatically correct to just re-create the JSON file with the new info but I need to keep the order of the keys and structure apart from moving the sub keys up one level.

You can use the dictionary update() function to achieve what you're trying to do, but it's important to note that dictionaries don't have an "order of the keys" - see: Key Order in Python Dictionaries.
Here's an example of one way to do this, starting with a dictionary definition.
d = {
"_index": "bulletins",
"_type": "bulletin",
"_id": "OPENWRT-SA-000001",
"_score": None,
"_source": {
"lastseen": "2016-09-26T15:45:23",
"references": [
"http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-3193",
],
"affectedPackage": [
{
"OS": "OpenWrt",
"OSVersion": "15.05",
"packageVersion": "9.9.8-P3-1",
"packageFilename": "UNKNOWN",
"arch": "all",
"packageName": "bind",
"operator": "lt"
}
],
"edition": 1,
"description": "value in here",
"reporter": "OpenWrt Project",
"published": "2016-01-24T13:33:41",
"title": "bind: Security update (4 CVEs)",
"type": "openwrt",
"bulletinFamily": "unix",
"cvelist": [
"CVE-2015-8704",
],
"modified": "2016-01-24T13:33:41",
"id": "OPENWRT-SA-000001",
"href": "https://lists.openwrt.org/pipermail/openwrt-security-announce/2016-January/000001.html",
"cvss": {
"score": 7.1,
"vector": "AV:NETWORK/AC:MEDIUM/Au:NONE/C:NONE/I:NONE/A:COMPLETE/"
}
}
}
# create a new dictionary with everything except the key "_source"
new_d = {key: d[key] for key in d if key != '_source'}
# add the keys/values from "_source" to new dictionary
new_d.update(d['_source']) # This will overwriting any existing keys
The output of new_d:
{'_id': 'OPENWRT-SA-000001',
'_index': 'bulletins',
'_score': None,
'_type': 'bulletin',
'affectedPackage': [{'OS': 'OpenWrt',
'OSVersion': '15.05',
'arch': 'all',
'operator': 'lt',
'packageFilename': 'UNKNOWN',
'packageName': 'bind',
'packageVersion': '9.9.8-P3-1'}],
'bulletinFamily': 'unix',
'cvelist': ['CVE-2015-8704'],
'cvss': {
'score': 7.1,
'vector': 'AV:NETWORK/AC:MEDIUM/Au:NONE/C:NONE/I:NONE/A:COMPLETE/'},
'description': 'value in here',
'edition': 1,
'href': 'https://lists.openwrt.org/pipermail/openwrt-security-announce/2016-January/000001.html',
'id': 'OPENWRT-SA-000001',
'lastseen': '2016-09-26T15:45:23',
'modified': '2016-01-24T13:33:41',
'published': '2016-01-24T13:33:41',
'references': ['http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2015-
3193'],
'reporter': 'OpenWrt Project',
'title': 'bind: Security update (4 CVEs)',
'type': 'openwrt'}

I managed to get it working by using the following code:
for vuln in vuln_content:
try:
del vuln['_type']
new_vuln = {key: vuln[key] for key in vuln if key != '_source'}
new_vuln.update(vuln['_source'])
coll.insert(new_vuln, continue_on_error=True)
vuln_counter +=1
except pymongo.errors.DuplicateKeyError:
duplicate_count +=1

Creating a jSON Array using Django

I am trying to create a jSON object with sample output such as
{
"pickups": [
{
"id": " ",
"name": " ",
"number": " ",
"time": " ",
"status": " "
},
{
"id": " ",
"name": " ",
"time": " ",
"number": " ",
"status": " "
}
]
}
I am getting a sample response like
{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}
What I have been tried
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
#pickup_records = json.dumps(pickup_records)
pickup_records = json.dumps(pickup_records, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
EDIT 1
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
print record
pickup_records.append(record)
pickup_response={"records":pickup_records}
print "before pickup+records",pickup_response
#pickup_records = json.dumps( pickup_response, sort_keys=True, indent=4)
print "after pickup+records"
#pickup_response={"pickup":pickup_records}
print "after pickup+response"
return HttpResponse(pickup_response, content_type="application/json")
LOG RESPONSE
before pickup+records {'records': [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr Ayurveda Delhi', 'id': 83L}, {'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Callmate India', 'id': 84L}]}
I suppose I am making mistakes on return HttpResponse(pickup_response, content_type="application/json") Please correct me

Here is the final working code
pickup_dict = {}
pickup_records=[]
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"number":pickup_number,"status":"1","time":"time"}
print record
pickup_records.append(record)
pickup_dict["pickup"]=pickup_records
return JsonResponse(pickup_dict)

I think you need to make sure you're declaring pickup_records as a list, and then check the way you're calling json.dumps.
pickup_records = []
for tmpPickUp in pickup:
pickup_date=tmpPickUp.pickup_date
pickup_time=tmpPickUp.pickup_time
pickup_id = tmpPickUp.id
pickup_name=tmpPickUp.customer_name
pickup_number=tmpPickUp.pieces
print pickup_date,pickup_time,pickup_id,pickup_name,pickup_number
record = {"name":pickup_name, "id":pickup_id,"time":pickup_date,"number":pickup_number,"status":"1"}
pickup_records.append(record)
pickup_records = json.dumps({'pickups': pickup_records}, indent=4)
pickup_response={"pickup":pickup_records}
return HttpResponse(pickup_response, content_type="application/json")
UPDATE
I've run the following in a console - (I think the error must be with your TmpPickUp items) -
>>> import json
>>> records = []
>>> for i in ["","",""]:
... record = {"name":i, "id":i,"time":i,"number":i,"status":i}
... records.append(record)
...
>>> print json.dumps({'pickups': records}, indent=4)
{
"pickups": [
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
},
{
"status": "",
"time": "",
"number": "",
"name": "",
"id": ""
}
]
}

from django.utils import simplejson
pickup_records = []
for tmpPickUp in pickup:
pickup_records.append({ "id": tmpPickUp.id })
pickup_records.append({ "name": tmpPickUp.customer_name })
pickup_records.append({ "number": tmpPickUp.pieces })
pickup_records.append({ "time": tmpPickUp.pickup_time })
pickup_records.append({ "status": "1" })
return HttpResponse(simplejson.dumps(pickup_records, indent=4),
mimetype="application/json")
Or maybe this would help you: https://github.com/praekelt/django-generate

At the first you should write your own serializer for datetime.date objects:
import datetime
class CustomEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.date):
return obj.strftime('%m-%d-%Y')
return json.JSONEncoder.default(self, obj)
after that you can use it:
json.dumps(d, cls=CustomEncoder)
'{"status": "1", "id": 83, "number": 4, "name": "Dr John", "time": "02-27-2013"}'
so you finale code is:
resp = [{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Dr John', 'id': 83L}{'status': '1', 'time': datetime.date(2013, 2, 27), 'number': 4L, 'name': u'Ricky', 'id': 84L}]
finale_struct = {'products':[]}
for res in resp:
finale_struct['products'].append(json.dumps(res, cls=CustomEncoder))