My code
import json
import re
from http.client import responses
import vt
import requests
with open('/home/asad/Downloads/ssh-log-parser/ok.txt', 'r') as file:
file = file.read()
pattern = re.compile(r'\\d{1,3}.\\d{1,3}.\\d{1,3}.\\d{1,3}')
ips = pattern.findall(file)
unique_ips = list(set(ips))
# print(unique_ips)
# print(len(unique_ips))
headers = {
"accept": "application/json",
"x-apikey": "9765ba5d9fd52f5747dde5240606019f83d32758cb664abc63a43488aa42812d"
}
i = 0
url = "https://www.virustotal.com/api/v3/ip_addresses/"
# messages = \[\]
f = open('formater.json')
# returns JSON object as
# a dictionary
data = json.load(f)
f.close()
no = 0
while i \< len(unique_ips):
furl = url + str(unique_ips[i])
# response = requests.get(furl, headers=headers)
# data_ = response.json()
# print(data_)
# messages = [data_['data']['attributes']['last_analysis_results']]
messages = [data['data']['attributes']['last_analysis_results']]
y = json.dumps(messages)
y1 = json.loads(y)
# print(y1)
a = []
r = []
v = []
cnt = 0
store = len(y1[0])
out_json_new = []
out_json1 ={}
for o in y1:
for k, vv in o.items():
a_ = vv['result']
a.append(a_)
r_ = vv['engine_name']
r.append(r_)
v_ = vv['category']
v.append(v_)
out_json = {
"indicators": [{
"value": str(unique_ips[i]),
"type": 'ip',
}]
}
out_json1 ={
"providers":[{
"provider": str(r),
"verdict": str(a),
"score": str(v)
}] }
out_json1['providers'].append(out_json1)
i += 1
print(out_json,out_json1)
\#for aaa in a:
\#print(a\[0\])\`
Outputs as
{'indicators': [{'value': '192.91.72.201', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
I want to change the output to this format.
{
"providers":\[
{
"provider":"['Bkav']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC Threat Intelligence']",
"verdict":"['clean']",
"score":"['harmless']"
},
{
"provider":"['CMC sarah']",
"verdict":"['hate']",
"score":"['harmless']"
}
]
}
My current code, groups under one key e.g provider instead it should be appended one after another like in output above. I tried to use append logic but its not working as i attended. It output as
`out_json1['providers'].append(out_json1)`
{'indicators': [{'value': '192.91.72.101', 'type': 'ip'}]} {'providers': [{'provider': "['Bkav', 'CMC Threat Intelligence', 'CMC sarah ']", 'verdict': "['clean', 'legs', 'hate']", 'score': "['harmless', 'harmless', 'sarah']"}, {...}]}
RELEVANT FILES
In order to run the code these files are required.
ok.txt
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.201
Aug 22 09:45:08 ip-170-32-23-64 sshd\[1546\]: Invalid user HPSupport from 192.91.72.101
formater.json
{
"data": {
"attributes": {
"country": "US",
"last_analysis_stats": {
"harmless": 86,
"malicious": 0,
"suspicious": 0,
"undetected": 0,
"timeout": 0
},
"last_analysis_results": {
"Bkav": {
"category": "harmless",
"result": "clean",
"method": "blacklist",
"engine_name": "Bkav"
},
"CMC Threat Intelligence": {
"category": "harmless",
"result": "legs",
"method": "blacklist",
"engine_name": "CMC Threat Intelligence"
},
"CMC sarah ": {
"category": "sarah",
"result": "hate",
"method": "you",
"engine_name": "CMC sarah "
}
}
}
}
}
Related
i'm trying to load some tweets via TWINT into a kafka topic with the confluent-kafka[avro]
Producer. I don't get any errors but my topic wont receive any events from twint. I even get succes msg, when debugging(with try and except).
My Code:
import twint
import sys
import json
from time import sleep
from confluent_kafka import avro
from confluent_kafka.avro import AvroProducer
# Define Avro schema
value_schema_str = """
{
"namespace": "my.test",
"name": "value",
"type": "record",
"fields" : [
{ "name": "id", "type": "long" },
{ "name": "tweet", "type": "string" },
{ "name": "datetime", "type": "string" },
{ "name": "username", "type": "string" },
{ "name": "user_id", "type": "long" },
{ "name": "name", "type": "string" }
]
}
"""
key_schema_str = """
{
"namespace": "my.test",
"name": "key",
"type": "record",
"fields" : [
{
"name" : "name",
"type" : "string"
}
]
}
"""
kafka_broker = 'host.docker.internal:9092'
schema_registry = 'http://host.docker.internal:8081'
value_schema = avro.loads(value_schema_str)
key_schema = avro.loads(key_schema_str)
producer = AvroProducer({
'bootstrap.servers': kafka_broker,
'schema.registry.url': schema_registry
}, default_key_schema=key_schema, default_value_schema=value_schema)
module = sys.modules["twint.storage.write"]
def Json(obj, config):
tweet = obj.__dict__
tweet_new = {}
tweet_new['id'] = tweet['id']
tweet_new['tweet'] = tweet['tweet']
tweet_new['datetime'] = tweet['datetime']
tweet_new['username'] = tweet['username']
tweet_new['user_id'] = tweet['user_id']
tweet_new['name'] = tweet['name']
print(tweet_new)
try:
producer.produce(topic='tweets_test', key={"name": "Key"}, value=tweet_new)
except Exception as e:
print(f"Exception while producing record value - {tweet_new} to topic - tweets_test: {e}")
else:
print(f"Successfully producing record value - {tweet_new} to topic - tweets_test")
try:
producer.flush()
except Exception as e:
print(f"Exception while flush record value - {tweet_new} to topic - tweets_test: {e}")
else:
print(f"Successfully flushed record value - {tweet_new} to topic - tweets_test")
module.Json = Json
c2 = twint.Config()
c2.Search = "corona OR regen OR \"stark regen\" OR \"sturm\" OR überschwemmung OR landunter OR #hagel OR #regen OR #sturm OR flut"
c2.Store_json = True
c2.Custom["user"] = ["id", "tweet", "user_id", "username", "hashtags"]
c2.User_full = True
c2.Output = "tweets.json"
c2.Since = '2019-05-20'
c2.Hide_output = True
twint.run.Search(c2)
When i run it i get the following output:
{'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'}
Successfully producing record value - {'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'} to topic - tweets_test
Successfully flushed record value - {'id': 1513818741057937408, 'tweet': 'RKI: Bundesweite Sieben-Tage-Inzidenz steigt leicht auf 1087 | #corona #rki #test ', 'datetime': '2022-04-12 09:58:07 UTC', 'username': 'flashupde', 'user_id': 1179376986516606978, 'name': 'FLASH UP'} to topic - tweets_test
Any help how i can debug it better or any advice would be awesome.
I'm calling an API service which returns JSON (with Czech language values) that looks like:
{
"model": "czech-morfflex-pdt-161115",
"acknowledgements": [
"http://ufal.mff.cuni.cz/morphodita#morphodita_acknowledgements",
"http://ufal.mff.cuni.cz/morphodita/users-manual#czech-morfflex-pdt_acknowledgements"
],
"result": [
[
{
"token": "Děti",
"analyses": [
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=1|Neg=A"
},
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=4|Neg=A"
},
{
"lemma": "dítě",
"tag": "POS=N|SubPOS=N|Gen=F|Num=P|Cas=5|Neg=A"
}
],
"space": " "
},
...
I want to return "lemma" value where "tag" values Cas=3
I tried:
import json
import os
import httpx
service_url = "http://lindat.mff.cuni.cz/services/morphodita/api"
output_format = "json"
model = "czech-morfflex"
text = "Děti pojedou k babičce Martě. Už se těší."
anal_service_url = "/".join([service_url, "analyze"])
params = {"output": output_format, "model": model, "data": text}
response = httpx.request("GET", anal_service_url, params=params)
response.raise_for_status()
response_dict = response.json()
result = response_dict.get("result")
print(type(result))
for res in result:
for a in res:
for b in a['analyses']:
for case in b['tag'][4]:
for i in [i for i,x in enumerate(case) if x == '3']:
print(i) # print position
But I don't know how to access "lemma" if case=3.
Help would be appreciated.
You can use an if statement to find the tag in the string:
case_tag = 'Cas=3'
for res_list in result:
for res_list_elem in res_list:
for item in res_list_elem['analyses']:
if case_tag in item['tag']:
print(item['lemma'])
I have this sample list of dictionaries:
[
{
"name": "like father",
"director": "Ajun kun",
"edited": "2014-12-20T21:23:49.867000Z",
"similar_movies": [
"http://movies.dev/api/films/1/",
"http://movies.dev/api/films/3/",
],
"rating": "2.0",
},
{
"name": "be like her",
"director": tuned ku",
"edited": "2014-12-20T21:23:49.870000Z",
"similar_movies": [
"http://movies.dev/api/films/1/"
]
}, .......
]
Some of the dictionaries in the list contain ratings while others do not. I want to generate a new dictionary of like the dictionary below sorted by the ratings:
{
"movies":[
{"name": "movie_4", "rating" : 0.1},
{"name": "movie_1", "rating" : 0.3},
{"name": "movies_5", "rating" : 0.5}
],
"movies_without_rating": [
{"name": "movie_8"},
{"name": "movie_3"}
]
}
Here is my sample code:
from flask import Flask, jsonify, request
import requests
from collections import ChainMap
app = Flask(__name__)
#app.route('/movies', methods=['GET'])
def returnAll():
#note: the URL is just a demo url
response = requests.get("https://movies.dev/api/movies/")
results = response.json()['results']
general_dic = {}
for result in result:
for key, val in result:
if (key == 'rating'):
general_dic['movies']
else:
general_dic['movies_with_rating']
return general_dic
return jsonify(results)
if __name__ == "__main__":
app.run(debug=True)
I got stuck and I couldn't continue, I will greatly appreciate your help.
You can use this example to integrate in your code:
lst = [
{
"movies": "like father",
"rating": "2.0",
},
{
"movies": "other movie",
"rating": "2.5",
},
{
"movies": "be like her",
},
{
"movies": "other movie 2",
"rating": "5.5",
},
{
"movies": "other movie 3",
},
]
out = {'movies':[], 'movies_without_rating':[]}
for movie in lst:
if 'rating' in movie:
out['movies'].append({'name': movie['movies'], 'rating': float(movie['rating'])})
else:
out['movies_without_rating'].append({'name': movie['movies']})
# sort it
out['movies'] = sorted(out['movies'], key=lambda k: k['rating'])
# pretty print on screen:
from pprint import pprint
pprint(out)
Prints:
{'movies': [{'name': 'like father', 'rating': 2.0},
{'name': 'other movie', 'rating': 2.5},
{'name': 'other movie 2', 'rating': 5.5}],
'movies_without_rating': [{'name': 'be like her'}, {'name': 'other movie 3'}]}
It seems something like this is what you'd like:
def separate_movies(movies_list):
movies_with_rating = []
movies_without_rating = []
for movie in movies_list:
name = movie["movies"]
if "rating" in movie:
movies_with_rating.append({
"name": name,
"rating": movie["rating"]
})
else:
movies_without_rating.append({
"name": name
})
movies_with_rating.sort(key = lambda movie: movie["rating"])
return {
"movies": movies_with_rating,
"movies_without_rating": movies_without_rating
}
The key here is using the in keyword to check whether a movie has a rating.
I would like to iterate through a list of dictionaries and save values of certain keys (in my case "consumer Key" and "consumer Secret") as many times they are present into another dictionary.
Problem: I'm able to iterate through the list but my code is not saving the second consumer key and consumer secret, instead it is saving the first consumer key and consumer secret twice.
Input:
{
"accessType": "",
"apiProducts": [],
"appFamily": "default",
"appId": "ac56c8b2-6ac1-4971-a1d3-4bf97893c067",
"attributes": [
{
"name": "DisplayName",
"value": "quotaapp"
},
{
"name": "Notes",
"value": ""
}
],
"callbackUrl": "",
"createdAt": 1549274952045,
"createdBy": "suraj.pai.airody#sap.com",
"credentials": [
{
"apiProducts": [
{
"apiproduct": "apiprod",
"status": "approved"
}
],
"attributes": [],
"consumerKey": "xyz",
"consumerSecret": "abc",
"expiresAt": -1,
"issuedAt": 1549274952051,
"scopes": [],
"status": "approved"
},
{
"apiProducts": [
{
"apiproduct": "ouathTest-Product",
"status": "approved"
}
],
"attributes": [],
"consumerKey": "pqr",
"consumerSecret": "wmn",
"expiresAt": -1,
"issuedAt": 1554802431452,
"scopes": [],
"status": "approved"
}
],
"developerId": "xyz",
"lastModifiedAt": 1554802431662,
"lastModifiedBy": "suraj.pai.airody#sap.com",
"name": "quotaapp",
"scopes": [],
"status": "approved"
}
Code:
import requests
import json
from requests.auth import HTTPBasicAuth
import csv
def get_v2details():
a = 'orgID1'
b = 'appID1'
c = 'ConKey1'
d = 'ConSecret1'
e = 'appName1'
org_lst = []
some_dict = {}
con_blst = [] # variable to append the dictionary app level
n = int(input("Enter number of orgs from Landscape 1: "))
for i in range(0, n):
ele = str(input())
org_lst.append(ele)
cmp_orglst = list(org_lst)
print(cmp_orglst)
for j in cmp_orglst:
url = "https://canarydevmgmtsrv.dmzmo.sap.corp/v1/o/" + str(j) + "/apps/"
headers = {'Content-Type': 'application/json'}
response = requests.get(url, auth=HTTPBasicAuth('xyz', 'xyz'), headers=headers, verify=False)
app_data = json.loads(response.text)
print(app_data)
for k in app_data:
url1 = "https://canarydevmgmtsrv.dmzmo.sap.corp/v1/o/" + str(j) + "/apps/" + str(k)
headers = {'Content-Type': 'application/json'}
response1 = requests.get(url1, auth=HTTPBasicAuth('xyz', 'xyz'), headers=headers, verify=False)
consumer_data = json.loads(response1.text)
print(" Consumer Data is ", consumer_data)
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][0]['consumerKey']
some_dict[d] = consumer_data['credentials'][0]['consumerSecret']
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
print(con_blst)
csv_columns = ['orgID1', 'appName1', 'appID1', 'ConKey1', 'ConSecret1']
csv_file = "Names1.csv"
try:
with open(csv_file, 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
writer.writeheader()
for data in con_blst:
writer.writerow(data)
except IOError:
print("I/O error")
Expected result:
orgID1 appName1 appID1 ConKey1 ConSecret1
VALIDATE quotaapp 4bf97893c067 xyz abc
VALIDATE quotaapp 4bf97893c067 pqr wmn
Actual result:
orgID1 appName1 appID1 ConKey1 ConSecret1
VALIDATE quotaapp 4bf97893c067 xyz abc
VALIDATE quotaapp 4bf97893c067 xyz abc
It seems you just made a small error.
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][0]['consumerKey'] #this line
some_dict[d] = consumer_data['credentials'][0]['consumerSecret'] #and this line
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
Should be
for l in range(len(consumer_data['credentials'])):
some_dict[a] = str(j)
some_dict[b] = consumer_data['appId']
some_dict[e] = consumer_data['name']
some_dict[c] = consumer_data['credentials'][l]['consumerKey'] # Here
some_dict[d] = consumer_data['credentials'][l]['consumerSecret'] # Here
print(some_dict) # Print dictionary of each app ID
con_blst.append(some_dict.copy())
You weren't looping through consumer_data['credentials'], you were just storing consumer_data['credentials'][0] twice
As the question explains the problem, I've been trying to generate nested JSON object. In this case I have for loops getting the data out of dictionary dic. Below is the code:
f = open("test_json.txt", 'w')
flag = False
temp = ""
start = "{\n\t\"filename\"" + " : \"" +initial_filename+"\",\n\t\"data\"" +" : " +" [\n"
end = "\n\t]" +"\n}"
f.write(start)
for i, (key,value) in enumerate(dic.iteritems()):
f.write("{\n\t\"keyword\":"+"\""+str(key)+"\""+",\n")
f.write("\"term_freq\":"+str(len(value))+",\n")
f.write("\"lists\":[\n\t")
for item in value:
f.write("{\n")
f.write("\t\t\"occurance\" :"+str(item)+"\n")
#Check last object
if value.index(item)+1 == len(value):
f.write("}\n"
f.write("]\n")
else:
f.write("},") # close occurrence object
# Check last item in dic
if i == len(dic)-1:
flag = True
if(flag):
f.write("}")
else:
f.write("},") #close lists object
flag = False
#check for flag
f.write("]") #close lists array
f.write("}")
Expected output is:
{
"filename": "abc.pdf",
"data": [{
"keyword": "irritation",
"term_freq": 5,
"lists": [{
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 2
}]
}, {
"keyword": "bomber",
"lists": [{
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 2
}],
"term_freq": 5
}]
}
But currently I'm getting an output like below:
{
"filename": "abc.pdf",
"data": [{
"keyword": "irritation",
"term_freq": 5,
"lists": [{
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 2
},] // Here lies the problem "," before array(last element)
}, {
"keyword": "bomber",
"lists": [{
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 1
}, {
"occurance": 2
},], // Here lies the problem "," before array(last element)
"term_freq": 5
}]
}
Please help, I've trying to solve it, but failed. Please don't mark it duplicate since I have already checked other answers and didn't help at all.
Edit 1:
Input is basically taken from a dictionary dic whose mapping type is <String, List>
for example: "irritation" => [1,3,5,7,8]
where irritation is the key, and mapped to a list of page numbers.
This is basically read in the outer for loop where key is the keyword and value is a list of pages of occurrence of that keyword.
Edit 2:
dic = collections.defaultdict(list) # declaring the variable dictionary
dic[key].append(value) # inserting the values - useless to tell here
for key in dic:
# Here dic[x] represents list - each value of x
print key,":",dic[x],"\n" #prints the data in dictionary
What #andrea-f looks good to me, here another solution:
Feel free to pick in both :)
import json
dic = {
"bomber": [1, 2, 3, 4, 5],
"irritation": [1, 3, 5, 7, 8]
}
filename = "abc.pdf"
json_dict = {}
data = []
for k, v in dic.iteritems():
tmp_dict = {}
tmp_dict["keyword"] = k
tmp_dict["term_freq"] = len(v)
tmp_dict["lists"] = [{"occurrance": i} for i in v]
data.append(tmp_dict)
json_dict["filename"] = filename
json_dict["data"] = data
with open("abc.json", "w") as outfile:
json.dump(json_dict, outfile, indent=4, sort_keys=True)
It's the same idea, I first create a big json_dict to be saved directly in json. I use the with statement to save the json avoiding the catch of exception
Also, you should have a look to the doc of json.dumps() if you need future improve in your json output.
EDIT
And just for fun, if you don't like tmp var, you can do all the data for loop in a one-liner :)
json_dict["data"] = [{"keyword": k, "term_freq": len(v), "lists": [{"occurrance": i} for i in v]} for k, v in dic.iteritems()]
It could gave for final solution something not totally readable like this:
import json
json_dict = {
"filename": "abc.pdf",
"data": [{
"keyword": k,
"term_freq": len(v),
"lists": [{"occurrance": i} for i in v]
} for k, v in dic.iteritems()]
}
with open("abc.json", "w") as outfile:
json.dump(json_dict, outfile, indent=4, sort_keys=True)
EDIT 2
It looks like you don't want to save your json as the desired output, but be abble to read it.
In fact, you can also use json.dumps() in order to print your json.
with open('abc.json', 'r') as handle:
new_json_dict = json.load(handle)
print json.dumps(json_dict, indent=4, sort_keys=True)
There is still one problem here though, "filename": is printed at the end of the list because the d of data comes before the f.
To force the order, you will have to use an OrderedDict in the generation of the dict. Be careful the syntax is ugly (imo) with python 2.X
Here is the new complete solution ;)
import json
from collections import OrderedDict
dic = {
'bomber': [1, 2, 3, 4, 5],
'irritation': [1, 3, 5, 7, 8]
}
json_dict = OrderedDict([
('filename', 'abc.pdf'),
('data', [ OrderedDict([
('keyword', k),
('term_freq', len(v)),
('lists', [{'occurrance': i} for i in v])
]) for k, v in dic.iteritems()])
])
with open('abc.json', 'w') as outfile:
json.dump(json_dict, outfile)
# Now to read the orderer json file
with open('abc.json', 'r') as handle:
new_json_dict = json.load(handle, object_pairs_hook=OrderedDict)
print json.dumps(json_dict, indent=4)
Will output:
{
"filename": "abc.pdf",
"data": [
{
"keyword": "bomber",
"term_freq": 5,
"lists": [
{
"occurrance": 1
},
{
"occurrance": 2
},
{
"occurrance": 3
},
{
"occurrance": 4
},
{
"occurrance": 5
}
]
},
{
"keyword": "irritation",
"term_freq": 5,
"lists": [
{
"occurrance": 1
},
{
"occurrance": 3
},
{
"occurrance": 5
},
{
"occurrance": 7
},
{
"occurrance": 8
}
]
}
]
}
But be carefull, most of the time, it is better to save a regular .json file in order to be cross languages.
Your current code is not working because the loop iterates through the before-last item adding the }, then when the loop runs again it sets the flag to false, but the last time it ran it added a , since it thought that there will be another element.
If this is your dict: a = {"bomber":[1,2,3,4,5]} then you can do:
import json
file_name = "a_file.json"
file_name_input = "abc.pdf"
new_output = {}
new_output["filename"] = file_name_input
new_data = []
i = 0
for key, val in a.iteritems():
new_data.append({"keyword":key, "lists":[], "term_freq":len(val)})
for p in val:
new_data[i]["lists"].append({"occurrance":p})
i += 1
new_output['data'] = new_data
Then save the data by:
f = open(file_name, 'w+')
f.write(json.dumps(new_output, indent=4, sort_keys=True, default=unicode))
f.close()