We Keep Coding

converting txt file into json using python?

I have a log file that has the format as follows:
Nov 28 06:26:45 server-01 dhcpd: DHCPDISCOVER from cc:d3:e2:7a:af:40 via 10.39.192.1
Nov 28 06:26:45 server-01 dhcpd: DHCPOFFER on 10.39.255.253 to cc:d3:e2:7a:af:40 via 10.39.192.1
The next step is to convert the text data into a JSON using Python. So far, I have the python script.
Now, the JSON file is created in the following format:
# Python program to convert text
# file to JSON
import json
# the file to be converted
filename = 'Logs.txt'
# resultant dictionary
dict1 = {}
# fields in the sample file
fields =['timestamp', 'Server', 'Service', 'Message']
with open(filename) as fh:
# count variable for employee id creation
l = 1
for line in fh:
# reading line by line from the text file
description = list( line.strip().split(None, 4))
# for output see below
print(description)
# for automatic creation of id for each employee
sno ='emp'+str(l)
# loop variable
i = 0
# intermediate dictionary
dict2 = {}
while i<len(fields):
# creating dictionary for each employee
dict2[fields[i]]= description[i]
i = i + 1
# appending the record of each employee to
# the main dictionary
dict1[sno]= dict2
l = l + 1
# creating json file
out_file = open("test5.json", "w")
json.dump(dict1, out_file, indent = 4)
out_file.close()
which gives the following output:
{
"emp1": { "timestamp": "Nov", "Server": "28", "Service": "06:26:45", "Message": "server-01" },
"emp2": { "timestamp": "Nov", "Server": "28", "Service": "06:26:45", "Message": "server-01" }
}
But I need an ouput like:
{
"timestamp":"Nov 28 06:26:26",
"Server":"server-01",
"Service":"dhcpd",
"Message":"DHCPOFFER on 10.45.45.31 to cc:d3:e2:7a:b9:6b via 10.45.0.1",
}
I don't know why it's not printing the whole data. Can anyone help me with this?

The problem with your code is that you did .split(None, 4), which allows only 4 splits on the input string. Since the date contains spaces too, the result of this will be (e.g. for the first line of your input):
['Nov', # timestamp
'28', # Server
'06:26:45', # Service
'server-01', # Message
'dhcpd: DHCPDISCOVER from cc:d3:e2:7a:af:40 via 10.39.192.1']
You even printed this, so I'm surprised you didn't notice something is wrong.
Now, the first element of the list is assigned to the key 'timestamp', the second element to the key 'Server', and so on. This is how you get a dict that looks like:
{ "timestamp": "Nov", "Server": "28", "Service": "06:26:45", "Message": "server-01" }
Instead, you want to split a maximum of five times. The first three elements of the resultant split are the timestamp.
# Don't need that extra list(), since .split() already returns a list
description = line.strip().split(None, 5)
# Join the first three elements,
joined_timestamp = " ".join(description[:3])
# and replace them in the list
# Setting a slice of a list: See https://stackoverflow.com/q/10623302/843953
description[:3] = [joined_timestamp]
Then, your description looks like this:
['Nov 28 06:26:45',
'server-01',
'dhcpd:',
'DHCPDISCOVER from cc:d3:e2:7a:af:40 via 10.39.192.1']
and the elements fields now correspond to the values in description.
Note that you could replace that entire while i < len(fields)... loop with simply dict2 = dict(zip(fields, description))
P.S.: You might want to clean up other elements of description, such as description[2] = description[2].rstrip(":") to remove the trailing colon in 'dhcpd:'

Can't access nested dictionary data **kwargs python and best practices for nested data mutation in GraphQL

In Graphene-Django and GraphQL I am trying to create a resolve_or_create method for nested data creation inside my mutations.
I'm trying to pass a dictionary with the input from the user as a **kwarg to my resolve_or_create function, and though I can see "location" in the variable watcher (in VSCode), I continuously am getting the error 'dict' object has no attribute 'location'
Here's my resolve_or_create method:
def resolve_or_create(*args, **kwargs):
input = {}
result = {}
input.location = kwargs.get('location', None)
if input.location is not None:
input.location = Location.objects.filter(pk=input.location.id).first()
if input.location is None:
location = Location.objects.create(
location_city = input.location.location_city,
location_state = input.location.location_state,
location_sales_tax_rate = input.location.location_sales_tax_rate
)
if location is None:
return None
result.location = location
return result
and my CreateCustomer definition, where this method is being called
class CreateCustomer(graphene.Mutation):
class Arguments:
input = CustomerInput(required=True)
ok = graphene.Boolean()
customer = graphene.Field(CustomerType)
#staticmethod
def mutate(root, info, input=None):
ok = True
resolved = resolve_or_create(**{'location':input.customer_city})
customer_instance = Customer(
customer_name = input.customer_name,
customer_address = input.customer_address,
customer_city = resolved.location,
customer_state = input.customer_state,
customer_zip = input.customer_zip,
customer_email = input.customer_email,
customer_cell_phone = input.customer_cell_phone,
customer_home_phone = input.customer_home_phone,
referred_from = input.referred_from
)
customer_instance.save()
return CreateCustomer(ok=ok, customer=customer_instance)
Here is an example mutation that would create a new customer with an existing location
mutation createCustomer {
createCustomer(input: {
customerName: "Ricky Bobby",
customerAddress: "1050 Airport Drive",
customerCity: {id:1},
customerState: "TX",
customerZip: "75222",
customerEmail: "mem",
customerCellPhone: "124567894",
referredFrom: "g"
}) {
ok,
customer{
id,
customerName,
customerAddress,
customerCity {
id
},
customerState,
customerZip,
customerEmail,
customerCellPhone,
referredFrom
}
}
}
and here is an example mutation that would create a customer with a new location
mutation createCustomer {
createCustomer(input: {
customerName: "Ricky Bobby",
customerAddress: "1050 Airport Drive",
customerCity: {locationCity: "Dallas", locationState: "TX", locationSalesTaxRate:7.77},
customerState: "TX",
customerZip: "75222",
customerEmail: "mem",
customerCellPhone: "124567894",
referredFrom: "g"
}) {
ok,
customer{
id,
customerName,
customerAddress,
customerCity {
id
},
customerState,
customerZip,
customerEmail,
customerCellPhone,
referredFrom
}
}
}
So my question is two-fold.
First, how can I retrieve my passed in location dict from kwargs?
Second, is there a better way to be resolving and creating nested data than this? Would this be expected behavior in a best-practice GraphQL API?
I have also tried resolve_or_create(location=input.customer_city})

I realized my syntax for dictionary assignment and access was wrong.
Corrected function:
def resolve_or_create(*args, **kwargs):
input = {}
result = {}
candidate = None
input['location'] = kwargs['location']
input['customer'] = kwargs.get('customer', None)
input['technician'] = kwargs.get('tech', None)
if input['location'] is not None:
if 'id' in input['location']:
candidate = Location.objects.filter(pk=input['location']['id']).first()
result['location'] = candidate
if candidate is None:
result['location'] = Location.objects.create(
location_city = input['location']['location_city'],
location_state = input['location']['location_state'],
location_sales_tax_rate = input['location']['location_sales_tax_rate']
)
if result['location'] is None:
return None
return result
I would still like to discuss the most effective way to accomplish created and mutating nested data in graphene-django. Is there a DRYer way to achieve what I'm looking to do or something I'm missing?

Passing multiple values from dictionary into method using loop

i have a question regarding adding dictionary key and value to method using loop
This is what i was thinking to write but it doesn't work how i want because it creates a packet just with one key/value every time
for key in packetData:
for name in packetData[key]:
packets = Ether()/IP()/UDP()/createsPacket(key, name=packetData[key][name])
print ("as name " + name + " \n as value " + str(packetData[key][name]))
Instead of writing this manually like that :
packets1 = Ether()/IP()/UDP()/createsPacket("65", UserID = "name", Password = "pass123", ETX = 123)
packets2 = Ether()/IP()/UDP()/createsPacket("72", PriceID = 123, Side = 12, MaxAmount = 123, MinAmount = 123, Price = 123000)
    json then converted to dictionary in python , this is data that i want to pass in
{
"65":{
"UserID":"vcjazfan",
"Password":"ejujwlhk",
"SessionID":115,
"ETX":192
},
"66":{
"UserID":"dzmtrssy",
"SessionID":35,
"Reason":"zbwivjcv",
"ETX":43
},
"72":{
"InstrumentIndex":171,
"PriceID":217,
"Side":226,
"MaxAmount":210,
"MinAmount":219,
"Price":47,
"PriceProvider":207,
"ETX":78
},
Made more generic for easier understanding, hoping it helps
Generic code
dictionary = {"65":{ "UserID":"vcjazfan", "Password":"ejujwlhk", "ETX":192} , "72":{ "InstrumentIndex":171, "PriceID":217, } }
#This is what i was thinking to write but it doesn't work how i want because it creates a packet just with one key/value every time
for key in dictionary:
for name in dictionary[key]:
value=dictionary[key][name]
packets = method(key, name=value) # in first iteration when key is 65 , name = "UserID" , value = "vcjazfan"
# in second iteration when key is 65 , name = "Password" , value = "ejujwlhk"
#Instead of writing this manually like that :
packets1 = method("65", UserID = "name", Password = "pass123", ETX = 123)
packets2 = method("72", InstrumentIndex = 123, PriceID = 12,)

This question solved my problem : How to pass dictionary items as function arguments in python?
solution to my original code:
Allpackets= []
for key in packetData:
Allpackets.append(packets/createsPacket(key, **packetData[key]))
Solution to generic one:
dictionary = {"65":{ "UserID":"vcjazfan", "Password":"ejujwlhk", "ETX":192} , "72":{ "InstrumentIndex":171, "PriceID":217, } }
Allpackets = []
for key in dictionary:
Allpackets.append( method(key, **dictionary))
#Instead of writing this manually like that :
packets1 = method("65", UserID = "name", Password = "pass123", ETX = 123)
packets2 = method("72", InstrumentIndex = 123, PriceID = 12,)

Access one item from a dict and store it into a variable

I am trying to get all the "uuid"'s from an API, and the issue is that it is stored into a dict (I think). Her is how it looks on the API:
{"guild": {
"_id": "5eba1c5f8ea8c960a61f38ed",
"name": "Creators Club",
"name_lower": "creators club",
"coins": 0,
"coinsEver": 0,
"created": 1589255263630,
"members":
[{ "uuid": "db03ceff87ad4909bababc0e2622aaf8",
"rank": "Guild Master",
"joined": 1589255263630,
"expHistory": {
"2020-06-01": 280,
"2020-05-31": 4701,
"2020-05-30": 0,
"2020-05-29": 518,
"2020-05-28": 1055,
"2020-05-27": 136665,
"2020-05-26": 34806}}]
}
}
Now I am interested in the "uuid" part there, and take note: There is multiple players, it can be 1 to 100 players, and I am going to need every UUID.
Now I have done this in my python to get the UUID's displayed on the website:
try:
f = requests.get(
"https://api.hypixel.net/guild?key=[secret]&id=" + guild).json()
guildName = f["guild"]["name"]
guildMembers = f["guild"]["members"]
members = client.getPlayer(uuid=guildMembers) #this converts UUID to player names
#I need to store all uuid's in variables and put them at "guildMembers"
And that gives me all the "UUID codes", and I will be using client.getPlayer(uuid=---) to convert the UUID into the Player Names. I have to loop through each "UUID" into that code client.getPlayer(uuid=---) . But first of I need to save the UUID'S in variables, I have been doing members.uuid to access the UUID on my HTML file, but I don't know how you do the .uuid part in python
If you need anything else, just comment :)

List comprehension is a powerful concept:
members = [client.getPlayer(member['uuid']) for member in guildMembers]
Edit:
If you want to insert the names back into your data (in guildMembers),
use a dictionary comprehension with {uuid: member_name,} format:
members = {member['uuid']: client.getPlayer(uuid=member['uuid']) for member in guildMembers}
Than you can update guildMembers with your results:
for member in guildMembers:
guildMembers[member]['name'] = members[member['uuid']]

Assuming that guild is the main dictionary in which a key called members exists with a list of "sub dictionaries", you can try
uuid = list()
for x in guild['members']:
uuid.append(x['uuid'])
uuid now has all the uuids

If i understood situation right, You just need to loop through all received uuids and get players' data. Something like this:
f = requests.get("https://api.hypixel.net/guild?key=[secret]&id=" + guild).json()
guildName = f["guild"]["name"]
guildMembers = f["guild"]["members"]
guildMembersData = dict() # Here we will save member's data from getPlayer method
for guildMember in guildMembers:
uuid = guildMember["uuid"]
memberData = client.getPlayer(uuid=uuid)
guildMembersData[uuid] = client.getPlayer(uuid=guildMember["uuid"])
print(guildMembersData) # Here will be players' Data.

How to decode dataTables Editor form in python flask?

I have a flask application which is receiving a request from dataTables Editor. Upon receipt at the server, request.form looks like (e.g.)
ImmutableMultiDict([('data[59282][gender]', u'M'), ('data[59282][hometown]', u''),
('data[59282][disposition]', u''), ('data[59282][id]', u'59282'),
('data[59282][resultname]', u'Joe Doe'), ('data[59282][confirm]', 'true'),
('data[59282][age]', u'27'), ('data[59282][place]', u'3'), ('action', u'remove'),
('data[59282][runnerid]', u''), ('data[59282][time]', u'29:49'),
('data[59282][club]', u'')])
I am thinking to use something similar to this really ugly code to decode it. Is there a better way?
from collections import defaultdict
# request.form comes in multidict [('data[id][field]',value), ...]
# so we need to exec this string to turn into python data structure
data = defaultdict(lambda: {}) # default is empty dict
# need to define text for each field to be received in data[id][field]
age = 'age'
club = 'club'
confirm = 'confirm'
disposition = 'disposition'
gender = 'gender'
hometown = 'hometown'
id = 'id'
place = 'place'
resultname = 'resultname'
runnerid = 'runnerid'
time = 'time'
# fill in data[id][field] = value
for formkey in request.form.keys():
exec '{} = {}'.format(d,repr(request.form[formkey]))

This question has an accepted answer and is a bit old but since the DataTable module seems being pretty popular among jQuery community still, I believe this approach may be useful for someone else. I've just wrote a simple parsing function based on regular expression and dpath module, though it appears not to be quite reliable module. The snippet may be not very straightforward due to an exception-relied fragment, but it was only one way to prevent dpath from trying to resolve strings as integer indices I found.
import re, dpath.util
rxsKey = r'(?P<key>[^\W\[\]]+)'
rxsEntry = r'(?P<primaryKey>[^\W]+)(?P<secondaryKeys>(\[' \
+ rxsKey \
+ r'\])*)\W*'
rxKey = re.compile(rxsKey)
rxEntry = re.compile(rxsEntry)
def form2dict( frmDct ):
res = {}
for k, v in frmDct.iteritems():
m = rxEntry.match( k )
if not m: continue
mdct = m.groupdict()
if not 'secondaryKeys' in mdct.keys():
res[mdct['primaryKey']] = v
else:
fullPath = [mdct['primaryKey']]
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
k = sk.groupdict()['key']
try:
dpath.util.get(res, fullPath)
except KeyError:
dpath.util.new(res, fullPath, [] if k.isdigit() else {})
fullPath.append(int(k) if k.isdigit() else k)
dpath.util.new(res, fullPath, v)
return res
The practical usage is based on native flask request.form.to_dict() method:
# ... somewhere in a view code
pars = form2dict(request.form.to_dict())
The output structure includes both, dictionary and lists, as one could expect. E.g.:
# A little test:
rs = jQDT_form2dict( {
'columns[2][search][regex]' : False,
'columns[2][search][value]' : None,
'columns[2][search][regex]' : False,
} )
generates:
{
"columns": [
null,
null,
{
"search": {
"regex": false,
"value": null
}
}
]
}
Update: to handle lists as dictionaries (in more efficient way) one may simplify this snippet with following block at else part of if clause:
# ...
else:
fullPathStr = mdct['primaryKey']
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
fullPathStr += '/' + sk.groupdict()['key']
dpath.util.new(res, fullPathStr, v)

I decided on a way that is more secure than using exec:
from collections import defaultdict
def get_request_data(form):
'''
return dict list with data from request.form
:param form: MultiDict from `request.form`
:rtype: {id1: {field1:val1, ...}, ...} [fieldn and valn are strings]
'''
# request.form comes in multidict [('data[id][field]',value), ...]
# fill in id field automatically
data = defaultdict(lambda: {})
# fill in data[id][field] = value
for formkey in form.keys():
if formkey == 'action': continue
datapart,idpart,fieldpart = formkey.split('[')
if datapart != 'data': raise ParameterError, "invalid input in request: {}".format(formkey)
idvalue = int(idpart[0:-1])
fieldname = fieldpart[0:-1]
data[idvalue][fieldname] = form[formkey]
# return decoded result
return data