We Keep Coding

Parse a Generated File Python

I'm trying to parse generated files into a list of objects.
Unfortunately the structure of the generated files is not always the same, but they contain the same fields (and lots of other garbage).
For example:
function foo(); # Don't Care
function maybeanotherfoo(); # Don't Care
int maybemoregarbage; # Don't Care
product_serial = "CDE1102"; # I want this <---------------------
unnecessary_info1 = 10; # Don't Care
unnecessary_info2 = "red" # Don't Care
product_id = 1134412; # I want this <---------------------
unnecessary_info3 = "88" # Don't Care
product_serial = "DD1232"; # I want this <---------------------
product_id = 3345111; # I want this <---------------------
unnecessary_info1 = "22" # Don't Care
unnecessary_info2 = "panda" # Don't Care
product_serial = "CDE1102"; # I want this <---------------------
unnecessary_info1 = 10; # Don't Care
unnecessary_info2 = "red" # Don't Care
unnecessary_info3 = "bear" # Don't Care
unnecessary_info4 = 119 # Don't Care
product_id = 1112331; # I want this <---------------------
unnecessary_info5 = "jj" # Don't Care
I want a list of objects (each object has: serial and id).
I have tried the following:
import re
class Product:
def __init__(self, id, serial):
self.product_id = id
self.product_serial = serial
linenum = 0
first_string = "product_serial"
second_string = "product_id"
with open('products.txt', "r") as products_file:
for line in products_file:
linenum += 1
if line.find(first_string) != -1:
product_serial = re.search('\"([^"]+)', line).group(1)
#How do I proceed?
Any advice would be greatly appreciated!
Thanks!

I've inlined the data here using an io.StringIO(), but you can substitute data for your products_file.
The idea is that we gather key/values into current_object, and as soon as we have all the data we know we need for a single object (the two keys), we push it onto a list of objects and prime a new current_object.
You could use something like if line.startswith('product_serial') instead of the admittedly complex regexp.
import io
import re
data = io.StringIO("""
function foo();
function maybeanotherfoo();
int maybemoregarbage;
product_serial = "CDE1102";
unnecessary_info1 = 10;
unnecessary_info2 = "red"
product_id = 1134412;
unnecessary_info3 = "88"
product_serial = "DD1232";
product_id = 3345111;
unnecessary_info1 = "22"
unnecessary_info2 = "panda"
product_serial = "CDE1102";
unnecessary_info1 = 10;
unnecessary_info2 = "red"
unnecessary_info3 = "bear"
unnecessary_info4 = 119
product_id = 1112331;
unnecessary_info5 = "jj"
""")
objects = []
current_object = {}
for line in data:
line = line.strip() # Remove leading and trailing whitespace
m = re.match(r'^(product_id|product_serial)\s*=\s*(\d+|"(?:.+?)");?$', line)
if m:
key, value = m.groups()
current_object[key] = value.strip('"')
if len(current_object) == 2: # Got the two keys we want, ship the object
objects.append(current_object)
current_object = {}
print(objects)

How to add the object names in a dict based on their type in python

I have several objects e.g.,
obj_1 = class(obj_type, "name_1", stereotype = "type_1")
obj_2 = class(obj_type, "name_2", stereotype = "type_2")
...
and I would like to create a dict with the names as keys and the stereotypes as values:
{obj_1: type_1, obj_2: type_2}

Use package python-varname
from varname import nameof
output = {}
obj_1 = class(obj_type, "name_1", stereotype = "type_1")
obj_2 = class(obj_type, "name_2", stereotype = "type_2")
output[nameof(obj1)] = obj1.stereotype
output[nameof(obj2)] = obj2.stereotype
print(output)

Multiple identical keys from Python dict to JSON

I am trying to create JSON object in Python, and it works just fine despite the fact that I can't get multiple keys with the same name - but I need to do it.
Here's a function:
findings = AutoTree()
findings['report']['numberOfConditions'] = num_cond
if r == 'Mammography':
f_temp = df['Relevant findings'].values.tolist()[0:8]
f_list = [x for i, x in enumerate(f_temp) if i == f_temp.index(x)]
f_num_total = len(f_list)
f_rand = random.randrange(1, f_num_total + 1)
iter_params_mass = ['shape', 'margin', 'density']
for i in range(num_cond):
br = get_birad(row, 2, 7)
cond = camelCase(get_cond_name())
findings[cond]['biRad'] = br
for k in range(f_rand + 1):
f = camelCase(random.choice(f_list))
#f = 'mass'
if f == 'mass':
rep_temp = create_rep(iter_params_mass, row, f, r)
findings[cond][f] = rep_temp
"""I also have a lot elif conditions, and it just grabs parameters."""
report = json.dumps(findings)
print(report)
Output:
{
"report":{
"id":85,
"name":"Lydia",
"age":39,
"relevantModality":"Mammography",
"numberOfConditions":2
},
"ductEctasia":{
"biRad":"birad[1]",
"calcifications":[
{
"typicallyBenign":"Vascular",
"suspiciousMorphology":"Coarse heterogeneous",
"distribution":"Diffuse"
}
],
"lymphNodes":[
{
"lymphNodes":"Lymph nodes \u2013 axillary"
}
]
}
}
And I want to have multiple "lymphNodes" and "calcifications" objects. Is it possible? Maybe, you can suggest another way to create JSON object, not nested dictionaries? The problem is that I need to create object respectively to random parameter chosen from the database.

python generating nested dictionary key error

I am trying to create a nested dictionary from a mysql query but I am getting a key error
result = {}
for i, q in enumerate(query):
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
error
KeyError: 'data'
desired result
result = {
'data': {
0: {'firstName': ''...}
1: {'firstName': ''...}
2: {'firstName': ''...}
}
}

You wanted to create a nested dictionary
result = {} will create an assignment for a flat dictionary, whose items can have any values like "string", "int", "list" or "dict"
For this flat assignment
python knows what to do for result["first"]
If you want "first" also to be another dictionary you need to tell Python by an assingment
result['first'] = {}.
otherwise, Python raises "KeyError"
I think you are looking for this :)
>>> from collections import defaultdict
>>> mydict = lambda: defaultdict(mydict)
>>> result = mydict()
>>> result['Python']['rules']['the world'] = "Yes I Agree"
>>> result['Python']['rules']['the world']
'Yes I Agree'

result = {}
result['data'] = {}
for i, q in enumerate(query):
result['data']['i'] = {}
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
Alternatively, you can use you own class which adds the extra dicts automatically
class AutoDict(dict):
def __missing__(self, k):
self[k] = AutoDict()
return self[k]
result = AutoDict()
for i, q in enumerate(query):
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email

result['data'] does exist. So you cannot add data to it.
Try this out at the start:
result = {'data': []};

You have to create the key data first:
result = {}
result['data'] = {}
for i, q in enumerate(query):
result['data'][i] = {}
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email

re reading a csv file in python without loading it again

I made the following code which works but I want to improve it. I don't want to re-read the file, but if I delete sales_input.seek(0) it won't iterate throw each row in sales. How can i improve this?
def computeCritics(mode, cleaned_sales_input = "data/cleaned_sales.csv"):
if mode == 1:
print "creating customer.critics.recommendations"
critics_output = open("data/customer/customer.critics.recommendations",
"wb")
ID = getCustomerSet(cleaned_sales_input)
sales_dict = pickle.load(open("data/customer/books.dict.recommendations",
"r"))
else:
print "creating books.critics.recommendations"
critics_output = open("data/books/books.critics.recommendations",
"wb")
ID = getBookSet(cleaned_sales_input)
sales_dict = pickle.load(open("data/books/users.dict.recommendations",
"r"))
critics = {}
# make critics dict and pickle it
for i in ID:
with open(cleaned_sales_input, 'rb') as sales_input:
sales = csv.reader(sales_input) # read new
for j in sales:
if mode == 1:
if int(i) == int(j[2]):
sales_dict[int(j[6])] = 1
else:
if int(i) == int(j[6]):
sales_dict[int(j[2])] = 1
critics[int(i)] = sales_dict
pickle.dump(critics, critics_output)
print "done"
cleaned_sales_input looks like
6042772,2723,3546414,9782072488887,1,9.99,314968
6042769,2723,3546414,9782072488887,1,9.99,314968
...
where number 6 is the book ID and number 0 is the customer ID
I want to get a dict wich looks like
critics = {
CustomerID1: {
BookID1: 1,
BookID2: 0,
........
BookIDX: 0
},
CustomerID2: {
BookID1: 0,
BookID2: 1,
...
}
}
or
critics = {
BookID1: {
CustomerID1: 1,
CustomerID2: 0,
........
CustomerIDX: 0
},
BookID1: {
CustomerID1: 0,
CustomerID2: 1,
...
CustomerIDX: 0
}
}
I hope this isn't to much information

Here are some suggestions:
Let's first look at this code pattern:
for i in ID:
for j in sales:
if int(i) == int(j[2])
notice that i is only being compared with j[2]. That's its only purpose in the loop. int(i) == int(j[2]) can only be True at most once for each i.
So, we can completely remove the for i in ID loop by rewriting it as
for j in sales:
key = j[2]
if key in ID:
Based on the function names getCustomerSet and getBookSet, it sounds as if
ID is a set (as opposed to a list or tuple). We want ID to be a set since
testing membership in a set is O(1) (as opposed to O(n) for a list or tuple).
Next, consider this line:
critics[int(i)] = sales_dict
There is a potential pitfall here. This line is assigning sales_dict to
critics[int(i)] for each i in ID. Each key int(i) is being mapped to the very same dict. As we loop through sales and ID, we are modifying sales_dict like this, for example:
sales_dict[int(j[6])] = 1
But this will cause all values in critics to be modified simultaneously, since all keys in critics point to the same dict, sales_dict. I doubt that is what you want.
To avoid this pitfall, we need to make copies of the sales_dict:
critics = {i:sales_dict.copy() for i in ID}
def computeCritics(mode, cleaned_sales_input="data/cleaned_sales.csv"):
if mode == 1:
filename = 'customer.critics.recommendations'
path = os.path.join("data/customer", filename)
ID = getCustomerSet(cleaned_sales_input)
sales_dict = pickle.load(
open("data/customer/books.dict.recommendations", "r"))
key_idx, other_idx = 2, 6
else:
filename = 'books.critics.recommendations'
path = os.path.join("data/books", filename)
ID = getBookSet(cleaned_sales_input)
sales_dict = pickle.load(
open("data/books/users.dict.recommendations", "r"))
key_idx, other_idx = 6, 2
print "creating {}".format(filename)
ID = {int(item) for item in ID}
critics = {i:sales_dict.copy() for i in ID}
with open(path, "wb") as critics_output:
# make critics dict and pickle it
with open(cleaned_sales_input, 'rb') as sales_input:
sales = csv.reader(sales_input) # read new
for j in sales:
key = int(j[key_idx])
if key in ID:
other_key = int(j[other_idx])
critics[key][other_key] = 1
critics[key] = sales_dict
pickle.dump(dict(critics), critics_output)
print "done"

#unutbu's answer is better but if you are stuck with this structure you can put the whole file in memory:
sales = []
with open(cleaned_sales_input, 'rb') as sales_input:
sales_reader = csv.reader(sales_input)
[sales.append(line) for line in sales_reader]
for i in ID:
for j in sales:
#do stuff