We Keep Coding

How to extract value from json and increment - python

Sample json is below. I want to save id which is completed (False and True) into seperated dictionaryies
todos = [{'userId': 1, 'id': 1, 'title': 'delectus aut autem', 'completed': False},
{'userId': 1, 'id': 2, 'title': 'quis ut nam facil ', 'completed': False},
{'userId': 1, 'id': 1, 'title': 'fugiat veniam minus', 'completed': False},
{'userId': 1, 'id': 2, 'title': 'et porro tempora', 'completed': True},
{'userId': 1, 'id': 1,'title': 'laprovident illum', 'completed': False}]
Expected out is below
todos_by_user_true = {1:0,2:1}
todos_by_user_false = {1:3,2:1}
code is below? Why my code not working. I am getting blank dictionary
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
if todo["completed"]==True:
try:
# Increment the existing user's count.
todos_by_user_true[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_true[todo["id"]] = 0
elif todo["completed"]==False:
try:
# Increment the existing user's count.
todos_by_user_false[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_false[todo["id"]] = 0
I am getting not proper dictionary
My output is below
todos_by_user_false {1: 2, 2: 0}
todos_by_user_true {2: 0}
Disclaimer: I need to take care of exception also

Looking at your input data, it is so that:
userId 1, id 1 has 0 true, and 3 false
userId 1, id 2 has 1 true, and 1 false
Given the required output, it looks like you really want to use id rather than userId in your lookups. Besides that, there's an issue with accounting the first time you insert the id in the resulting dictionary. I would fix it like this:
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
if todo["completed"]==True:
try:
# Increment the existing user's count.
todos_by_user_true[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_true[todo["id"]] = 1
elif todo["completed"]==False:
try:
# Increment the existing user's count.
todos_by_user_false[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_false[todo["id"]] = 1
which (btw) is already what's in your comments.
Personally, I would check the dictionary for the key before insertion, instead of using try..except, like this:
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
key = todo["id"]
if todo["completed"]: # true case
# If `id` not there yet, insert it to 0
if key not in todos_by_user_true:
todos_by_user_true[key] = 0
# increment
todos_by_user_true[key] += 1
else: # false case
# If `id` not there yet, insert it to 0
if key not in todos_by_user_false:
todos_by_user_false[key] = 0
# increment
todos_by_user_false[key] += 1
This gives out:
todos_by_user_true = {2:1}
todos_by_user_false = {1:3,2:1}
The logic being this, you cannot have:
todos_by_user_true = {1:0}
You account for the value when you find it; rather than iterating for id from a separate list.

Related

Team: need some assistance..
sub: loosing dict content as soon as am out of loop. dict is populated with loop vars that are added to dict using subscript approach.
below foo() is always getting executed because the team_oncall_dict is empty outside. any hint how can I retain it as it was inside loop?
def askduty_oncall(self, *args):
session = APISession(PD_API_KEY, default_from=PD_USER_EMAIL)
total = 1 #true or false
limit = 40
teamteamnm = "Team Test Team"
team_esp_name = “Team Test Escalation Policy"
teamteamid = ""
teamesplcyid = ""
team_oncall_dict = {}
if args:
offset = args[0]
total_teams = args[1]
if offset <= total_teams:
print("\nfunc with args with new offset {} called\n".format(offset))
teams = session.get('/teams?limit={0}&total={1}&offset={2}'.format(limit,total,offset))
else:
print("Reached max teams, no more team records to pull")
return
else:
print("\nFunc with no args called, hence pull first set of {} teams as defined by limit var\n".format(limit))
teams = session.get('/teams?limit={0}&total={1}'.format(limit,total))
if not teams.ok:
return
else:
tj = teams.json()
tjd = tj['teams']
for adict in tjd:
if not adict['name'] == teamteamnm:
continue
elif adict['name'] == teamteamnm:
teamteamid = adict['id']
print("\nFound team..\nFetched",adict['name'], "id: {0}".format(teamteamid))
print("Pull escalation policy for team '{}':'{}'".format(teamteamnm,teamteamid))
esclp = session.get('/escalation_policies?total={0}&team_ids%5B%5D={1}'.format(total,teamteamid))
if not esclp.ok:
print("Pulling Escalation polices for team '{}' failed".format(teamteamnm))
return
else:
ep = esclp.json()
epj = esclp.json()['escalation_policies']
if not epj:
print("Escalation polices for team '{}' not defined".format(teamteamnm))
return
else:
for adict2 in epj:
if not adict2['summary'] == team_esp_name:
continue
else:
print("***************FOUND FOUND********************")
teamesplcyid = adict2['id']
print("\nFetched {} id: {}\n".format(team_esp_name, teamesplcyid))
oncalls = session.get('/oncalls?total={0}&escalation_policy_ids%5B%5D={1}'.format(total,teamesplcyid))
if not oncalls.ok:
print(“issue “with oncalls)
return
else:
ocj = oncalls.json()['oncalls']
for adict3 in ocj:
print("\n")
print(adict3['escalation_level'])
if i['escalation_level'] == 1:
print(adict3['schedule']['summary'], adict3['user']['summary'])
team_oncall_dict[adict3['schedule']['summary']] = adict3['user']['summary']
print(team_oncall_dict)
return team_oncall_dict
if not team_oncall_dict: #part of func def
do foo()
output
foo stuff
sample data is a list of dicts
[{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team Escalation Policy'}, 'escalation_level': 3, 'schedule': None, 'user': {'id': 'PX8XYFT', 'type': 'user_reference', 'summary': 'M1’}, 'start': None, 'end': None},
{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team Escalation Policy'}, 'escalation_level': 1, 'schedule': None, 'user': {'id': 'PKXXVJI', 'type': 'user_reference', 'summary': ‘R1’}, 'start': None, 'end': None},
{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team’}, 'escalation_level': 2, 'schedule': None, 'user': {'d': 'PN8F9PC', 'type': 'user_reference’,'summary': ‘T1’}],'start': None, 'end': None}]
btw: above is 4th inner loop.
so flow is like this diagramatically.
def func1()
team_oncall_dict = {}
loop1
loop2
loop3
loop4
...
team_oncall_dict
if not team_oncall_dict:
print("dict is empty")
output
dict is empty

t was local vs global. fixed it by declaring the team_oncall_dict globally outside the function.
intead of
def func1()
team_oncall_dict = {}
team_oncall_dict = {}
def func1()

I try to return a dictionary,that contains 3 different values.
(The total number of sales per Branch and per Customer Type)
I need to return dictinary like
{"A": {"Member": 230, "Normal": 351},
"B": {"Member": 123, "Normal": 117},
"C": {"Member": 335, "Normal": 18}}
What am I doing wrong, can someone explain me, please ?
import csv
dict_from_csv = {}
file = open('supermarket_sales.csv')
csv_reader = csv.reader(file)
next(csv_reader)
for row in csv_reader:
branch= row[1]
Customer = row[3]
total = float(row[9])
curent_total= dict_from_csv.get(branch)
if dict_from_csv.get(branch) is None:
dict_from_csv[branch]={}
else:
if dict_from_csv[branch].get(Customer) is None:
dict_from_csv[branch]={Customer: 0}
else :
curent_total= dict_from_csv.get(branch)
print(dict_from_csv)
I can return only:
{'A': {'Member': 0},
'C': {'Normal': 0},
'B': {'Normal': 0}}
scv file: https://app.box.com/s/f4hcfkferizntbev3ou8hso6nyfccf74

First mistake:
dict_from_csv[branch]={Customer: 0}
it creates new dictionary with new Customer but it also removes previous dictionary with prediction customer.
It should be
dict_from_csv[branch][Customer] = 0
and it adds new customer to existing dictionary.
Second mistake: you don't have += 1 to count elements
dict_from_csv[branch][customer] += 1
Full working code:
import csv
f = open('supermarket_sales.csv')
csv_reader = csv.reader(f)
next(csv_reader)
dict_from_csv = {}
for row in csv_reader:
branch= row[1]
customer = row[3]
if branch not in dict_from_csv:
dict_from_csv[branch] = {}
if customer not in dict_from_csv[branch]:
dict_from_csv[branch][customer] = 0
dict_from_csv[branch][customer] += 1
print(dict_from_csv)
Result:
{
'A': {'Member': 167, 'Normal': 173},
'C': {'Normal': 159, 'Member': 169},
'B': {'Member': 165, 'Normal': 167}
}
BTW: if you would use pandas.DataFrame then using
import pandas as pd
df = pd.read_csv('/home/furas/supermarket_sales.csv')
sizes = df.groupby(['Branch','Customer type']).size()
print(sizes)
you would get
Branch Customer type
A Member 167
Normal 173
B Member 165
Normal 167
C Member 169
Normal 159
dtype: int64
Bigger problem makes to convert it to expected dictionary.
There is .to_dict() and .to_json() but they don't create expected dictionary.
But this works for me:
result = dict()
for ((brand, customer), number) in sizes.items():
if brand not in result:
result[brand] = dict()
result[brand][customer] = number
print(result)

I'm working on database that uses lot of data. One invoice could have 7482 different articles. Validating invoice cost so much time, it took 26 minutes to validate one with 7482 articles. I find the method that take time to finish, it is the "action_move_create" inside "odoo\addons\account\models\account_invoice.py".
#api.multi
def action_move_create(self):
""" Creates invoice related analytics and financial move lines """
account_move = self.env['account.move']
for inv in self:
if not inv.journal_id.sequence_id:
raise UserError(_('Please define sequence on the journal related to this invoice.'))
if not inv.invoice_line_ids.filtered(lambda line: line.account_id):
raise UserError(_('Please add at least one invoice line.'))
if inv.move_id:
continue
if not inv.date_invoice:
inv.write({'date_invoice': fields.Date.context_today(self)})
if not inv.date_due:
inv.write({'date_due': inv.date_invoice})
company_currency = inv.company_id.currency_id
# create move lines (one per invoice line + eventual taxes and analytic lines)
iml = inv.invoice_line_move_line_get()
iml += inv.tax_line_move_line_get()
diff_currency = inv.currency_id != company_currency
# create one move line for the total and possibly adjust the other lines amount
total, total_currency, iml = inv.compute_invoice_totals(company_currency, iml)
name = inv.name or ''
if inv.payment_term_id:
totlines = inv.payment_term_id.with_context(currency_id=company_currency.id).compute(total, inv.date_invoice)[0]
res_amount_currency = total_currency
for i, t in enumerate(totlines):
if inv.currency_id != company_currency:
amount_currency = company_currency._convert(t[1], inv.currency_id, inv.company_id, inv._get_currency_rate_date() or fields.Date.today())
else:
amount_currency = False
# last line: add the diff
res_amount_currency -= amount_currency or 0
if i + 1 == len(totlines):
amount_currency += res_amount_currency
iml.append({
'type': 'dest',
'name': name,
'price': t[1],
'account_id': inv.account_id.id,
'date_maturity': t[0],
'amount_currency': diff_currency and amount_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
else:
iml.append({
'type': 'dest',
'name': name,
'price': total,
'account_id': inv.account_id.id,
'date_maturity': inv.date_due,
'amount_currency': diff_currency and total_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
part = self.env['res.partner']._find_accounting_partner(inv.partner_id)
line = [(0, 0, self.line_get_convert(l, part.id)) for l in iml]
line = inv.group_lines(iml, line)
line = inv.finalize_invoice_move_lines(line)
date = inv.date or inv.date_invoice
move_vals = {
'ref': inv.reference,
'line_ids': line,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
move = account_move.create(move_vals)
# Pass invoice in method post: used if you want to get the same
# account move reference when creating the same invoice after a cancelled one:
move.post(invoice = inv)
# make the invoice point to that move
vals = {
'move_id': move.id,
'date': date,
'move_name': move.name,
}
inv.write(vals)
return True
Could you suggest some solutions?
We suppose that the hardware is efficient to run odoo correctly.

I optimize it by using raw sql query. I made these codes in account.invoice model:
The first one is the definition of _mock_create_move_line (called in action_move_create).
def _mock_create_move_line(self, model, values, move):
bad_names = ["analytic_line_ids", "tax_ids", "analytic_tag_ids"]
other_fields = [
"currency_id", "debit", "credit", "balance",
"debit_cash_basis", "credit_cash_basis", "balance_cash_basis",
"company_currency_id", "amount_residual",
"amount_residual_currency", "tax_base_amount", "reconciled",
"company_id", "counterpart"
]
cr = self.env.cr
quote = '"{}"'.format
columns = []
columns1 = []
for i, v in enumerate(values):
v = model._add_missing_default_values(v)
account_id = self.env['account.account'].browse(v['account_id'])
# compulsory columns and some stored related columns
# related fields are not triggered, krrrrr
v.update({
'move_id': move.id,
'date_maturity': move.date,
'company_id': account_id.company_id.id,
'date': move.date,
'journal_id': move.journal_id.id,
'user_type_id': account_id.user_type_id.id,
'create_uid': self.env.uid,
'create_date': fields.Datetime.now()
})
######
temp_column = []
for name, val in sorted(v.items()):
if name in bad_names:
continue
field = model._fields[name]
if field.column_type:
col_val = field.convert_to_column(val, model, v)
temp_column.append(col_val)
if not i:
columns1.append((name, field.column_format, col_val))
columns.append(tuple(temp_column))
model.check_access_rule('create')
try:
query = "INSERT INTO {} ({}) VALUES {} RETURNING id".format(
quote(model._table),
", ".join(quote(name) for name, fmt, val in columns1),
", ".join('%s' for fmt in columns),
)
cr.execute(query, columns)
ids = cr.fetchall()
# clear the model cache to take account of the new insertion
# if not executed, relationnal field will not be updated
model.invalidate_cache()
account_move_line_ids = model.browse(ids)
account_move_line_ids.modified(other_fields)
account_move_line_ids.recompute()
# update parent_path
account_move_line_ids._parent_store_create()
except Exception as e:
_logger.info(e)
cr.rollback()
return
The second one is the overriding of native method action_move_create. I make some modification, call _mock_create_move_line if there is 'raw_sql' in the context.
#api.multi
def action_move_create(self):
""" Creates invoice related analytics and financial move lines """
# TODO : make choice between ORM or raw sql according to the context
account_move = self.env['account.move']
for inv in self:
if not inv.journal_id.sequence_id:
raise UserError(_('Please define sequence on the journal related to this invoice.'))
if not inv.invoice_line_ids.filtered(lambda line: line.account_id):
raise UserError(_('Please add at least one invoice line.'))
if inv.move_id:
continue
if not inv.date_invoice:
inv.write({'date_invoice': fields.Date.context_today(self)})
if not inv.date_due:
inv.write({'date_due': inv.date_invoice})
company_currency = inv.company_id.currency_id
# create move lines (one per invoice line + eventual taxes and analytic lines)
iml = inv.invoice_line_move_line_get()
iml += inv.tax_line_move_line_get()
diff_currency = inv.currency_id != company_currency
# create one move line for the total and possibly adjust the other lines amount
total, total_currency, iml = inv.compute_invoice_totals(company_currency, iml)
name = inv.name or ''
if inv.payment_term_id:
totlines = \
inv.payment_term_id.with_context(currency_id=company_currency.id).compute(total, inv.date_invoice)[0]
res_amount_currency = total_currency
for i, t in enumerate(totlines):
if inv.currency_id != company_currency:
amount_currency = company_currency._convert(t[1], inv.currency_id, inv.company_id,
inv._get_currency_rate_date() or fields.Date.today())
else:
amount_currency = False
# last line: add the diff
res_amount_currency -= amount_currency or 0
if i + 1 == len(totlines):
amount_currency += res_amount_currency
iml.append({
'type': 'dest',
'name': name,
'price': t[1],
'account_id': inv.account_id.id,
'date_maturity': t[0],
'amount_currency': diff_currency and amount_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
else:
iml.append({
'type': 'dest',
'name': name,
'price': total,
'account_id': inv.account_id.id,
'date_maturity': inv.date_due,
'amount_currency': diff_currency and total_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
part = self.env['res.partner']._find_accounting_partner(inv.partner_id)
line = [(0, 0, self.line_get_convert(l, part.id)) for l in iml]
line = inv.group_lines(iml, line)
line = inv.finalize_invoice_move_lines(line)
date = inv.date or inv.date_invoice
if self.env.context.get('raw_sql', None):
move_vals = {
'ref': inv.reference,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
# remove (0, 0, ...)
# override the group_lines method to avoid looping on next instruction
new_lines = [nl[2] for nl in line]
# TODO do not call compute here, add with ...norecompute()
move = account_move.create(move_vals)
move.env.cr.commit()
self._mock_create_move_line(self.env['account.move.line'], new_lines, move)
# Pass invoice in method post: used if you want to get the same
# account move reference when creating the same invoice after a cancelled one:
# compute move, it is not triggered automatically bc raw sql insertion
# is it correct to call it like this ? find better way
move._amount_compute()
move._compute_partner_id()
move._compute_matched_percentage()
else:
# make default behavior
move_vals = {
'ref': inv.reference,
'line_ids': line,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
move = account_move.create(move_vals)
move.post(invoice=inv)
# make the invoice point to that move
vals = {
'move_id': move.id,
'date': date,
'move_name': move.name,
}
inv.write(vals)
return True
Now, the execution time is less than 1 minutes for about 7000 records to insert inside invoice.move.line

I am trying to insert records into a table, but only last record(result data) from the loop is inserting into the table
Here is the code i tried:
CDates = ['2020-05-10','2020-05-12','2020-05-13','2020-05-16','2020-05-20']
ResultData = {}
for date in CDates:
filterDate = Key('Date').eq(id)
appResponse = appTable.scan(FilterExpression = filterDate)
accResp = table.query(KeyConditionExpression = Key('PrimaryId').eq('Key'),FilterExpression = Key('Date').eq(date))
if len(accResp['Items']) == 0:
ResultData['PrimaryId'] = 'Key'
ResultData['CreatedDate'] = date
ResultData['Type'] = 'Appt'
ResultData['Id'] = str(uuid.uuid4())
print(ResultData)
table.put_item(Item=ResultData)
Not getting where did I go wrong

You assigned ResultData outside of the loop and changed the values for the same keys every time the loop ran. Try this:
CDates = ['2020-05-10', '2020-05-12', '2020-05-13', '2020-05-16', '2020-05-20']
for date in CDates:
filterDate = Key('Date').eq(id)
appResponse = appTable.scan(FilterExpression=filterDate)
accResp = table.query(
KeyConditionExpression=Key('PrimaryId').eq('Key'),
FilterExpression=Key('Date').eq(date))
if len(accResp['Items']) == 0:
ResultData = {
'PrimaryId': 'Key',
'CreationDate': date,
'Type': 'Appt',
'Id': str(uuid.uuid4())
}
print(ResultData)
table.put_item(Item=ResultData)

I am trying to load the 8m[rows] * 1k[columns] python dataframe into Mongo. For the performance improvement I have planned to use Mongo bulk operation.There will be update also I have to do in the collections on daily basis so that I used the upsert method of bulk operation. Below is the code which I have prepared,
def BulkLoad(self, Dataset):
counter = 0;
empty = '000'
columns = []
records = []
DataDict = {}
for col in Dataset.columns:
columns.append(col)
try:
db = self.getDatabase()
bulk = db.collection.initialize_ordered_bulk_op()
for j in range(len(Dataset)):
records.clear()
DataDict.clear()
DataDict.update(
{'CreatedBy': empty, 'ModifiedBy': empty, 'Value': Score})
for column in columns:
colValue = str(Dataset[column][j])
if (colValue == 'nan'):
colValue = colValue.replace('nan', '')
DataDict.update({column: colValue})
records.append(DataDict)
print("list is ",records)
Id = DataDict['Id']
Number = DataDict['Number']
print(DataDict)
bulk.find(
{'Id': Id, 'Number': Number}).upsert().update(
{
'$set': {'Id': Id, 'Number': Number,'Events':records}
})
counter += 1
if counter % 1000 == 0:
result = bulk.execute()
logging.info(pprint(result))
bulk = db.coll.initialize_ordered_bulk_op()
if counter % 1000 != 0:
result = bulk.execute()
logging.info(pprint(result))
except Exception as e:
logging.exception(e)
except BulkWriteError as bre:
logging.error(pprint(bre.details))
If I am loading the sample rows of 10 into Mongo collections, All the documents are having the same value of 10th row. I knew its because of python dictionary reference problem .
Can you please anyone give me suggestion on that ?

def BulkLoad(self, Dataset):
counter = 0;
empty = '000'
columns = []
records = []
for col in Dataset.columns:
columns.append(col)
try:
db = self.getDatabase()
bulk = db.collection.initialize_ordered_bulk_op()
for j in range(len(Dataset)):
DataDict = {}
DataDict.update(
{'CreatedBy': empty, 'ModifiedBy': empty, 'Value': Score})
for column in columns:
colValue = str(Dataset[column][j])
if (colValue == 'nan'):
colValue = colValue.replace('nan', '')
DataDict.update({column: colValue})
records.append(DataDict)
print("list is ",records)
Id = DataDict['Id']
Number = DataDict['Number']
print(DataDict)
bulk.find(
{'Id': Id, 'Number': Number}).upsert().update(
{
'$set': {'Id': Id, 'Number': Number,'Events':records}
})
counter += 1
if counter % 1000 == 0:
result = bulk.execute()
logging.info(pprint(result))
bulk = db.coll.initialize_ordered_bulk_op()
if counter % 1000 != 0:
result = bulk.execute()
logging.info(pprint(result))
except Exception as e:
logging.exception(e)
except BulkWriteError as bre:
logging.error(pprint(bre.details))