I'm planning to write a test for a query for the database but I wonder what is the best way to write it and can I use #pytest.mark.parametrize for this type of test, because I need to test 4 type of tests that looks very similar
def test_get_number(self):
#first create test data in the database
#then make queries
test_number = 'test_number'
data_base_records_number = self.get_records_query(number)
# data_base_records_number =
# [{'id': 1 ,
# 'number':"test_number",
# 'company':"test_company1",
# 'warehouse' : "test_warehouse1",
# 'product': "test_product1"},
# {'id': 2 ,
# 'number':"test_number",
# 'company':"test_company2",
# 'warehouse' : "test_warehouse2",
# 'product': "test_product2"},
# {'id': 3,
# 'number': "test_number",
# 'company': "test_company3",
# 'warehouse': "test_warehouse3",
# 'product': "test_product3"},
# {'id': 4,
# 'number': "test_number",
# 'company': "test_company4",
# 'warehouse': "test_warehouse4",
# 'product': "test_product4"}
# ]
assert len(data_base_records_number) == 4
for record in data_base_records:
assert record['number'] == number
def test_get_number_and_company(self):
#first create test data in the database
#then make queries
test_number = 'test_number'
company = 'test_company1'
data_base_records_number_company = self.get_records_query(number, company)
#Output
# data_base_records_number_company =
# [{'id': 1 ,
# 'number':"test_number",
# 'company':"test_company1",
# 'warehouse' : "test_warehouse1",
# 'product': "test_product1"},
#
# ]
assert len(data_base_records_number_company) == 1
assert data_base_records_number_company['number'] == number
assert data_base_records_number_company['company'] == company
def test_get_number_and_warehouse(self):
...
def test_get_number_and_product(self):
...
Related
Team: need some assistance..
sub: loosing dict content as soon as am out of loop. dict is populated with loop vars that are added to dict using subscript approach.
below foo() is always getting executed because the team_oncall_dict is empty outside. any hint how can I retain it as it was inside loop?
def askduty_oncall(self, *args):
session = APISession(PD_API_KEY, default_from=PD_USER_EMAIL)
total = 1 #true or false
limit = 40
teamteamnm = "Team Test Team"
team_esp_name = “Team Test Escalation Policy"
teamteamid = ""
teamesplcyid = ""
team_oncall_dict = {}
if args:
offset = args[0]
total_teams = args[1]
if offset <= total_teams:
print("\nfunc with args with new offset {} called\n".format(offset))
teams = session.get('/teams?limit={0}&total={1}&offset={2}'.format(limit,total,offset))
else:
print("Reached max teams, no more team records to pull")
return
else:
print("\nFunc with no args called, hence pull first set of {} teams as defined by limit var\n".format(limit))
teams = session.get('/teams?limit={0}&total={1}'.format(limit,total))
if not teams.ok:
return
else:
tj = teams.json()
tjd = tj['teams']
for adict in tjd:
if not adict['name'] == teamteamnm:
continue
elif adict['name'] == teamteamnm:
teamteamid = adict['id']
print("\nFound team..\nFetched",adict['name'], "id: {0}".format(teamteamid))
print("Pull escalation policy for team '{}':'{}'".format(teamteamnm,teamteamid))
esclp = session.get('/escalation_policies?total={0}&team_ids%5B%5D={1}'.format(total,teamteamid))
if not esclp.ok:
print("Pulling Escalation polices for team '{}' failed".format(teamteamnm))
return
else:
ep = esclp.json()
epj = esclp.json()['escalation_policies']
if not epj:
print("Escalation polices for team '{}' not defined".format(teamteamnm))
return
else:
for adict2 in epj:
if not adict2['summary'] == team_esp_name:
continue
else:
print("***************FOUND FOUND********************")
teamesplcyid = adict2['id']
print("\nFetched {} id: {}\n".format(team_esp_name, teamesplcyid))
oncalls = session.get('/oncalls?total={0}&escalation_policy_ids%5B%5D={1}'.format(total,teamesplcyid))
if not oncalls.ok:
print(“issue “with oncalls)
return
else:
ocj = oncalls.json()['oncalls']
for adict3 in ocj:
print("\n")
print(adict3['escalation_level'])
if i['escalation_level'] == 1:
print(adict3['schedule']['summary'], adict3['user']['summary'])
team_oncall_dict[adict3['schedule']['summary']] = adict3['user']['summary']
print(team_oncall_dict)
return team_oncall_dict
if not team_oncall_dict: #part of func def
do foo()
output
foo stuff
sample data is a list of dicts
[{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team Escalation Policy'}, 'escalation_level': 3, 'schedule': None, 'user': {'id': 'PX8XYFT', 'type': 'user_reference', 'summary': 'M1’}, 'start': None, 'end': None},
{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team Escalation Policy'}, 'escalation_level': 1, 'schedule': None, 'user': {'id': 'PKXXVJI', 'type': 'user_reference', 'summary': ‘R1’}, 'start': None, 'end': None},
{'escalation_policy': {'id': 'P8RKTEE', 'type': 'escalation_policy_reference', 'summary': 'Team’}, 'escalation_level': 2, 'schedule': None, 'user': {'d': 'PN8F9PC', 'type': 'user_reference’,'summary': ‘T1’}],'start': None, 'end': None}]
btw: above is 4th inner loop.
so flow is like this diagramatically.
def func1()
team_oncall_dict = {}
loop1
loop2
loop3
loop4
...
team_oncall_dict
if not team_oncall_dict:
print("dict is empty")
output
dict is empty
t was local vs global. fixed it by declaring the team_oncall_dict globally outside the function.
intead of
def func1()
team_oncall_dict = {}
team_oncall_dict = {}
def func1()
I have a list of items that I need to filter based on some conditions. I'm wondering whether Dask could do this filtering in parallel, as the list is very long (a few dozen million records).
Basically, what I need to do is this:
items = [
{'type': 'dog', 'weight': 10},
{'type': 'dog', 'weight': 20},
{'type': 'cat', 'weight': 15},
{'type': 'dog', 'weight': 30},
]
def item_is_valid(item):
item_is_valid = True
if item['type']=='cat':
item_is_valid = False
elif item['weight']>20:
item_is_valid = False
# ...
# elif for n conditions
return item_is_valid
items_filtered = [item for item in items if item_is_valid(item)]
With Dask, what I have achieved to do is the following:
def item_is_valid_v2(item):
"""Return the whole item if valid."""
item_is_valid = True
if item['type']=='cat':
item_is_valid = False
elif item['weight']>20:
item_is_valid = False
# ...
# elif for n conditions
if item_is_valid:
return item
results = []
item = []
for item in items:
delayed = dask.delayed(item_is_valid)(item)
results.append(delayed)
results = dask.compute(*results)
However, the result I get contains a few None values, which then need to be filtered out somehow in a non-parallel way.
({'type': 'dog', 'weight': 10}, {'type': 'dog', 'weight': 20}, None, None)
Perhaps the bag API will work you, this is a rough pseudo-code:
import dask.bag as db
bag = db.from_sequence() # or better yet read it from disk
result = bag.filter(item_is_valid) # note this uses the first version (bool)
To inspect if this is working, inspect the outcome of result.take(5) and if that is satisfactory:
computed_result = result.compute()
I'm working on database that uses lot of data. One invoice could have 7482 different articles. Validating invoice cost so much time, it took 26 minutes to validate one with 7482 articles. I find the method that take time to finish, it is the "action_move_create" inside "odoo\addons\account\models\account_invoice.py".
#api.multi
def action_move_create(self):
""" Creates invoice related analytics and financial move lines """
account_move = self.env['account.move']
for inv in self:
if not inv.journal_id.sequence_id:
raise UserError(_('Please define sequence on the journal related to this invoice.'))
if not inv.invoice_line_ids.filtered(lambda line: line.account_id):
raise UserError(_('Please add at least one invoice line.'))
if inv.move_id:
continue
if not inv.date_invoice:
inv.write({'date_invoice': fields.Date.context_today(self)})
if not inv.date_due:
inv.write({'date_due': inv.date_invoice})
company_currency = inv.company_id.currency_id
# create move lines (one per invoice line + eventual taxes and analytic lines)
iml = inv.invoice_line_move_line_get()
iml += inv.tax_line_move_line_get()
diff_currency = inv.currency_id != company_currency
# create one move line for the total and possibly adjust the other lines amount
total, total_currency, iml = inv.compute_invoice_totals(company_currency, iml)
name = inv.name or ''
if inv.payment_term_id:
totlines = inv.payment_term_id.with_context(currency_id=company_currency.id).compute(total, inv.date_invoice)[0]
res_amount_currency = total_currency
for i, t in enumerate(totlines):
if inv.currency_id != company_currency:
amount_currency = company_currency._convert(t[1], inv.currency_id, inv.company_id, inv._get_currency_rate_date() or fields.Date.today())
else:
amount_currency = False
# last line: add the diff
res_amount_currency -= amount_currency or 0
if i + 1 == len(totlines):
amount_currency += res_amount_currency
iml.append({
'type': 'dest',
'name': name,
'price': t[1],
'account_id': inv.account_id.id,
'date_maturity': t[0],
'amount_currency': diff_currency and amount_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
else:
iml.append({
'type': 'dest',
'name': name,
'price': total,
'account_id': inv.account_id.id,
'date_maturity': inv.date_due,
'amount_currency': diff_currency and total_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
part = self.env['res.partner']._find_accounting_partner(inv.partner_id)
line = [(0, 0, self.line_get_convert(l, part.id)) for l in iml]
line = inv.group_lines(iml, line)
line = inv.finalize_invoice_move_lines(line)
date = inv.date or inv.date_invoice
move_vals = {
'ref': inv.reference,
'line_ids': line,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
move = account_move.create(move_vals)
# Pass invoice in method post: used if you want to get the same
# account move reference when creating the same invoice after a cancelled one:
move.post(invoice = inv)
# make the invoice point to that move
vals = {
'move_id': move.id,
'date': date,
'move_name': move.name,
}
inv.write(vals)
return True
Could you suggest some solutions?
We suppose that the hardware is efficient to run odoo correctly.
I optimize it by using raw sql query. I made these codes in account.invoice model:
The first one is the definition of _mock_create_move_line (called in action_move_create).
def _mock_create_move_line(self, model, values, move):
bad_names = ["analytic_line_ids", "tax_ids", "analytic_tag_ids"]
other_fields = [
"currency_id", "debit", "credit", "balance",
"debit_cash_basis", "credit_cash_basis", "balance_cash_basis",
"company_currency_id", "amount_residual",
"amount_residual_currency", "tax_base_amount", "reconciled",
"company_id", "counterpart"
]
cr = self.env.cr
quote = '"{}"'.format
columns = []
columns1 = []
for i, v in enumerate(values):
v = model._add_missing_default_values(v)
account_id = self.env['account.account'].browse(v['account_id'])
# compulsory columns and some stored related columns
# related fields are not triggered, krrrrr
v.update({
'move_id': move.id,
'date_maturity': move.date,
'company_id': account_id.company_id.id,
'date': move.date,
'journal_id': move.journal_id.id,
'user_type_id': account_id.user_type_id.id,
'create_uid': self.env.uid,
'create_date': fields.Datetime.now()
})
######
temp_column = []
for name, val in sorted(v.items()):
if name in bad_names:
continue
field = model._fields[name]
if field.column_type:
col_val = field.convert_to_column(val, model, v)
temp_column.append(col_val)
if not i:
columns1.append((name, field.column_format, col_val))
columns.append(tuple(temp_column))
model.check_access_rule('create')
try:
query = "INSERT INTO {} ({}) VALUES {} RETURNING id".format(
quote(model._table),
", ".join(quote(name) for name, fmt, val in columns1),
", ".join('%s' for fmt in columns),
)
cr.execute(query, columns)
ids = cr.fetchall()
# clear the model cache to take account of the new insertion
# if not executed, relationnal field will not be updated
model.invalidate_cache()
account_move_line_ids = model.browse(ids)
account_move_line_ids.modified(other_fields)
account_move_line_ids.recompute()
# update parent_path
account_move_line_ids._parent_store_create()
except Exception as e:
_logger.info(e)
cr.rollback()
return
The second one is the overriding of native method action_move_create. I make some modification, call _mock_create_move_line if there is 'raw_sql' in the context.
#api.multi
def action_move_create(self):
""" Creates invoice related analytics and financial move lines """
# TODO : make choice between ORM or raw sql according to the context
account_move = self.env['account.move']
for inv in self:
if not inv.journal_id.sequence_id:
raise UserError(_('Please define sequence on the journal related to this invoice.'))
if not inv.invoice_line_ids.filtered(lambda line: line.account_id):
raise UserError(_('Please add at least one invoice line.'))
if inv.move_id:
continue
if not inv.date_invoice:
inv.write({'date_invoice': fields.Date.context_today(self)})
if not inv.date_due:
inv.write({'date_due': inv.date_invoice})
company_currency = inv.company_id.currency_id
# create move lines (one per invoice line + eventual taxes and analytic lines)
iml = inv.invoice_line_move_line_get()
iml += inv.tax_line_move_line_get()
diff_currency = inv.currency_id != company_currency
# create one move line for the total and possibly adjust the other lines amount
total, total_currency, iml = inv.compute_invoice_totals(company_currency, iml)
name = inv.name or ''
if inv.payment_term_id:
totlines = \
inv.payment_term_id.with_context(currency_id=company_currency.id).compute(total, inv.date_invoice)[0]
res_amount_currency = total_currency
for i, t in enumerate(totlines):
if inv.currency_id != company_currency:
amount_currency = company_currency._convert(t[1], inv.currency_id, inv.company_id,
inv._get_currency_rate_date() or fields.Date.today())
else:
amount_currency = False
# last line: add the diff
res_amount_currency -= amount_currency or 0
if i + 1 == len(totlines):
amount_currency += res_amount_currency
iml.append({
'type': 'dest',
'name': name,
'price': t[1],
'account_id': inv.account_id.id,
'date_maturity': t[0],
'amount_currency': diff_currency and amount_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
else:
iml.append({
'type': 'dest',
'name': name,
'price': total,
'account_id': inv.account_id.id,
'date_maturity': inv.date_due,
'amount_currency': diff_currency and total_currency,
'currency_id': diff_currency and inv.currency_id.id,
'invoice_id': inv.id
})
part = self.env['res.partner']._find_accounting_partner(inv.partner_id)
line = [(0, 0, self.line_get_convert(l, part.id)) for l in iml]
line = inv.group_lines(iml, line)
line = inv.finalize_invoice_move_lines(line)
date = inv.date or inv.date_invoice
if self.env.context.get('raw_sql', None):
move_vals = {
'ref': inv.reference,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
# remove (0, 0, ...)
# override the group_lines method to avoid looping on next instruction
new_lines = [nl[2] for nl in line]
# TODO do not call compute here, add with ...norecompute()
move = account_move.create(move_vals)
move.env.cr.commit()
self._mock_create_move_line(self.env['account.move.line'], new_lines, move)
# Pass invoice in method post: used if you want to get the same
# account move reference when creating the same invoice after a cancelled one:
# compute move, it is not triggered automatically bc raw sql insertion
# is it correct to call it like this ? find better way
move._amount_compute()
move._compute_partner_id()
move._compute_matched_percentage()
else:
# make default behavior
move_vals = {
'ref': inv.reference,
'line_ids': line,
'journal_id': inv.journal_id.id,
'date': date,
'narration': inv.comment,
}
move = account_move.create(move_vals)
move.post(invoice=inv)
# make the invoice point to that move
vals = {
'move_id': move.id,
'date': date,
'move_name': move.name,
}
inv.write(vals)
return True
Now, the execution time is less than 1 minutes for about 7000 records to insert inside invoice.move.line
Im trying to append additional info to an existing list but i received an error message instead.
Error: 4.Invalid embedded document instance provided to an
EmbeddedDocumentField: ['family']
class Family(db.EmbeddedDocument):
name = db.StringField()
# gender = db.StringField()
class House(db.Document):
house_id = db.IntField(required=True, unique=True)
housingType = db.StringField(required=True)
family = db.EmbeddedDocumentListField(Family)
def to_json(self):
return {
"house_id": self.house_id,
"housingType": self.housingType,
"family_members": self.family
}
#app.route('/api/add_family/<h_id>', methods=['POST'])
def add_family(h_id):
content = request.json
h = House.objects(house_id=h_id).get()
h.family.append(content['family'])
h.save()
return make_response("Added family member successfully", 201)
What im trying to achieve is as follows:
Current data:
{
'house_id': 1,
'family': [{'name': 'John', 'Gender': 'Male'}]
}
After appending, it should look like this:
{
'house_id': 1,
'family': [{'name': 'John, 'Gender': 'Male'}, {'name': 'Peter', 'Gender': 'Male'}]
}
Here is my solution. Hopefully it helps.
#app.route('/api/add_family/<h_id>', methods=['POST'])
def add_family(h_id):
'''
family member is added only if its not already in the database
'''
edited = False
content = request.json
h = House.objects.get(house_id=h_id).to_json()
h = json.loads(h)
family_arr = h['family']
if family_arr:
# family_arr not empty
count = family_arr[-1].get('id') + 1
else:
count = 1
for new_name in content['family']:
if not dup_name_check(family_arr, new_name['name']):
new_name.update({'id': count})
family_arr.append(new_name)
count += 1
edited = True
if edited:
House.objects.get(house_id=h_id).update(family=family_arr)
return make_response(f"Successfully added family member in House ID:{h_id}", 201)
else:
return make_response(f"Duplicated entries detected!", 400)
Sample json is below. I want to save id which is completed (False and True) into seperated dictionaryies
todos = [{'userId': 1, 'id': 1, 'title': 'delectus aut autem', 'completed': False},
{'userId': 1, 'id': 2, 'title': 'quis ut nam facil ', 'completed': False},
{'userId': 1, 'id': 1, 'title': 'fugiat veniam minus', 'completed': False},
{'userId': 1, 'id': 2, 'title': 'et porro tempora', 'completed': True},
{'userId': 1, 'id': 1,'title': 'laprovident illum', 'completed': False}]
Expected out is below
todos_by_user_true = {1:0,2:1}
todos_by_user_false = {1:3,2:1}
code is below? Why my code not working. I am getting blank dictionary
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
if todo["completed"]==True:
try:
# Increment the existing user's count.
todos_by_user_true[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_true[todo["id"]] = 0
elif todo["completed"]==False:
try:
# Increment the existing user's count.
todos_by_user_false[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_false[todo["id"]] = 0
I am getting not proper dictionary
My output is below
todos_by_user_false {1: 2, 2: 0}
todos_by_user_true {2: 0}
Disclaimer: I need to take care of exception also
Looking at your input data, it is so that:
userId 1, id 1 has 0 true, and 3 false
userId 1, id 2 has 1 true, and 1 false
Given the required output, it looks like you really want to use id rather than userId in your lookups. Besides that, there's an issue with accounting the first time you insert the id in the resulting dictionary. I would fix it like this:
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
if todo["completed"]==True:
try:
# Increment the existing user's count.
todos_by_user_true[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_true[todo["id"]] = 1
elif todo["completed"]==False:
try:
# Increment the existing user's count.
todos_by_user_false[todo["id"]] += 1
except KeyError:
# This user has not been seen. Set their count to 1.
todos_by_user_false[todo["id"]] = 1
which (btw) is already what's in your comments.
Personally, I would check the dictionary for the key before insertion, instead of using try..except, like this:
todos_by_user_true = {}
todos_by_user_false = {}
# Increment complete TODOs count for each user.
for todo in todos:
key = todo["id"]
if todo["completed"]: # true case
# If `id` not there yet, insert it to 0
if key not in todos_by_user_true:
todos_by_user_true[key] = 0
# increment
todos_by_user_true[key] += 1
else: # false case
# If `id` not there yet, insert it to 0
if key not in todos_by_user_false:
todos_by_user_false[key] = 0
# increment
todos_by_user_false[key] += 1
This gives out:
todos_by_user_true = {2:1}
todos_by_user_false = {1:3,2:1}
The logic being this, you cannot have:
todos_by_user_true = {1:0}
You account for the value when you find it; rather than iterating for id from a separate list.