We Keep Coding

Replace dot product for loop Numpy

I am trying to replace the dot product for loop using something faster like NumPy
I did research on dot product and kind of understand and can get it working with toy data in a few ways in but not 100% when it comes to implementing it for actual use with a data frame.
I looked at these and other SO threads to no luck avoide loop dot product, matlab and dot product subarrays without for loop and multiple numpy dot products without a loop
looking to do something like this which works with toy numbers in np array
u1 =np.array([1,2,3])
u2 =np.array([2,3,4])
v1.dot(v2)
20
u1 =np.array([1,2,3])
u2 =np.array([2,3,4])
(u1 * u2).sum()
20
u1 =np.array([1,2,3])
u2 =np.array([2,3,4])
sum([x1*x2 for x1, x2 in zip (u1, u2)])
20
this is the current working get dot product
I would like to do this with out the for loop
def get_dot_product(self, courseid1, courseid2, unit_vectors):
u1 = unit_vectors[courseid1]
u2 = unit_vectors[courseid2]
dot_product = 0.0
for dimension in u1:
if dimension in u2:
dot_product += u1[dimension] * u2[dimension]
return dot_product
** code**
#!/usr/bin/env python
# coding: utf-8
class SearchRecommendationSystem:
def __init__(self):
pass
def get_bag_of_words(self, titles_lines):
bag_of_words = {}
for index, row in titles_lines.iterrows():
courseid, course_bag_of_words = self.get_course_bag_of_words(row)
for word in course_bag_of_words:
word = str(word).strip() # added
if word not in bag_of_words:
bag_of_words[word] = course_bag_of_words[word]
else:
bag_of_words[word] += course_bag_of_words[word]
return bag_of_words
def get_course_bag_of_words(self, line):
course_bag_of_words = {}
courseid = line['courseid']
title = line['title'].lower()
description = line['description'].lower()
wordlist = title.split() + description.split()
if len(wordlist) >= 10:
for word in wordlist:
word = str(word).strip() # added
if word not in course_bag_of_words:
course_bag_of_words[word] = 1
else:
course_bag_of_words[word] += 1
return courseid, course_bag_of_words
def get_sorted_results(self, d):
kv_list = d.items()
vk_list = []
for kv in kv_list:
k, v = kv
vk = v, k
vk_list.append(vk)
vk_list.sort()
vk_list.reverse()
k_list = []
for vk in vk_list[:10]:
v, k = vk
k_list.append(k)
return k_list
def get_keywords(self, titles_lines, bag_of_words):
n = sum(bag_of_words.values())
keywords = {}
for index, row in titles_lines.iterrows():
courseid, course_bag_of_words = self.get_course_bag_of_words(row)
term_importance = {}
for word in course_bag_of_words:
word = str(word).strip() # extra
tf_course = (float(course_bag_of_words[word]) / sum(course_bag_of_words.values()))
tf_overall = float(bag_of_words[word]) / n
term_importance[word] = tf_course / tf_overall
keywords[str(courseid)] = self.get_sorted_results(term_importance)
return keywords
def get_inverted_index(self, keywords):
inverted_index = {}
for courseid in keywords:
for keyword in keywords[courseid]:
if keyword not in inverted_index:
keyword = str(keyword).strip() # added
inverted_index[keyword] = []
inverted_index[keyword].append(courseid)
return inverted_index
def get_search_results(self, query_terms, keywords, inverted_index):
search_results = {}
for term in query_terms:
term = str(term).strip()
if term in inverted_index:
for courseid in inverted_index[term]:
if courseid not in search_results:
search_results[courseid] = 0.0
search_results[courseid] += (
1 / float(keywords[courseid].index(term) + 1) *
1 / float(query_terms.index(term) + 1)
)
sorted_results = self.get_sorted_results(search_results)
return sorted_results
def get_titles(self, titles_lines):
titles = {}
for index, row in titles_lines.iterrows():
titles[row['courseid']] = row['title'][:60]
return titles
def get_unit_vectors(self, keywords, categories_lines):
norm = 1.884
cat = {}
subcat = {}
for line in categories_lines[1:]:
courseid_, category, subcategory = line.split('\t')
cat[courseid_] = category.strip()
subcat[courseid_] = subcategory.strip()
unit_vectors = {}
for courseid in keywords:
u = {}
if courseid in cat:
u[cat[courseid]] = 1 / norm
u[subcat[courseid]] = 1 / norm
for keyword in keywords[courseid]:
u[keyword] = (1 / float(keywords[courseid].index(keyword) + 1) / norm)
unit_vectors[courseid] = u
return unit_vectors
def get_dot_product(self, courseid1, courseid2, unit_vectors):
u1 = unit_vectors[courseid1]
u2 = unit_vectors[courseid2]
dot_product = 0.0
for dimension in u1:
if dimension in u2:
dot_product += u1[dimension] * u2[dimension]
return dot_product
def get_recommendation_results(self, seed_courseid, keywords, inverted_index, unit_vectors):
courseids = []
seed_courseid = str(seed_courseid).strip()
for keyword in keywords[seed_courseid]:
for courseid in inverted_index[keyword]:
if courseid not in courseids and courseid != seed_courseid:
courseids.append(courseid)
dot_products = {}
for courseid in courseids:
dot_products[courseid] = self.get_dot_product(seed_courseid, courseid, unit_vectors)
sorted_results = self.get_sorted_results(dot_products)
return sorted_results
def Final(self):
print("Reading Title file.......")
titles_lines = open('s2-titles.txt', encoding="utf8").readlines()
print("Reading Category file.......")
categories_lines = open('s2-categories.tsv', encoding = "utf8").readlines()
print("Getting Supported Functions Data")
bag_of_words = self.get_bag_of_words(titles_lines)
keywords = self.get_keywords(titles_lines, bag_of_words)
inverted_index = self.get_inverted_index(keywords)
titles = self.get_titles(titles_lines)
print("Getting Unit Vectors")
unit_vectors = self.get_unit_vectors(keywords=keywords, categories_lines=categories_lines)
#Search Part
print("\n ############# Started Search Query System ############# \n")
query = input('Input your search query: ')
while query != '':
query_terms = query.split()
search_sorted_results = self.get_search_results(query_terms, keywords, inverted_index)
print(f"==> search results for query: {query.split()}")
for search_result in search_sorted_results:
print(f"{search_result.strip()} - {str(titles[search_result]).strip()}")
#ask again for query or quit the while loop if no query is given
query = input('Input your search query [hit return to finish]: ')
print("\n ############# Started Recommendation Algorithm System ############# \n")
# Recommendation ALgorithm Part
seed_courseid = (input('Input your seed courseid: '))
while seed_courseid != '':
seed_courseid = str(seed_courseid).strip()
recom_sorted_results = self.get_recommendation_results(seed_courseid, keywords, inverted_index, unit_vectors)
print('==> recommendation results:')
for rec_result in recom_sorted_results:
print(f"{rec_result.strip()} - {str(titles[rec_result]).strip()}")
get_dot_product_ = self.get_dot_product(seed_courseid, str(rec_result).strip(), unit_vectors)
print(f"Dot Product Value: {get_dot_product_}")
seed_courseid = (input('Input seed courseid [hit return to finish]:'))
if __name__ == '__main__':
obj = SearchRecommendationSystem()
obj.Final()
s2-categories.tsv
courseid category subcategory
21526 Design 3D & Animation
153082 Marketing Advertising
225436 Marketing Affiliate Marketing
19482 Office Productivity Apple
33883 Office Productivity Apple
59526 IT & Software Operating Systems
29219 Personal Development Career Development
35057 Personal Development Career Development
40751 Personal Development Career Development
65210 Personal Development Career Development
234414 Personal Development Career Development
Example of how s2-titles.txt looks
courseidXXXYYYZZZtitleXXXYYYZZZdescription
3586XXXYYYZZZLearning Tools for Mrs B's Science Classes This is a series of lessons that will introduce students to the learning tools that will be utilized throughout the schoXXXYYYZZZThis is a series of lessons that will introduce students to the learning tools that will be utilized throughout the school year The use of these tools serves multiple purposes 1 Allow the teacher to give immediate and meaningful feedback on work that is in progress 2 Allow students to have access to content and materials when outside the classroom 3 Provide a variety of methods for students to experience learning materials 4 Provide a variety of methods for students to demonstrate learning 5 Allow for more time sensitive correction grading and reflections on concepts that are assessed

Evidently unit_vectors is a dictionary, from which you extract to 2 values, u1 and u2.
But what are those? Evidently dicts as well (this iteration would not make sense with a list):
for dimension in u1:
if dimension in u2:
dot_product += u1[dimension] * u2[dimension]
But what is u1[dimension]? A list? An array.
Normally dict are access by key as you do here. There isn't a numpy style "vectorization". vals = list(u1.values()) gets a lists of all values, and conceivably that could be made into an array (if the elements are right)
arr1 = np.array(list(u1.values()))
and a np.dot(arr1, arr2) might work
You'll get the best answers if you give small concrete examples - with real working data (and skip the complex generating code). Focus on the core of the problem, so we can grasp the issue with a 30 second read!
===
Looking more in depth at your dot function; this replicates the core (I think). Initially I missed the fact that you aren't iterating on u2 keys, but rather seeking matching ones.
def foo(dd):
x = 0
u1 = dd['u1']
u2 = dd['u2']
for k in u1:
if k in u2:
x += u1[k]*u2[k]
return x
Then making a dictionary of dictionaries:
In [30]: keys=list('abcde'); values=[1,2,3,4,5]
In [31]: adict = {k:v for k,v in zip(keys,values)}
In [32]: dd = {'u1':adict, 'u2':adict}
In [41]: dd
Out[41]:
{'u1': {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5},
'u2': {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}}
In [42]: foo(dd)
Out[42]: 55
In this case the subdictionaries match, so we get the same value with a simple array dot:
In [43]: np.dot(values,values)
Out[43]: 55
But if u2 was different, with different key/value pairs, and possibly different keys the result will be different. I don't see a way around the iterative access by keys. The sum-of-products part of the job is minor compared to the dictionary access.
In [44]: dd['u2'] = {'e':3, 'f':4, 'a':3}
In [45]: foo(dd)
Out[45]: 18
We could construct other data structures that are more suitable to a fast dot like calculation. But that's another topic.

Modified method
def get_dot_product(self, courseid1, courseid2, unit_vectors):
# u1 = unit_vectors[courseid1]
# u2 = unit_vectors[courseid2]
# dimensions = set(u1).intersection(set(u2))
# dot_product = sum(u1[dimension] * u2.get(dimension, 0) for dimension in dimensions)
u1 = unit_vectors[courseid1]
u2 = unit_vectors[courseid2]
dot_product = sum(u1[dimension] * u2.get(dimension, 0) for dimension in u2)
return dot_product

How to get big amount of data as fast as possible

I am trying to return an array of constructed objects that are build on top of objects that I retrieve from some url plus another fields that I get from another url.
I have an array that consists of two arrays that each has about 8000 objects...
I have tried to make each object construction as a thread however it still takes a lot of time...
Any solution? Here is my code:
def get_all_players_full_data(ea_players_json):
all = []
ea_players_json = list(ea_players_json.values())
for i in range(len(ea_players_json)):
for player_obj in ea_players_json[i]:
all.append(player_obj)
for player_obj in range(len(all)):
all_data = []
with concurrent.futures.ThreadPoolExecutor(len(all)) as executor:
for player_data in all:
future = executor.submit(build_full_player_data_obj, player_data)
print(future.result())
all_data.append(future.result())
def build_full_player_data_obj(ea_player_data):
if ea_player_data.get("c") is not None:
player_full_name = ea_player_data.get("c")
else:
player_full_name = ea_player_data.get("f") + " " + ea_player_data.get("l")
player_id = ea_player_data.get("id")
# go to futhead to find all cards of that player
futhead_url_player_data = f'{FUTHEAD_PLAYER}{player_full_name}'
details_of_specific_player = json.loads(requests.get(futhead_url_player_data).content)
cards_from_the_same_id = []
for player_in_json_futhead in details_of_specific_player:
if player_in_json_futhead["player_id"] == player_id:
rating = player_in_json_futhead["rating"]
specific_card_id = player_in_json_futhead["def_id"]
revision = player_in_json_futhead["revision_type"]
name = player_in_json_futhead["full_name"]
nation = player_in_json_futhead["nation_name"]
position = player_in_json_futhead["position"]
club = player_in_json_futhead["club_name"]
cards_from_the_same_id.append(Player(specific_card_id, name, rating, revision, nation,
position, club))
return cards_from_the_same_id

Split Purchase Order according to Product Category

I wanna split the Purchase Order according to Product Category.
My Code so far:
_inherit ='purchase.order.line'
split = fields.Boolean(string='Split')
_inherit ='purchase.order'
def btn_split_rfq(self):
flag = []
for record in self:
if record.order_line:
for rec in record.order_line:
rec.split = True # oles tis eggrafes true
flag.append(rec.product_id.categ_id.id) # lista me ta categ ids
newlist=[ii for n,ii in enumerate(flag) if ii not in flag[:n]] # ta krata mono mia fora an uparxoun polles
for index in newlist: # gia 2 katigories 8a treksi 2 fores
quotation_id = self.copy()
for index in record.order_line:
if index.split:
self.env['purchase.order.line'].browse(index.id).unlink()
else:
raise ValidationError(_('Please Select Order Line To Split'))
The code so far, is split to multiple POs e.g. if i have 2 type of categories is making 2 POs but and the two POs is taking and the 4 products not only of product category(see image below).
Output:
But i want this kind of Output:
Any solution?

I tried to just ignore your code example, because it is difficult to understand for me. If you want try out my attempt:
def button_split_by_prod_categ(self):
self.ensure_one()
groups = {}
# group lines by product category
for line in self.order_line:
if line.product_id.categ_id not in groups:
groups[line.product_id.categ_id] = line
else:
groups[line.product_id.categ_id] =| line
skip = True
orders = self
for lines in groups.values():
# skip first group
if skip:
skip = False
continue
# or create a new order without lines and connect
# the group's lines with it
else:
default_values = {'order_line': []}
new_order = self.copy(default=default_values)
lines.write({'order_id': new_order.id})
orders |= new_order
# now you could return a list view with all orders
# or just do 'nothing'
return

I found solution to my problem, i dont think is pretty but it does the job. Thanks # CZoellner and #Charif DZ for the effort!!!
def btn_split_rfq(self):
flag =[]
for record in self:
if record.order_line:
for rec in record.order_line: #run for all products on purchase order
flag.append(rec.product_id.categ_id.id) # append product category ids
categ_ids=[ii for n,ii in enumerate(flag) if ii not in flag[:n]] # filter list,keep only one time every product category id
categ_ids.sort() # sorting list
for index in categ_ids: # will run 2 times if there is 2 product categories
quotations_ids = [self.copy()]
for order_line in quotations_ids:
prods = self.env['purchase.order.line'].search([('product_categ_id' ,'!=',index),('order_id','=',int(order_line))])
for ids in prods:
self.env['purchase.order.line'].browse(ids.id).unlink()
else:
raise ValidationError(_('Not Available Purchase Order Lines'))

How do I improve my 'for' loop performance in terms of speed?

I have a for loop which it is taking almost 20 seconds for iterating 6907 rows. That loop does the job of making a list of unique region names in the given queryset.
I have placed timestamps at various places in the code to record the timings. The 'for' loop which is taking more time is between variables 't3' and 't4'.
timestamps
t = 12:27:22:169533
t2 = 12:27:22:173535
t3 = 12:27:22:793567
6907
t4 = 12:27:42:907362
t5 = 12:27:43:242596
t6 = 12:27:43:242596
6907 is the length of my queryset sales_data
views.py
class MSZoneProduct(generic.TemplateView):
template_name = 'sales/MSZoneProduct.html'
form_class = MSZoneProductForm
def get(self, request, *args, **kwargs):
if request.user.is_authenticated:
form = self.form_class(request.GET)
context = {'form': form}
if form.is_valid():
zone_code_ = form.cleaned_data['zone_name']
product_code_ = form.cleaned_data['product_name']
t = datetime.now().strftime('%H:%M:%S:%f')
print("t = " + t)
product = Product.objects.get(product_code=product_code_)
t2 = datetime.now().strftime('%H:%M:%S:%f')
print("t2 = " + t2)
sales_data = Sales.objects.filter(zone_code=zone_code_, product_code=product).select_related()
t3 = datetime.now().strftime('%H:%M:%S:%f')
print("t3 = " + t3)
print(len(sales_data))
regions = []
message = ""
regions_dict = {}
for x in sales_data:
if x.region_name not in regions:
regions.append(x.region_name)
else:
continue
t4 = datetime.now().strftime('%H:%M:%S:%f')
print("t4 = " + t4)
for x in regions:
sum_ = 0
for y in sales_data:
if y.region_name == x:
sum_ = sum_ + y.quantity
regions_dict[x] = sum_
t5 = datetime.now().strftime('%H:%M:%S:%f')
print("t5 = " + t5)
if len(regions) == 0:
message = "There is no data available for this product in this particular region."
context = {'form': form, 'message': message, 'data': regions_dict}
t6 = datetime.now().strftime('%H:%M:%S:%f')
print("t6 = " + t6)
return render(request, 'sales/MSZoneProduct.html', context)
return render(request, 'sales/MSZoneProduct.html', context)
else:
return redirect('/sales/')

Based on your view, you want to sum up the quantities per region. We can move all this logic into a database query. This will not only make it more efficient, but more elegant as well:
from django.db.models import Sum
sales_data = Sales.objects.filter(
zone_code=zone_code_, product_code=product
).values('region_name').annotate(
total_quantity=Sum('quantity')
).order_by('region_name')
This will result in a QuerySet that contains dictionaries with two elements: 'region_name' that maps to the name of the region, and total_quantity that sums up the quantity for that region.
Next we can convert it to a dictionary regions_dict with:
regions_dict = { r['region_name']: r['total_quantity'] for r in sales_data }

The entire code from sales_data = ... to t5 = ... can be much more efficiently done with dict comprehension and itertools.groupby:
from itertools import groupby
from operator import itemgetter
regions_dict = {k: sum(map(itemgetter('quantity'), g)) for k, g in groupby(Sales.objects.filter(zone_code=zone_code_, product_code=product).order_by('region_name').values('region_name', 'quantity'), itemgetter('region_name'))}

Use a set constructed with list comprehensions?
regions = set(x.region_name for x in sales_data)
The only problem is if you need the items in order. They could be sorted after the fact, your you could append them to a list depending on if they are successfully added to the set.
Edit: I agree that doing it in the database is the better approach (#Willem Van Onsem's answer), but this answer will be relevant for anyone who isn't using Django.

Improving the for
It is possible to directly store the sums at the first go in the dictionary with complexity O(N)
regions_dict_sums = {x.region_name:0 for x in sales_data}
for x in sales_data:
regions_dict_sums[x.region_name]+=x.quantity
So the total regions where sales are greater than zero will be hashed in the dictionary as keys and their values are the total sales for that region, accessible at constant time.

Odoo 8 Allow Negative Invoice Line Item

I have a product called "Coupon" with negative amount which is used to offset the product price. However, it seems like Odoo 8 does not allow computation of negative amount to price_subtotal (it becomes 0.00):
Coupon ... ... 1 Each -40.0000 0.0000
When I remove the negative sign, it computes
Coupon ... ... 1 Each 40.0000 40.0000
From an accounting perspective, the total invoice should not be negative. That stays true. However, I do need to allow negative computation of invoice line item(s). Where and what do I need to change? I tried looking into account/account.py but to no avail so far - it's all just "tax" related.
Thanks in advance!
Details of the amount column for the line total
class account_invoice(models.Model)
....
#api.one
#api.depends('invoice_line.price_subtotal', 'tax_line.amount')
def _compute_amount(self):
self.amount_untaxed = sum(line.price_subtotal for line in self.invoice_line)
self.amount_tax = sum(line.amount for line in self.tax_line)
self.amount_total = self.amount_untaxed + self.amount_tax
....
class account_invoice_line(models.Model):
_name = "account.invoice.line"
_description = "Invoice Line"
_order = "invoice_id,sequence,id"
#api.one
#api.depends('price_unit', 'discount', 'invoice_line_tax_id', 'quantity',
'product_id', 'invoice_id.partner_id', 'invoice_id.currency_id')
def _compute_price(self):
price = self.price_unit * (1 - (self.discount or 0.0) / 100.0)
taxes = self.invoice_line_tax_id.compute_all(price, self.quantity, product=self.product_id, partner=self.invoice_id.partner_id)
self.price_subtotal = taxes['total']
if self.invoice_id:
self.price_subtotal = self.invoice_id.currency_id.round(self.price_subtotal)
#api.model
def _default_price_unit(self):
if not self._context.get('check_total'):
return 0
total = self._context['check_total']
for l in self._context.get('invoice_line', []):
if isinstance(l, (list, tuple)) and len(l) >= 3 and l[2]:
vals = l[2]
price = vals.get('price_unit', 0) * (1 - vals.get('discount', 0) / 100.0)
total = total - (price * vals.get('quantity'))
taxes = vals.get('invoice_line_tax_id')
if taxes and len(taxes[0]) >= 3 and taxes[0][2]:
taxes = self.env['account.tax'].browse(taxes[0][2])
tax_res = taxes.compute_all(price, vals.get('quantity'),
product=vals.get('product_id'), partner=self._context.get('partner_id'))
for tax in tax_res['taxes']:
total = total - tax['amount']
return total

Odoo's default behaviour is handling it as expected. The problem is custom code. (For more information read the questions comments)