So I have a set of tests where I'd like to test multiple versions of a solution. Currently I have
import pytest
import product_not_at_index
functions_to_test = [
product_not_at_index.product_not_at_index_n_squared,
product_not_at_index.product_not_at_index,
]
def run_test(function_input, expected_result, test_func):
actual_result = test_func(function_input)
assert actual_result == expected_result
#pytest.mark.parametrize("test_func", functions_to_test)
def test_empty_list(test_func):
input_data = []
expected_result = []
run_test(input_data, expected_result, test_func)
#pytest.mark.parametrize("test_func", functions_to_test)
def test_single_item(test_func):
input_data = [1]
expected_result = [1]
run_test(input_data, expected_result, test_func)
#pytest.mark.parametrize("test_func", functions_to_test)
def test_one_times_one(test_func):
input_data = [1, 1]
expected_result = [1, 1]
run_test(input_data, expected_result, test_func)
#pytest.mark.parametrize("test_func", functions_to_test)
def test_normal_use_case(test_func):
input_data = [1, 7, 3, 4]
expected_result = [84, 12, 28, 21]
run_test(input_data, expected_result, test_func)
And this works great. But looking at my solution I see that all of my tests have the same basic set of code. How can I parameterize a function twice so that I can just have a single test function and stop repeating myself?
I thought that I could do something like
import pytest
import product_not_at_index
functions_to_test = [product_not_at_index.product_not_at_index_n_squared]
test_data = [
[], [],
[1], [1],
[1, 1], [1, 1],
[1, 7, 3, 4], [84, 12, 28, 21],
]
#pytest.mark.parametrize("function_input,expected_result", test_data)
#pytest.mark.parametrize("test_func", functions_to_test)
def test_run(function_input, expected_result, test_func):
actual_result = test_func(function_input)
assert actual_result == expected_result
but that just returns this error
E assert 0 == 2
E + where 0 = len([])
E + and 2 = len(['function_input', 'expected_result'])
The solution I ended up using is this one
import pytest
import product_not_at_index
functions_to_test = [product_not_at_index.product_not_at_index_n_squared]
test_data = [
([], []),
([1], [1]),
([1, 1], [1, 1]),
([1, 7, 3, 4], [84, 12, 28, 21]),
]
# TODO: turn into a list comprehension.
test_paramaters = []
for func in functions_to_test:
for test_input, expected_result in test_data:
test_paramaters.append([test_input, expected_result, func])
#pytest.mark.parametrize("function_input,expected_result,test_func", test_paramaters)
def test_run(function_input, expected_result, test_func):
actual_result = test_func(function_input)
assert actual_result == expected_result
Related
I am trying to run a user-generated simple code (two sum) inside AWS lambda python runtime Runtime.PYTHON_3_9.
def handler(event, context):
a = {
"code": "def twoSum(nums, target):\n hash_map = {}\n\n for i, num in enumerate(nums):\n if target - num in hash_map:\n return list(sorted([i, hash_map[target - num]]))\n hash_map[num] = i\n\ndef solve():\r\n testcases = [[[2, 7, 11, 15], 9]]\r\n output = [[0, 1]]\r\n\r\n total = len(testcases)\r\n correct = 0\r\n\r\n for i in range(len(testcases)):\r\n result = twoSum(*testcases[i])\r\n\r\n if result == output[i]:\r\n correct += 1\r\n\r\n print(correct, total)\r\n\r\n\r\nsolve()\r\n"
}
code = a["code"]
print(code)
exec(code)
This code prints
def twoSum(nums, target):
hash_map = {}
for i, num in enumerate(nums):
if target - num in hash_map:
return list(sorted([i, hash_map[target - num]]))
hash_map[num] = i
def solve():
testcases = [[[2, 7, 11, 15], 9]]
output = [[0, 1]]
total = len(testcases)
correct = 0
for i in range(len(testcases)):
result = twoSum(*testcases[i])
if result == output[i]:
correct += 1
print(correct, total)
solve()
while the actual code is
def twoSum(nums, target):
hash_map = {}
for i, num in enumerate(nums):
if target - num in hash_map:
return list(sorted([i, hash_map[target - num]]))
hash_map[num] = i
def solve():
testcases = [[[2, 7, 11, 15], 9]]
output = [[0, 1]]
total = len(testcases)
correct = 0
for i in range(len(testcases)):
result = twoSum(*testcases[i])
if result == output[i]:
correct += 1
print(correct, total)
solve()
All the spaces inside the exec strings are getting removed for some reason. Tried the same code on local Windows and macOS. It ran without any issues.
And this is the error I received
{
"errorMessage": "name 'twoSum' is not defined",
"errorType": "NameError",
"requestId": "<some-request-id>",
"stackTrace": [
" File \"/var/task/python.py\", line 10, in handler\n exec(code)\n",
" File \"<string>\", line 25, in <module>\n",
" File \"<string>\", line 17, in solve\n"
]
}
Is there any way to resolve this and make the exec work?
I'm trying to do multiprocessing with python but I have duplicated values in results files.
Could you please help me to solve that?
here is my code:
import itertools
from multiprocessing import Pool
from multiprocessing import Manager
import pandas as pd
PARAMS = {}
LPT_LIMIT = [4, 6, 8, 10]
HPT_LIMIT = [1.6, 1.8, 2.0]
NB_FLIGHT = [10, 30]
LPT_EXCEEDENCE = [1, 4]
HPT_EXCEEDENCE = [3, 4]
tmp = [LPT_LIMIT, HPT_LIMIT, NB_FLIGHT, LPT_EXCEEDENCE, HPT_EXCEEDENCE]
parameters = list(itertools.product(*tmp))
def toto(param):
PARAMS['LPT_LMIT'] = param[0]
PARAMS['HPT_LMIT'] = param[1]
PARAMS['NB_FLIGHT'] = param[2]
PARAMS['LPT_EXCEEDENCE'] = param[3]
PARAMS['HPT_EXCEEDENCE'] = param[4]
return PARAMS
if __name__=='__main__':
pool = Pool()
manager = Manager()
my_list = manager.list()
my_list.append(pool.map(toto, parameters))
flat_list = [item for sublist in my_list for item in sublist]
pd.DataFrame(flat_list).to_excel('results.xlsx', index=False)
the results is that I have only value of 4 in HPT_EXCEEDENCE (please see attached file)
enter image description here
I have only HPT_EXCEEDENCE = 4 but HPT_EXCEEDENCE is 3 or 4
So I don't know what's wrong with my code
Unless you are doing something more complex, you don't need Manager(). The problem specifically was the location of PARAMS = {}. See updated code below. This seems to get the result you want.
import itertools
from multiprocessing import Pool
from multiprocessing import Manager
import pandas as pd
LPT_LIMIT = [4, 6, 8, 10]
HPT_LIMIT = [1.6, 1.8, 2.0]
NB_FLIGHT = [10, 30]
LPT_EXCEEDENCE = [1, 4]
HPT_EXCEEDENCE = [3, 4]
tmp = [LPT_LIMIT, HPT_LIMIT, NB_FLIGHT, LPT_EXCEEDENCE, HPT_EXCEEDENCE]
parameters = list(itertools.product(*tmp))
def toto(param):
PARAMS = {}
PARAMS['LPT_LMIT'] = param[0]
PARAMS['HPT_LMIT'] = param[1]
PARAMS['NB_FLIGHT'] = param[2]
PARAMS['LPT_EXCEEDENCE'] = param[3]
PARAMS['HPT_EXCEEDENCE'] = param[4]
return PARAMS
if __name__=='__main__':
pool = Pool()
my_list = pool.map(toto, parameters)
pd.DataFrame(my_list).to_excel('results1.xlsx', index=False)
I want to realize the bert model.
So I built a class with __getitem__ in it.
I can print something like test[0], but when I assign a value, like data = test[0], a KeyError occurs.
import random
"""
corpus_file = 'vocab'
vocab_size = 6
vocab_freq = 1
save_path = 'obj/'
max_sentence = 16
corpus -> org_line -> ope_line
corpus -> org_line -> token_list -> idx_to_token + token_to_idx
"""
class vocab():
def __init__(self, corpus_file, vocab_size, vocab_freq,save_path,max_sentence):
self.max_sentence = max_sentence
self.special_labels = ['PAD', 'UNK', 'SEP', 'CLS', 'MASK']
# output
self.data = []
self.idx_to_token = []
self.token_to_idx = {}
# ope
self.pre_ope(corpus_file,vocab_size,vocab_freq)
#self.save_data(save_path)
#self.print_data()
def pre_ope(self,corpus_file,vocab_size,vocab_freq):
token_list = {}
with open(corpus_file, 'r') as f:
while 1:
new_org_line = f.readline()
if new_org_line != '':
new_org_line = new_org_line.strip('\n')
new_sentence = new_org_line.split('\t')
sentence = []
for tmp in new_sentence:
token_sentence = tmp.split()
sentence.append(token_sentence)
for token in token_sentence:
if token_list.get(token):
token_list[token] += 1
else:
new_token = {token: 1}
token_list.update(new_token)
self.data.append(sentence)
else:
break
f.close()
token_list = sorted(token_list.items(), key=lambda i: (-i[1], i[0]))
self.build_dictionary(token_list,vocab_freq,vocab_size)
'''
Special labels:
PAD
UNK
SEP sentence separator
CLS classifier token
MASK
'''
def build_dictionary(self,token_list,vocab_freq,vocab_size):
for idx, label in enumerate(self.special_labels):
self.idx_to_token.append(label)
self.token_to_idx[label] = idx
for idx, (token, freq) in enumerate(token_list):
if freq >= vocab_freq :
self.idx_to_token.append(token)
self.token_to_idx[token] = idx + len(self.special_labels)
if len(self.idx_to_token) >= vocab_size + len(self.special_labels) and vocab_size != 0 :
break
def __len__(self):
return len(self.data)
def print_data(self):
print(self.data)
print(self.idx_to_token)
print(self.token_to_idx)
def __getitem__(self, item):
s1,s2,is_next_sentence = self.get_random_next_sentence(item)
s1,s1_label = self.get_random_sentence(s1)
s2,s2_label = self.get_random_sentence(s2)
sentence = [self.token_to_idx['CLS']] +s1 +[self.token_to_idx['SEP']] +s2 +[self.token_to_idx['SEP']]
label = [-1] +s1_label +[-1] +s2_label +[-1]
if len(sentence) > self.max_sentence :
print('sentence is greater than the setting of max sentence')
for pos in range(len(sentence),self.max_sentence):
sentence.append(self.token_to_idx['PAD'])
label.append(-1)
return {
'token' : sentence,
'label' : label,
'is_next' : is_next_sentence
}
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
def get_random_line(self,item):
rand = random.randint(0,len(self.data)-1)
while rand == item :
rand = random.randint(0,len(self.data)-1)
return rand
def get_random_sentence(self,sentence):
label = []
for idx,token in enumerate(sentence):
rand = random.random()
if rand < 0.15:
rand = rand/0.15
if rand < 0.8: #mask
sentence[idx] = self.token_to_idx['MASK']
elif rand < 0.9: #rand
sentence[idx] = random.randint(len(self.special_labels),len(self.token_to_idx)-1)
else: # still
sentence[idx] = self.token_to_idx[token]
label.append(self.token_to_idx[token])
else:
sentence[idx] = self.token_to_idx[token]
label.append(-1)
return sentence,label
if __name__ == '__main__':
test = vocab('vocab', 0, 1,'obj/',16)
print(len(test))
print(test[0])
print(test[1])
data = test[0]
Result:
2
{'token': [3, 4, 18, 12, 15, 11, 2, 7, 9, 13, 2, 0, 0, 0, 0, 0], 'label': [-1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
{'token': [3, 6, 4, 5, 8, 5, 17, 2, 16, 5, 14, 20, 2, 0, 0, 0], 'label': [-1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
Traceback (most recent call last):
File "vocab.py", line 146, in <module>
data = test[0]
File "vocab.py", line 90, in ```__getitem__```
s1,s1_label = self.get_random_sentence(s1)
File "vocab.py", line 136, in get_random_sentence
sentence[idx] = self.token_to_idx[token]
KeyError: 4
vocab file:
hello this is my home nice to meet you
I want to go to school and have lunch
change the code:
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
to:
def get_random_next_sentence(self,item):
s1 = copy.deepcopy(self.data[item][0])
s2 = copy.deepcopy(self.data[item][1])
if random.random() < 0.5 :
is_next = 0
s2 = copy.deepcopy(self.data[self.get_random_line(item)][1])
print(s2)
else:
is_next = 1
return s1,s2,is_next
For all other times I've done things like this, it's worked. But now, suddenly, my Python is messing up.
Here's the code block that's returning errors.
enemyLS = ['Nazi Solder', 'Nazi Scout', 'Nazi Ranger', 'Nazi Commander', 'Armored Nazi']
# [HP, ranAtk, melAtk]
enemy = ''
enemyHP = ''
enemyRan = ''
enemyMel = ''
enemyMan = {
'Nazi Soldier': [5, 3, 3],
'Nazi Scout': [3, 2, 2],
'Nazi Ranger': [5, 4, 2],
'Nazi Commander': [6, 4, 4],
'Armored Nazi': [7, 2, 3],
}
def fight():
fighting = True
while fighting:
enIn = rand.randint(0, len(enemyMan) - 1)
enemy = enemyLS[enIn]
enemyHP = enemyMan[enemyLS][1]
enemyRan = enemyMan[enemyLS][2]
And the enemyMan[enemyLS][1] stuff has always worked. Always. I have absolutely no idea why it isn't working.
I'm trying to apply a json-patch to a Mongoengine Document.
I'm using these json-patch library: https://github.com/stefankoegl/python-json-patch and mongoengine 0.14.3 with python 3.6.3
This is my actual code:
json_patch = JsonPatch.from_string(jp_string)
document = Document.objects(id=document_id)
json_documents = json.loads(document.as_pymongo().to_json())
json_patched_document = json_patch.apply(json_documents[0])
Document.objects(id=document_id).first().delete()
Document
.from_json(json.dumps(json_patched_document))
.save(force_insert=True)
Is there a better way to save an edited json document?
I've enhanced a little bit the code:
json_patch = JsonPatch.from_string(jp_string)
document = Document.objects(id=document_id)
json_document = json.loads(document.as_pymongo().to_json())
json_patched_document = json_patch.apply(json_documents[0])
Document
.from_json(json.dumps(json_patched_document), created=True)
.save()
but, is there a way to not convert the document to json?
I had slightly similar problem, the part that I dont wanted the complete Document for saving, I just wanted to update fields which are modified/added.
heres the code I tests on below inputs:
def tryjsonpatch():
doc_in_db = {'foo': 'bar', "name": "aj", 'numbers': [1, 3, 7, 8]}
input = {'foo': 'bar', "name": "dj", 'numbers': [1, 3, 4, 8]}
input2 = {'foo': 'bar', "name": "aj", 'numbers': [1, 3, 7, 8], "extera": "12"}
input3 = {'foo': 'bar', "name": "dj", 'numbers': [1, 3, 4, 8], "extera": "12"}
patch = jsonpatch.JsonPatch.from_diff(doc_in_db, input3)
print("\n***patch***\n", patch)
doc = get_minimal_doc(doc_in_db, patch)
result = patch.apply(doc, in_place=True)
print("\n###result###\n", result,
"\n###present###\n", doc_in_db)
def get_minimal_doc(present, patch):
cur_dc = {}
for change in patch.patch:
if change['op'] not in ("add"):
keys = change['path'].split("/")[1:]
present_move = {}
old_key = 1
first = True
for key in keys:
if key.isdigit(): # old_key represented a array
cur_dc[old_key] = present_move
else:
if first:
cur_dc[key] = {}
first = False
else:
cur_dc[old_key][key] = {}
old_key = key
present_move = present[old_key]
return cur_dc
tryjsonpatch()