Related
I am trying to run a user-generated simple code (two sum) inside AWS lambda python runtime Runtime.PYTHON_3_9.
def handler(event, context):
a = {
"code": "def twoSum(nums, target):\n hash_map = {}\n\n for i, num in enumerate(nums):\n if target - num in hash_map:\n return list(sorted([i, hash_map[target - num]]))\n hash_map[num] = i\n\ndef solve():\r\n testcases = [[[2, 7, 11, 15], 9]]\r\n output = [[0, 1]]\r\n\r\n total = len(testcases)\r\n correct = 0\r\n\r\n for i in range(len(testcases)):\r\n result = twoSum(*testcases[i])\r\n\r\n if result == output[i]:\r\n correct += 1\r\n\r\n print(correct, total)\r\n\r\n\r\nsolve()\r\n"
}
code = a["code"]
print(code)
exec(code)
This code prints
def twoSum(nums, target):
hash_map = {}
for i, num in enumerate(nums):
if target - num in hash_map:
return list(sorted([i, hash_map[target - num]]))
hash_map[num] = i
def solve():
testcases = [[[2, 7, 11, 15], 9]]
output = [[0, 1]]
total = len(testcases)
correct = 0
for i in range(len(testcases)):
result = twoSum(*testcases[i])
if result == output[i]:
correct += 1
print(correct, total)
solve()
while the actual code is
def twoSum(nums, target):
hash_map = {}
for i, num in enumerate(nums):
if target - num in hash_map:
return list(sorted([i, hash_map[target - num]]))
hash_map[num] = i
def solve():
testcases = [[[2, 7, 11, 15], 9]]
output = [[0, 1]]
total = len(testcases)
correct = 0
for i in range(len(testcases)):
result = twoSum(*testcases[i])
if result == output[i]:
correct += 1
print(correct, total)
solve()
All the spaces inside the exec strings are getting removed for some reason. Tried the same code on local Windows and macOS. It ran without any issues.
And this is the error I received
{
"errorMessage": "name 'twoSum' is not defined",
"errorType": "NameError",
"requestId": "<some-request-id>",
"stackTrace": [
" File \"/var/task/python.py\", line 10, in handler\n exec(code)\n",
" File \"<string>\", line 25, in <module>\n",
" File \"<string>\", line 17, in solve\n"
]
}
Is there any way to resolve this and make the exec work?
I want to realize the bert model.
So I built a class with __getitem__ in it.
I can print something like test[0], but when I assign a value, like data = test[0], a KeyError occurs.
import random
"""
corpus_file = 'vocab'
vocab_size = 6
vocab_freq = 1
save_path = 'obj/'
max_sentence = 16
corpus -> org_line -> ope_line
corpus -> org_line -> token_list -> idx_to_token + token_to_idx
"""
class vocab():
def __init__(self, corpus_file, vocab_size, vocab_freq,save_path,max_sentence):
self.max_sentence = max_sentence
self.special_labels = ['PAD', 'UNK', 'SEP', 'CLS', 'MASK']
# output
self.data = []
self.idx_to_token = []
self.token_to_idx = {}
# ope
self.pre_ope(corpus_file,vocab_size,vocab_freq)
#self.save_data(save_path)
#self.print_data()
def pre_ope(self,corpus_file,vocab_size,vocab_freq):
token_list = {}
with open(corpus_file, 'r') as f:
while 1:
new_org_line = f.readline()
if new_org_line != '':
new_org_line = new_org_line.strip('\n')
new_sentence = new_org_line.split('\t')
sentence = []
for tmp in new_sentence:
token_sentence = tmp.split()
sentence.append(token_sentence)
for token in token_sentence:
if token_list.get(token):
token_list[token] += 1
else:
new_token = {token: 1}
token_list.update(new_token)
self.data.append(sentence)
else:
break
f.close()
token_list = sorted(token_list.items(), key=lambda i: (-i[1], i[0]))
self.build_dictionary(token_list,vocab_freq,vocab_size)
'''
Special labels:
PAD
UNK
SEP sentence separator
CLS classifier token
MASK
'''
def build_dictionary(self,token_list,vocab_freq,vocab_size):
for idx, label in enumerate(self.special_labels):
self.idx_to_token.append(label)
self.token_to_idx[label] = idx
for idx, (token, freq) in enumerate(token_list):
if freq >= vocab_freq :
self.idx_to_token.append(token)
self.token_to_idx[token] = idx + len(self.special_labels)
if len(self.idx_to_token) >= vocab_size + len(self.special_labels) and vocab_size != 0 :
break
def __len__(self):
return len(self.data)
def print_data(self):
print(self.data)
print(self.idx_to_token)
print(self.token_to_idx)
def __getitem__(self, item):
s1,s2,is_next_sentence = self.get_random_next_sentence(item)
s1,s1_label = self.get_random_sentence(s1)
s2,s2_label = self.get_random_sentence(s2)
sentence = [self.token_to_idx['CLS']] +s1 +[self.token_to_idx['SEP']] +s2 +[self.token_to_idx['SEP']]
label = [-1] +s1_label +[-1] +s2_label +[-1]
if len(sentence) > self.max_sentence :
print('sentence is greater than the setting of max sentence')
for pos in range(len(sentence),self.max_sentence):
sentence.append(self.token_to_idx['PAD'])
label.append(-1)
return {
'token' : sentence,
'label' : label,
'is_next' : is_next_sentence
}
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
def get_random_line(self,item):
rand = random.randint(0,len(self.data)-1)
while rand == item :
rand = random.randint(0,len(self.data)-1)
return rand
def get_random_sentence(self,sentence):
label = []
for idx,token in enumerate(sentence):
rand = random.random()
if rand < 0.15:
rand = rand/0.15
if rand < 0.8: #mask
sentence[idx] = self.token_to_idx['MASK']
elif rand < 0.9: #rand
sentence[idx] = random.randint(len(self.special_labels),len(self.token_to_idx)-1)
else: # still
sentence[idx] = self.token_to_idx[token]
label.append(self.token_to_idx[token])
else:
sentence[idx] = self.token_to_idx[token]
label.append(-1)
return sentence,label
if __name__ == '__main__':
test = vocab('vocab', 0, 1,'obj/',16)
print(len(test))
print(test[0])
print(test[1])
data = test[0]
Result:
2
{'token': [3, 4, 18, 12, 15, 11, 2, 7, 9, 13, 2, 0, 0, 0, 0, 0], 'label': [-1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
{'token': [3, 6, 4, 5, 8, 5, 17, 2, 16, 5, 14, 20, 2, 0, 0, 0], 'label': [-1, -1, 19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], 'is_next': 0}
Traceback (most recent call last):
File "vocab.py", line 146, in <module>
data = test[0]
File "vocab.py", line 90, in ```__getitem__```
s1,s1_label = self.get_random_sentence(s1)
File "vocab.py", line 136, in get_random_sentence
sentence[idx] = self.token_to_idx[token]
KeyError: 4
vocab file:
hello this is my home nice to meet you
I want to go to school and have lunch
change the code:
def get_random_next_sentence(self,item):
s1 = self.data[item][0]
s2 = self.data[item][1]
if random.random() < 0.5 :
is_next = 0
s2 = self.data[self.get_random_line(item)][1]
else:
is_next = 1
return s1,s2,is_next
to:
def get_random_next_sentence(self,item):
s1 = copy.deepcopy(self.data[item][0])
s2 = copy.deepcopy(self.data[item][1])
if random.random() < 0.5 :
is_next = 0
s2 = copy.deepcopy(self.data[self.get_random_line(item)][1])
print(s2)
else:
is_next = 1
return s1,s2,is_next
I'm having a hard time getting the following to print correctly:
core = 1, 2, 3, 4, 5
glutes = 6, 7, 8, 9, 10
upper = 11, 12, 13, 14, 15
lower = 16, 17, 18, 19, 20
conditioning = 21, 22, 23, 24, 25
core_ability = int(input("Core: "))
glute_ability = int(input("Glutes: "))
if core_ability > 4:
upper_ability = int(input("Upper body: "))
else:
""
lower_ability = int(input("Lower body: "))
conditioning_ability = int(input("\nConditioning ability level:"))
newcore = core[0:core_ability]
newglutes = glutes[0:glute_ability]
if core_ability > 4:
newupper = upper[0:upper_ability]
newlower = lower[0:lower_ability]
newconditioning = conditioning[0:conditioning_ability]
if core_ability > 4:
movement_bank = str(newcore) + str(newglutes) + str(newupper) + str(newlower) + str(conditioning_ability)
else:
movement_bank = str(newcore) + str(newglutes) + str(newlower) + str(conditioning_ability)
sections = int(input("\nNumber of GPP sections in the session: "))
print("\nSPECIFY THE NUMBER OF MOVEMENTS PER SECTION")
if sections == 1:
section1_num = int(input("Section 1:"))
print(random.sample(movement_bank[0:], k=section1_num))
I get an output the looks like:
' ', ' ', 'r'
when I'd like to get something like:
'1', '16', '8'
I added "str()" to each list in the "movement_bank" list because without it I got an error of: TypeError: can only concatenate list (not "int") to list.
All help is greatly appreciated.
It seems, you have different lists, and want to combine them all into one list.
Use extend:
core = 1, 2, 3, 4, 5
glutes = 6, 7, 8, 9, 10
upper = 11, 12, 13, 14, 15
lower = 16, 17, 18, 19, 20
conditioning = 21, 22, 23, 24, 25
movement_bank = []
core_ability = int(input("Core: "))
movement_bank.extend(core[:core_ability])
glute_ability = int(input("Glutes: "))
movement_bank.extend(glutes[:glute_ability])
if core_ability > 4:
upper_ability = int(input("Upper body: "))
movement_bank.extend(upper[:upper_ability])
lower_ability = int(input("Lower body: "))
movement_bank.extend(lower[:lower_ability])
conditioning_ability = int(input("\nConditioning ability level:"))
movement_bank.extend(conditioning[:conditioning_ability])
sections = int(input("\nNumber of GPP sections in the session: "))
print("\nSPECIFY THE NUMBER OF MOVEMENTS PER SECTION")
if sections == 1:
section1_num = int(input("Section 1:"))
print(random.sample(movement_bank, k=section1_num))
Whenever I try to call this function in the MyHDL implementation of MD5 I've been working on I get this error:
File "/usr/local/lib/python2.7/dist-packages/myhdl/conversion/_misc.py", line 149, in raiseError
raise ConversionError(kind, msg, info)
myhdl.ConversionError: in file MD5-8, line 85:
Free variable should be a Signal or an int: calculate
Here is the entire script. If anyone has any light to shed on this or anything else that would be extremely helpful.
Thanks so much.
from myhdl import *
def operative(M0,M1,M2,M3,M4,M5,M6,M7,M8,M9,M10,M11,M12,M13,M14,M15,enable,clock):
singletick = 0
def toW32(x):
return(modbv(x, min=0, max=2**32)[32:])
def leftrotate(x, c):
return ((x<<c) | (x>>(32-c)) % (2**32))
def calculate(M, A, B, C, D, count):
KCONTENT = (0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391)
SCONTENT = (7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21)
F = int(0)
G = int(0)
#get G and F
if(count<=15):
if(count >=0):
F= (B&C)|((~B)&D)
G = int(count)
elif(count<=31):
if(count >= 15):
F = (D&B)|((~D)&C)
G = int(5*count +1) % 16
elif(count<=47):
if(count>=32):
F = (B^C^D)
G = int(3*count+5) % 16
elif(count <= 63):
if(count >= 48):
F = C^(B|(~D))
G = int(7*count) % 16
#swapping A through D and then calling again
temp = D
D = C
C = B
F = toW32(F)
G = toW32(G)
currentM = toW32(int(M[G]))
currentK = toW32(int(KCONTENT[count]))
currentS = toW32(int(SCONTENT[count]))
#B = leftrotate((((A + F) % (2**32) + (M[G]+KCONTENT[count]) % (2**32)) % (2**32)), SCONTENT[count])
A2 = toW32(A)
F2 = toW32(F)
bcomp0 = toW32((A2 + F2) % (2**32))
bcomp1 = toW32((currentM + currentK) % (2**32))
bcomp = toW32((bcomp0 + bcomp1) % (2**32))
bcomp2 = (leftrotate(bcomp, currentS))
B = toW32((B + bcomp2) % (2**32))
A = temp
print(B)
if(count>=63):
outA = (toW32((0x67452301+A) % (2**32)))
outB = (toW32((0xefcdab89+B) % (2**32)))
outC = (toW32((0x98badcfe+C) % (2**32)))
outD = (toW32((0x10325476+D) % (2**32)))
print(hex(concat(outA, outB, outC, outD)))
return(outA, outB, outC, outD)
else:
count = count + 1
calculate(M, A, B, C, D, count)
HALF_PERIOD = delay(10)
#always(HALF_PERIOD)
def clockGen():
clock.next = not clock
M = (M0,M1,M2,M3,M4,M5,M6,M7,M8,M9,M10,M11,M12,M13,M14,M15)
#always(clock.posedge)
def central():
if(enable == 1):
A = toW32(0x67452301)
B = toW32(0xefcdab89)
C = toW32(0x98badcfe)
D = toW32(0x10325476)
count = toW32(0)
final = int(0)
final = calculate(M, A, B, C, D, count)
return clockGen, central
I'm writing my first multiprocessing program in python.
I want to create a list of values to be processed, and 8 processes (number os CPU cores) will consume and process the list of values.
I wrote the following python code:
__author__ = 'Rui Martins'
from multiprocessing import cpu_count, Process, Lock, Value
def proc(lock, number_of_active_processes, valor):
lock.acquire()
number_of_active_processes.value+=1
print "Active processes:", number_of_active_processes.value
lock.release()
# DO SOMETHING ...
for i in range(1, 100):
valor=valor**2
# (...)
lock.acquire()
number_of_active_processes.value-=1
lock.release()
if __name__ == '__main__':
proc_number=cpu_count()
number_of_active_processes=Value('i', 0)
lock = Lock()
values=[11, 24, 13, 40, 15, 26, 27, 8, 19, 10, 11, 12, 13]
values_processed=0
processes=[]
for i in range(proc_number):
processes+=[Process()]
while values_processed<len(values):
while number_of_active_processes.value < proc_number and values_processed<len(values):
for i in range(proc_number):
if not processes[i].is_alive() and values_processed<len(values):
processes[i] = Process(target=proc, args=(lock, number_of_active_processes, values[values_processed]))
values_processed+=1
processes[i].start()
while number_of_active_processes.value == proc_number:
# BUG: always number_of_active_processes.value == 8 :(
print "Active processes:", number_of_active_processes.value
print ""
print "Active processes at END:", number_of_active_processes.value
And, I have the following problem:
The program never stop
I get out of RAM
Simplifying your code to the following:
def proc(lock, number_of_active_processes, valor):
lock.acquire()
number_of_active_processes.value += 1
print("Active processes:", number_of_active_processes.value)
lock.release()
# DO SOMETHING ...
for i in range(1, 100):
print(valor)
valor = valor **2
# (...)
lock.acquire()
number_of_active_processes.value -= 1
lock.release()
if __name__ == '__main__':
proc_number = cpu_count()
number_of_active_processes = Value('i', 0)
lock = Lock()
values = [11, 24, 13, 40, 15, 26, 27, 8, 19, 10, 11, 12, 13]
values_processed = 0
processes = [Process() for _ in range(proc_number)]
while values_processed < len(values)-1:
for p in processes:
if not p.is_alive():
p = Process(target=proc,
args=(lock, number_of_active_processes, values[values_processed]))
values_processed += 1
p.start()
If you run it like above the print(valor) added you see exactly what is happening, you are exponentially growing valor to the point you run out of memory, you don't get stuck in the while you get stuck in the for loop.
This is the output at the 12th process adding a print(len(srt(valor))) after a fraction of a second and it just keeps on going:
2
3
6
11
21
.........
59185
70726
68249
73004
77077
83805
93806
92732
90454
104993
118370
136498
131073
Just changing your loop to the following:
for i in range(1, 100):
print(valor)
valor = valor *2
The last number created is:
6021340351084089657109340225536
Using your own code you seem to get stuck in the while but it is valor is growing in the for loop to numbers with as many digits as:
167609
180908
185464
187612
209986
236740
209986
And on....
The problem is not your multiprocessing code. It's the pow operator in the for loop:
for i in range(1, 100):
valor=valor**2
the final result would be pow(val, 2**100), and this is too big, and calculate it would cost too much time and memory. so you got out of memory error in the last.
4 GB = 4 * pow(2, 10) * pow(2, 10) * pow(2, 20) * 8 bit = 2**35 bit
and for your smallest number 8:
pow(8, 2**100) = pow(2**3, 2**100) = pow(2, 3*pow(2, 100))
pow(2, 3*pow(2, 100))bit/4GB = 3*pow(2, 100-35) = 3*pow(2, 65)
it need 3*pow(2, 65) times of 4 GB memory.