We Keep Coding

how to use multiprocessing inside class have to attribute are object and dict?

I have a class:
# myclass.py
class MyClass(object):
def __init__(self,*args):
self.env = args[0]
self.mydict = args[1]
def run(self):
list_data = [1,2,3,4,5,6,7,8,9,10]
pool = mp.Pool(3)
for _ in tqdm(pool.imap_unordered(self.exefunction, list_data), total = len(a), desc = 'Main process'):
pass
pool.close()
pool.join
def exefunction(self,number):
print(number)
print(self.env)
print(self.mydict)
# myenv.py
class MyEnv(object):
def __init__(self, *args, **kwargs):
self.database = args[0]
self.host = args[1]
self.port = args[2]
self.user = args[3]
self.pwd = args[4]
self.sp = None
self.param = None
self.output_exception_msg = None
self.output_sperror_msg = None
self.pool = self.create_pool()
def create_pool(self):
self.pool = MySQLConnectionPool(pool_name=config.pool_name, pool_size=config.pool_size,
user=self.user, password=self.pwd, host=self.host, port=self.port, database=self.database)
return self.pool
# main.py
if __name__ == '__main__':
list_dict = [{ 1 : 3 , 5 : 10 }]
env = MyEnv(**dbconfig)
build_class = MyClass(env, list_dict)
build_class.run()
self.env is an object that was created from another class.
When I call to function run I have a TypeError: cannot serialize socket object.
I know the problem is MyClass has an object and dict (I try to pass self.env and self.mydict are integer and it's working). I don't have any solution to fix it.
UPDATE: I opened a connection pool into my DB ( self.env contains that)

Each process runs in its own address space and therefore needs its own DB connection pool. So:
Class MyEnv should not explicitly call method create_pool from its __init__ method; this pool-creation needs to be postponed until later.
Each process in the multiprocessing pool will have its own instance of class MyEnv as a global variable that gets initialized when the pool is created. create_pool will be called on each of these instances as part of that initialization.
I obviously am not in a position to test this, but you should get the general idea if minimal tweaking is necessary:
def init_pool(the_env):
global env
env = the_env
env.create_pool() # now create the pool
class MyClass(object):
def __init__(self,*args):
self.env = args[0]
self.mydict = args[1]
def run(self):
list_data = [1,2,3,4,5,6,7,8,9,10]
# initialize the env global variable for each process in the process pool :
pool = mp.Pool(3, initializer=init_pool, initargs=(self.env,))
for _ in tqdm(pool.imap_unordered(self.exefunction, list_data), total = len(a), desc = 'Main process'):
pass
pool.close()
pool.join
def exefunction(self,number):
print(number)
print(env) # access global
print(self.mydict)
# myenv.py
class MyEnv(object):
def __init__(self, *args, **kwargs):
self.database = args[0]
self.host = args[1]
self.port = args[2]
self.user = args[3]
self.pwd = args[4]
self.sp = None
self.param = None
self.output_exception_msg = None
self.output_sperror_msg = None
# don't create the pool now:
self.pool = None
def create_pool(self):
self.pool = MySQLConnectionPool(pool_name=config.pool_name, pool_size=config.pool_size,
user=self.user, password=self.pwd, host=self.host, port=self.port, database=self.database)
return self.pool
# main.py
if __name__ == '__main__':
list_dict = [{ 1 : 3 , 5 : 10 }]
env = MyEnv(**dbconfig)
build_class = MyClass(env, list_dict)
build_class.run()

Attribute error when using Python multithreading

I get an error:
AttributeError: 'Park_Stat_Thread' object has no attribute 'park_stat
when I run the script below. Any idea how to pass back the park_stat variable?
As you can see I use multi-threading to crawl a list of URLs from the web:
class Park_Stat_Thread(Thread):
def __init__(self, thread_num):
super().__init__()
self.thread_num = thread_num
def run(self):
print('starting thread %s.' % self.thread_num)
process_queue(self)
print('exiting thread %s.' % self.thread_num)
def process_queue(self):
while True:
try:
x = my_queue.get(block=False)
except queue.Empty:
return
else:
parking_stat_getter(self,x)
time.sleep(1)
def parking_stat_getter(self,x):
base = 'http://www.ahuzot.co.il/Parking/ParkingDetails/?ID={}'
response = requests.get(base.format(self.thread_num))
soup = BeautifulSoup(response.text, "lxml")
try:
self.park_stat = 0
for img in soup.find_all('img' , attrs={'src': re.compile("ParkingIcons")}):
soup_img = str(img)
if 'male' in soup_img:
self.park_stat=1
elif 'meat' in soup_img:
self.park_stat=2
elif 'panui' in soup_img:
self.park_stat=3
except AttributeError:
self.park_stat = np.nan
return self.park_stat
def get_parkings_Stat(parking_id_lists):
threads = [Park_Stat_Thread(t) for t in range(1, 8)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
park_details = dict(zip(parking_id_lists, [thread.park_stat for thread in threads]))
return park_details

You are missing a class attribute from the class constructor. You cannot declare a class attribute later, but you can set it to some dummy value.
class Park_Stat_Thread(Thread):
def __init__(self, thread_num):
super().__init__()
self.thread_num = thread_num
self.park_stat = 0
def run(self):
print('starting thread %s.' % self.thread_num)
process_queue(self)
print('exiting thread %s.' % self.thread_num)
def process_queue(self):
while True:
try:
x = my_queue.get(block=False)
except queue.Empty:
return
else:
parking_stat_getter(self,x)
time.sleep(1)
def parking_stat_getter(self,x):
base = 'http://www.ahuzot.co.il/Parking/ParkingDetails/?ID={}'
response = requests.get(base.format(self.thread_num))
soup = BeautifulSoup(response.text, "lxml")
try:
self.park_stat = 0
for img in soup.find_all('img' , attrs={'src': re.compile("ParkingIcons")}):
soup_img = str(img)
if 'male' in soup_img:
self.park_stat=1
elif 'meat' in soup_img:
self.park_stat=2
elif 'panui' in soup_img:
self.park_stat=3
except AttributeError:
self.park_stat = np.nan
return self.park_stat
def get_parkings_Stat(parking_id_lists):
threads = [Park_Stat_Thread(t) for t in range(1, 8)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
park_details = dict(zip(parking_id_lists, [thread.park_stat for thread in threads]))
return park_details

How can I stop a long-running function when it is called multiple times?

Below I have an example program. When the button is pressed, it takes a second before it can calculate the value to show. If the user presses the button in rapid succession they end up waiting a long time to see the last answer, which is the only answer they care about. In the code, you can see that the _dataCruncher function needs to know self._count, but self._count does not depend on the output of _dataCruncher.
My question, therefore, is how can I interrupt the normal execution of _dataCruncher on subsequent calls in order to keep the GUI free to do other stuff, and to not waste processing time when it is not needed? I realize that I will likely need to use a thread to run _dataCruncher and some sort of Queue to get the appropriate val to display, but I do not understand how to put this all together.
from PyQt4 import QtGui, QtCore
import sys
import time
import random
import random
class MainWindow(QtGui.QMainWindow):
def __init__(self):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _dataCruncher(self, val):
time.sleep(1) # takes a long time to process data using val
return val * random.randint(1,10)
def _buttonClicked(self):
self.count += 1
val = self._dataCruncher(self.count)
self.text.setText('Value {}'.format(val))
def startup(self):
self.show()
result = self.app.exec_()
sys.exit(result)
if __name__ == '__main__':
random.seed()
myWindow = MainWindow()
myWindow.startup()

So, finding an answer to this was more complicated than I thought. As #MTset mentions in one of the comments, python does not offer any means by which to cancel the execution of a Thread. So, what I did was create a 'threadHandler' class that, well, handles thread. It keeps track of the last thread that was created and offers a means by which to get the result from the execution of the last thread.
I am posting an modified version of the test code from the original post as well as the threadHandler code in full in case anyone has use for it.
File 1 here
# tester.py, run this file
from PyQt4 import QtGui, QtCore
import random, sys, time
from threadHandler import MyThreadHandler
class MyModel(object):
def dataCruncher(self, val):
delay = random.randint(1,5)
print('{} sleeping for {}'.format(val, delay))
time.sleep(delay) # takes a long time to process data using val
print('{} done sleeping'.format(val))
return val
class MainWindow(QtGui.QMainWindow):
def __init__(self, threadHandler):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
self.button_clicked_events = Event()
self.threadHandler = threadHandler
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _buttonClicked(self):
self.count += 1
self.button_clicked_events(self.count)
def setLabel(self, val):
self.text.setText(str(val))
def startup(self):
self.show()
result = self.app.exec_()
return result
class Event(list):
"""Event subscription.
A list of callable objects. Calling an instance of this will cause a
call to each item in the list in ascending order by index.
Example Usage:
>>> def f(x):
... print 'f(%s)' % x
>>> def g(x):
... print 'g(%s)' % x
>>> e = Event()
>>> e()
>>> e.append(f)
>>> e(123)
f(123)
>>> e.remove(f)
>>> e()
>>> e += (f, g)
>>> e(10)
f(10)
g(10)
>>> del e[0]
>>> e(2)
g(2)
"""
def __init__(self):
self.output = {}
def __call__(self, *args, **kwargs):
for f,key in self:
output = f(*args, **kwargs)
self.output[key] = output
return self.output
def __repr__(self):
return "Event({})".format(list.__repr__(self))
if __name__ == '__main__':
def checker(handler, window):
if handler.isLastDone():
val = handler.getLastResult()
window.setLabel(val)
else:
window.setLabel('calculating...')
random.seed()
model = MyModel()
threadHandler = MyThreadHandler()
myWindow = MainWindow(threadHandler)
threadHandler.createTimer(1, checker, threadHandler, myWindow)
def getData(count):
threadHandler.createOneShot(model.dataCruncher, count)
myWindow.button_clicked_events.append((getData, 'dt'))
result = myWindow.startup()
print('ending')
threadHandler.end()
print('ended')
sys.exit(result)
File 2 below
#threadHandler.py, save this file in the same folder as tester.py
import threading, time
class MyThreadHandler(object):
def __init__(self):
self.oneShots = []
self.timers = []
self.oldOneShots = []
self.latest = None
self.cleaning = False
self._startCleaner()
def _startCleaner(self):
print('-'*20+'Starting cleaner'+'-'*20)
self.cleaner = self.createTimer(1, self._cleanupThreads)
def _stopCleaner(self):
print('-'*20+'Stopping cleaner'+'-'*20)
self.cleaner.stop()
def getNumThreads(self):
return len(self.oneShots)
def getNumOldThreads(self):
return len(self.oldOneShots)
def end(self):
for i,timer in enumerate(self.timers):
timer.stop()
self.timers.pop(i)
def createTimer(self, interval, func, *args, **kwargs):
timer = myTimer(interval, func, args, kwargs)
self.timers.append(timer)
return timer
def createOneShot(self, func, *args, **kwargs):
oneshot = myOneShot(func, args, kwargs)
self.oneShots.append(oneshot)
self.latest = oneshot
def isLastDone(self):
if not self.latest is None:
return not self.latest.running()
else:
return None
def getLastResult(self):
if self.latest is None:
raise ValueError('There have not been any oneshots created.')
while self.latest.running():
pass
result = self.latest.getResult()
if len(self.oneShots) > 0:
self.oldOneShots.append(myOneShot(self._cleanAll, (self.oneShots,)))
self.oneShots = []
return result
def _cleanAll(self, toClean):
# loop through toClean and pop up anything that's done. this DOES lock
while len(toClean) > 0:
toClean = self._cleanup(toClean)
def _cleanup(self, toCleanup):
while not self.cleaning:
self.cleaning = True
for i, thread in enumerate(toCleanup):
if not thread.running():
toCleanup.pop(i)
self.cleaning = False
return toCleanup
def _cleanupThreads(self):
# check each of these lists and pop out any threads that are done. This
# does not lock. This function should really only be called by the
# cleaner, which is set up in __init__
self.oneShots = self._cleanup(self.oneShots)
self.timers = self._cleanup(self.timers)
self.oldOneShots = self._cleanup(self.oldOneShots)
class myTimer(object):
def __init__(self, delay, func, args=tuple(), kwargs={}):
self.delay = delay
self.func = func
self.loop = True
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
while self.loop:
self.output = self.func(*self.args, **self.kwargs)
if self.delay > 0.1:
count = 0
while count <= self.delay:
count += 0.1
time.sleep(0.1)
else:
time.sleep(self.delay)
def stop(self):
self.loop = False
def running(self):
return self.loop
def getResult(self):
return self.output
class myOneShot(object):
def __init__(self, func, args=tuple(), kwargs={}):
self.func = func
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
self.output = self.func(*self.args, **self.kwargs)
def running(self):
return self.thread.is_alive()
def getResult(self):
return self.output
if __name__ == '__main__':
import random
random.seed()
def longFunc(num):
delay = random.randint(5,8)
if num in (3, 6):
delay = 2
print('-'*30+'func {} has sleep {}'.format(num, delay))
time.sleep(delay)
print('-'*30+'func {} is done'.format(num))
return num
def checker(handler):
if handler.isLastDone():
return handler.getLastResult()
else:
return None
myHandler = MyThreadHandler()
# The 'checker' function simulates something in my program that uses the
# data generated by the 'longFunc'. It waits until there are no more threads
# in the threadHandler, as that would indicate that the user is done
# switching back-and-forth between different values
checkTimer = myHandler.createTimer(1, checker, myHandler)
# create 10 one-shot threads that take a 'long' time. The delay is to keep
# them in order, as this loop is meant to simulate a user switching between
# items using a keyboard or mouse, which I imagine they couldn't do any
# faster than every 1/10th of a second
start = time.time()
for i in range(4):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
# wait until there are no more threads executing
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
for i in range(4, 7):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
while myHandler.getNumOldThreads() >0 or myHandler.getNumThreads() > 0:
pass
myHandler.end()
print('done ending')

You could disable the button after it's pressed until an answer is ready using:
setEnabled(False)
Then reset it just before providing the result.

Update client widget from server with rpyc callback

I want to edit all client QTextBrowser logs when any client sends some message to server.
my intended procedure is this:
[client]enter -> [client]chatUI.handleEnter -> (RFC)[server]exposed_send -> [server]broadcast -> (RFC)[clients]update.emit() -> [clients]listen -> log changed
When I run this code, other clients logs are not changed and only the client that give input to server has an updated log.
How I can solve this to update all clients properly?
chat_server.py
import rpyc
import random
import string
from threading import RLock
users = dict()
callbacks = dict()
user_num = 0
lock = RLock()
buf = dict()
class chatService(rpyc.Service):
def on_connect(self):
global user_num
with lock:
user_num = user_num+1
print ("connect user: %d" % user_num)
def on_disconnect(self):
global user_num
with lock:
user_num = user_num-1
print ("disconnect user: %d" % user_num)
def exposed_accept(self, idt, callback):
with lock:
global users
global callbacks
if not isinstance(idt, str) or len(idt) != 6:
return False
elif idt in users:
return -1
else:
pw = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
users[idt] = pw
callbacks[idt] = rpyc.async(callback)
return pw
def exposed_send(self, target, msg, idt, pw):
print ('here')
name = self.identify(idt, pw)
if name == False:
print ('here2')
return False
else:
print ('here3')
global callbacks
if target == None:
self.broadcast("[%s] %s" % (name, msg))
elif msg.target in callbacks:
self.send("[%s] %s" %(name, msg), target)
else:
return False
def exposed_order(self, msg, idt, pw):
pass
def identify(self, idt, pw):
global users
if users[idt] == pw:
return idt
else:
return False
def broadcast(self, msg):
with lock:
print("bloadcast calls")
global callbacks
global buf
for user, callback in callbacks.items():
if user not in buf or buf[user] == None:
buf[user] = (msg,)
else:
buf[user] = buf[user] + (msg,)
callback()
def send(self, msg, target):
global callbacks
global buf
if user not in buf or buf[user] == None:
buf[target] = (msg,)
else:
buf[target] = buf[target] + (msg,)
callbacks[target]()
def exposed_get_buf(self, user):
global buf
temp = buf[user]
buf[user] = None
return temp
if __name__ == '__main__':
from rpyc.utils.server import ThreadedServer
t = ThreadedServer(chatService, port = 3743)
t.start()
chat_client.py
from chatUI import *
import rpyc
import random
import string
if __name__ == '__main__':
service = rpyc.connect('floating.tk', 3743)
app, chat = UIReady(service)
while True:
idt = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
pw = service.root.accept(idt, chat.update.update.emit)
if pw != False and pw != -1:
break
chat.idt = idt
chat.pw = pw
sys.exit(app.exec_())
chatUI.py
import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
class Updater(QObject):
update = pyqtSignal()
class Chat(QWidget):
log = None
enter = None
def __init__(self, service) :
super().__init__()
self.service = service
self.idt = None
self.pw = None
self.initUI()
self.update = Updater()
self.update.update.connect(self.listen)
def initUI(self):
logLabel = QLabel('chat log')
enterLabel = QLabel('enter')
self.log = QTextBrowser()
self.enter = QLineEdit()
self.enter.returnPressed.connect(self.handleEnter)
layout = QGridLayout()
layout.addWidget(logLabel, 0, 0)
layout.addWidget(self.log, 0, 1, 5, 1)
layout.addWidget(enterLabel, 6, 0)
layout.addWidget(self.enter, 6, 1)
self.setLayout(layout)
self.setWindowTitle('chat')
self.resize(600, 600)
self.show()
def handleEnter(self):
msg = self.enter.text()
self.enter.setText("")
self.service.root.send(None, msg, self.idt, self.pw)
print('get enter')
def listen(self):
msg = self.service.root.get_buf(self.idt)
for m in msg:
self.log.append(m)
def UIReady(service):
app = QApplication(sys.argv)
chat = Chat(service)
return app, chat

python stop threading that are running

I know this question is asked a lot of time but i am still like to know.
def startMonitor(self,event):
selectedInterface = self.interfaces_cblist.GetValue()
Publisher().sendMessage(("test"),selectedInterface)
self.Close()
selectInterfaceStr = str(selectedInterface)
if len(selectedInterface) == 0:
noSelect_error = wx.MessageDialog(None,"Please select an interface","",wx.OK|wx.ICON_ERROR)
noSelect_error.ShowModal()
else:
monitorStarted = wx.MessageDialog(None,"Monitor on %s started"%selectInterfaceStr,"",wx.OK|wx.ICON_ERROR)
monitorStarted.ShowModal()
self.monitorInterface_button.Disable()
thread.start_new_thread(self.camtableDetection,(selectInterfaceStr,))
thread.start_new_thread(self.dhcpexhaustion,(selectInterfaceStr,))
how can i stop the threading?

You can have a stop method that assigns to a variable such as self.abort. Than, in the function you are threading, you should check for this variable regularly and stop the function(with return or something similar). Here's an example class that uses this technique for stopping the thread.
class PymineLogger:
def __init__(self):
self.file = open('server.log', 'a')
self.abort = False
self.log_queue = Queue.Queue()
threading.Thread(target=self.process_queue, args=()).start()
def error(self, i):
line = u'[%s] [ERROR] %s' % (str(time.time()), i)
self.log_queue.put(line)
def info(self, i):
line = u'[%s] [INFO] %s' % (str(time.time()), i)
self.log_queue.put(line)
def process_queue(self):
while not self.abort:
try:
log_line = self.log_queue.get(timeout=1)
print log_line
self.file.write("%s\n" % log_line)
self.file.flush()
except Queue.Empty:
pass
def stop(self):
self.abort = True
The stop method assigns the variable self.abort, which gets regularly checked by the thread.
Class source: pymine2 project