I am writing a server for some library (python).
I want that while the server is working in his loop it will open 1 thread to do something else.
I am controlling this thread with queue and until there is no return value in the queue i don't want the server to open another thread.
try:
#we have a return in the queqe
returnValue = threadQueue.get(False)
startAnotherThread = True
except Empty:
print "Waiting for return value from thread thread....."
if there is some return value in the queue then startAnotherThread will tell to some if statement to open another thread.
i don't know why it's not working mabye some one have an idea?
Solved:
Init before the server loop:
# thread queue
threadQueue = Queue()
# thread numbering
threadNumber = 0
# thread start
threadCanStart = True
Inside the server loop:
if threadCanStart:
myThread(threadNumber, threadQueue).start()
threadNumber += 1
threadCanStart = False
try:
#we have a return in the quqe
returnValue = threadQueue.get(False)
print "Queue return: ", returnValue
threadCanStart = True
except Empty:
print "Waiting for thread....."
Related
I would like to stop/kill N thread when I receive a socket event.
I don't find any way to do that with my current app structure.
Here is the code to start threads :
for i in range(news_viewers):
t = threading.Thread(target=bot, args=(i + 1,))
t.daemon = True
t.name = "Viewer"
t.start()
When I receive the event I would like to kill/stop N thread named Viewer on my list :
for i in range(number_of_threads_to_kill):
#number_of_threads_to_kill is received by sockerIO
for t in threading.enumerate():
if 'Viewer' in t.getName():
#I NEED TO CLOSE N THREAD HERE
print('CLOSE THIS THREAD')
I can't find a way to do it, I tried many things but none of them worked.
threading.enumerate() return this :
.., <Thread(Viewer, started daemon 41868)>, <Thread(Viewer, started daemon 53872)>, <Thread(Viewer, started daemon 54748)>, <Thread(Viewer, started daemon 50028)>,...
Can someone help me to setup that ?
I finally found a way to close thread.
When I start a thread I add the id of the thread in threads_ids array :
global stop_ids, threads_ids
for i in range(news_viewers):
tid = random.randint(1, 1000)
threads_ids.append(tid)
t = threading.Thread(target=bot, args=(tid,))
t.daemon = True
t.name = "Viewer"
t.start()
When I want to close a Thread I add the N first id on stop_ids array :
global stop_ids, threads_ids
n = threads_ids[:number_of_thread_to_close]
stop_ids = stop_ids + n
On my bot function I check every N second if the id of the current thread is on stop_ids :
def bot(id):
global stop_ids
...
q = True
while q:
if id in stop_ids:
stop_ids.remove(id)
driver.quit()
q = False
break
...
I have noticed that when I have many threads pulling elements from a queue, there are less elements processed than the number that I put into the queue. This is sporadic but seems to happen somewhere around half the time when I run the following code.
#!/bin/env python
from threading import Thread
import httplib, sys
from Queue import Queue
import time
import random
concurrent = 500
num_jobs = 500
results = {}
def doWork():
while True:
result = None
try:
result = curl(q.get())
except Exception as e:
print "Error when trying to get from queue: {0}".format(str(e))
if results.has_key(result):
results[result] += 1
else:
results[result] = 1
try:
q.task_done()
except:
print "Called task_done when all tasks were done"
def curl(ourl):
result = 'all good'
try:
time.sleep(random.random() * 2)
except Exception as e:
result = "error: %s" % str(e)
except:
result = str(sys.exc_info()[0])
finally:
return result or "None"
print "\nRunning {0} jobs on {1} threads...".format(num_jobs, concurrent)
q = Queue()
for i in range(concurrent):
t = Thread(target=doWork)
t.daemon = True
t.start()
for x in range(num_jobs):
q.put("something")
try:
q.join()
except KeyboardInterrupt:
sys.exit(1)
total_responses = 0
for result in results:
num_responses = results[result]
print "{0}: {1} time(s)".format(result, num_responses)
total_responses += num_responses
print "Number of elements processed: {0}".format(total_responses)
Tim Peters hit the nail on the head in the comments. The issue is that the tracking of results is threaded and isn't protected by any sort of mutex. That allows something like this to happen:
thread A gets result: "all good"
thread A checks results[result]
thread A sees no such key
thread A suspends # <-- before counting its result
thread B gets result: "all good"
thread B checks results[result]
thread B sees no such key
thread B sets results['all good'] = 1
thread C ...
thread C sets results['all good'] = 2
thread D ...
thread A resumes # <-- and remembers it needs to count its result still
thread A sets results['all good'] = 1 # resetting previous work!
A more typical workflow might have a results queue that the main thread is listening on.
workq = queue.Queue()
resultsq = queue.Queue()
make_work(into=workq)
do_work(from=workq, respond_on=resultsq)
# do_work would do respond_on.put_nowait(result) instead of
# return result
results = {}
while True:
try:
result = resultsq.get()
except queue.Empty:
break # maybe? You'd probably want to retry a few times
results.setdefault(result, 0) += 1
Here is my script:
# globals
MAX_PROCESSES = 50
my_queue = Manager().Queue() # queue to store our values
stop_event = Event() # flag which signals processes to stop
my_pool = None
def my_function(var):
while not stop_event.is_set():
#this script will run forever for each variable found
return
def var_scanner():
# Since `t` could have unlimited size we'll put all `t` value in queue
while not stop_event.is_set(): # forever scan `values` for new items
x = Variable.objects.order_by('var').values('var__var')
for t in x:
t = t.values()
my_queue.put(t)
time.sleep(10)
try:
var_scanner = Process(target=var_scanner)
var_scanner.start()
my_pool = Pool(MAX_PROCESSES)
while not stop_event.is_set():
try: # if queue isn't empty, get value from queue and create new process
var = my_queue.get_nowait() # getting value from queue
p = Process(target=my_function, args=("process-%s" % var))
p.start()
except Queue.Empty:
print "No more items in queue"
except KeyboardInterrupt as stop_test_exception:
print(" CTRL+C pressed. Stopping test....")
stop_event.set()
However I don't think this script is exactly what I want. Here's what I was looking for when I wrote the script. I want it to scan for variables in "Variables" table, add "new" variables if they don't already exists to the queue, run "my_function" for each variable in the queue.
I believe I have WAYYYY to many while not stop_event.is_set() functions. Because right now it just prints out "No more items in queue" about a million times.
Please HELP!! :)
I have 2 scripts that start multiple processes. Right now I'm opening up two different terminals and running python start.py to start both the scripts. How can I achieve this with one command, or one running one script.
Start.py 1
# globals
my_queue = multiprocessing.Manager().Queue() # queue to store our values
stop_event = multiprocessing.Event() # flag which signals processes to stop
my_pool = None
def my_function(foo):
print("starting %s" % foo)
try:
addnews.varfoo)
except Exception,e:
print str(e)
MAX_PROCESSES = 50
my_pool = multiprocessing.Pool(MAX_PROCESSES)
x = Var.objects.order_by('name').values('link')
for t in x:
t = t.values()[0]
my_pool.apply_async(my_function, args=(t,))
my_pool.close()
my_pool.join()
Start1.py 2
# globals
MAX_PROCESSES = 50
my_queue = multiprocessing.Manager().Queue() # queue to store our values
stop_event = multiprocessing.Event() # flag which signals processes to stop
my_pool = None
def my_function(var):
var.run_main(var)
stop_event.set()
def var_scanner():
# Since `t` could have unlimited size we'll put all `t` value in queue
while not stop_event.is_set(): # forever scan `values` for new items
y = Var.objects.order_by('foo).values('foo__foo')
for t in y:
t = t.values()[0]
my_queue.put(t)
try:
var_scanner_process = multiprocessing.Process(target=var_scanner)
var_scanner_process.start()
my_pool = multiprocessing.Pool(MAX_PROCESSES)
#while not stop_event.is_set():
try: # if queue isn't empty, get value from queue and create new process
var = my_queue.get_nowait() # getting value from queue
p = multiprocessing.Process(target=my_function, args=(var,))
p.start()
except Queue.Empty:
print "No more items in queue"
time.sleep(1)
#stop_event.set()
except KeyboardInterrupt as stop_test_exception:
print(" CTRL+C pressed. Stopping test....")
stop_event.set()
You can run the first script in the background on the same terminal, using the & shell modifier.
python start.py &
python start1.py
Thanks to those who helped me figure out I needed to use threading to run a loop in a control script I have run, I now have an issue to try and control the thread - by starting or stopping it based on a function:
I want to start a process to get a motor to cycle through a movement based on a 'start' parameter sent to the controlling function, also I want to send a 'stop' parameter to stop the thread too - here's where I got to:
def looper():
while True:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state):
t = threading.Thread(target=looper)
if state == 'start':
t.start()
elif state == 'stop':
t.join()
print 'looper stopped!!'
This starts the thread okay when I call looper_control('start') but throws an error when looper_control('stop'):
File "/usr/lib/python2.7/threading.py", line 657, in join
raise RuntimeError("cannot join thread before it is started")
RuntimeError: cannot join thread before it is started
EDIT: looper_control called from here
if "motor" in tmp:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
looper_control('stop')
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
looper_control('start')
UPDATE: Ive not been able to stop the thread using the method suggested - I thought I had it!
here's where I am:
class sliderControl(threading.Thread):
def __init__(self,stop_event):
super(sliderControl,self).__init__()
self.stop_event = stop_event
def run(self):
while self.stop_event:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state,stop_event):
if state == 'start':
t = sliderControl(stop_event=stop_event)
t.start()
elif state == 'stop':
#time.sleep(3)
stop_event.set()
#t.join()
print 'looper stopped!!'
called via:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
#stop_thread_event = threading.Event()
print 'stopping thread'
print stop_thread_event
looper_control('stop',stop_thread_event)
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
global stop_thread_event
stop_thread_event = threading.Event()
print stop_thread_event
looper_control('start', stop_thread_event)
It looked like a separate thread event was being called by loop and stop, so I thought a global would sort it out but its just not playing ball. When I start the loop - it runs, but when I try to stop it, I get looper stopped!! , but the process just keeps running
Your top-level thread routine will need to become an event handler that listens to a Queue object (as in from Queue import Queue) for messages, then handles them based on state. One of those messages can be a shutdown command, in which case the worker thread function simply exits, allowing the main thread to join it.
Instead of time.sleep, use threading.Timer with the body of the timer sending a message into your event queue.
This is a substantial refactoring. But especially if you plan on adding more conditions, you'll need it. One alternative is to use a package that handles this kind of thing for you, maybe pykka.
To stop a python thread you can use threading.Event()
try this:
class YourClass(threading.Thread):
def __init__(self, stop_event):
super(YourClass, self).__init__()
self.stop_event = stop_event
def run(self):
while not self.stop_event.is_set():
# do what you need here (what you had in looper)
def looper_control(state, stop_event):
if state == 'start':
t = YourClass(stop_event=stop_event)
t.start()
elif state == 'stop':
stop_event.set()
and call to looper_control:
stop_thread_event = threading.Event()
looper_control(state, stop_thread_event)
you only can "start" once a thread
but you can lock and unlock the thread.
the best way to stop and start a thread is with mutex, Example:
#!/usr/bin/python
import threading
from time import sleep
mutex2 = threading.Lock()
#This thread add values to d[]
class Hilo(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
mutex2.acquire()
#Add values to d[]
d.append("hi from Peru")
mutex2.release()
sleep(1)
d=[];
hilos = [Hilo()]
#Stop Thread
#If you have more threads you need make a mutex for every thread
mutex2.acquire()
#Start treades, but the thread is lock
for h in hilos:
h.start()
#so you need do
#unlock THREAD<
mutex2.release()
#>START THREAD
#Sleep for 4 seconds
sleep(4)
#And print d[]
print d
print "------------------------------------------"
#WAIT 5 SECONDS AND STOP THE THREAD
sleep(5)
try:
mutex2.acquire()
except Exception, e:
mutex2.release()
mutex2.acquire()
#AND PRINT d[]
print d
#AND NOW YOUR TRHEAD IS STOP#
#When the thread is lock(stop), you only need call: mutex2.release() for unlock(start)
#When your thread is unlock(start) and you want lock(stop):
#try:
# mutex2.acquire()
#except Exception, e:
# mutex2.release()
# mutex2.acquire()