This is my code:
import threading
import requests
class MyThread(threading.Thread):
def __init__(self, id):
threading.Thread.__init__(self)
self.id = id
def run(self):
print('Thread {0} is running...'.format(self.id))
req = requests.post(url='http://test.com', data={'id': id})
if req.json()['success']:
# stop other threads
print('Thread {0} is finished!'.format(self.id))
for id in range(0, 480):
t = MyThread(id)
t.start()
In every thread I send lots of requests. I want to stop other threads which are made in for loop, if the condition was true.
How to do that?!
Related
I have a list of strings in Queue
queue = ["First","Second","Third","Fourth","",etc]
Snippet Of The Code :
import thread
from thread import Threading
import Queue
import time
class MainThread(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
Details = self.queue.get()
Trail = Details
#..........#
# Now Mark If Trail Can Be Use Again Or Not
self.queue.put(Trail) # Now Put Them Back No Matter What
self.queue.task_done()
queue = Queue.Queue(maxsize=0)
while not #...........#
for i in range(TotalThreads):
try:
t = ThreadingPower(queue)
t.setDaemon(False)
t.start()
except:
time.sleep(5)
My question : if the string "First" from queue can't be used again,how can i mark it for a specific time,so the thread can continue grabbing other and ignore it until the specific time is over
Edit :
Currently the solution i have is
queue = [["First",1],["Second",1],["Third",1],["Fourth",1],etc]
class MainThread(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
Details = self.queue.get()
Trail,State = Details
if State == 1:
#................#
if #.............#:
State = 0
self.queue.put((Trail,State)) # Now Put Them Back No Matter What
self.queue.task_done()
queue = Queue.Queue(maxsize=0)
while not #...........#
for i in range(TotalThreads):
try:
t = ThreadingPower(queue)
t.setDaemon(False)
t.start()
except:
time.sleep(5)
However this only works if i want to disable them permanently,i need a solution where i can disable them for a specific time
I've implemented some threaded application using python. During runtime i want to catch the CTRL+C sigcall and exit the program. To do that I've registered a function called exit_gracefully which also takes care of stopping the threads in a more controlled way. However, it does not seem to work. It seems the handler is never called
Here's the example I'm working with:
import Queue
import threading
import signal
import sys
import time
queue = Queue.Queue()
workers = list()
def callback(id, item):
print("{}: {}".format(id, item))
time.sleep(1)
def exit_gracefully(signum, frame):
print("Ctrl+C was pressed. Shutting threads down ...")
print("Stopping workers ...")
for worker in workers:
worker.stop()
sys.exit(1)
class ThreadedTask(threading.Thread):
def __init__(self, id, queue, callbacks):
threading.Thread.__init__(self)
self._stop_event = threading.Event()
self.id = str(id)
self.queue = queue
self.callbacks = callbacks
self._stopped = False
def run(self):
while not self.stopped():
item = self.queue.get()
for callback in self.callbacks:
callback(self.id, item)
self.queue.task_done()
def stop(self):
self._stop_event.set()
self._stopped = True
def stopped(self):
return self._stop_event.is_set() or self._stopped
def main(input_file, thread_count, callbacks):
print("Initializing queue ...")
queue = Queue.Queue()
print("Parsing '{}' ...".format(input_file))
with open(input_file) as f:
for line in f:
queue.put(line.replace("\n", ""))
print("Initializing {} threads ...".format(thread_count))
for id in range(thread_count):
worker = ThreadedTask(id, queue, callbacks)
worker.setDaemon(True)
workers.append(worker)
print("Starting {} threads ...".format(thread_count))
for worker in workers:
worker.start()
queue.join()
if __name__ == '__main__':
signal.signal(signal.SIGINT, exit_gracefully)
print("Starting main ...")
input_file = "list.txt"
thread_count = 10
callbacks = [
callback
]
main(input_file, thread_count, callbacks)
If you want to try the example above you may generate some test-data first:
seq 1 10000 > list.txt
Any help is appreciated!
Here's a solution that seems to work.
One issue is that Queue.get() will ignore SIGINT unless a timeout is set. That's documented here: https://bugs.python.org/issue1360.
Another issue is that Queue.join() also seems to ignore SIGINT. I worked around that by polling the queue in a loop to see if it's empty.
These issues appear to have been fixed in Python 3.
I also added a shared event that's used in the SIGINT handler to tell all the threads to shut down.
import Queue
import signal
import sys
import threading
import time
def callback(id, item):
print '{}: {}'.format(id, item)
time.sleep(1)
class ThreadedTask(threading.Thread):
def __init__(self, id, queue, run_event, callbacks):
super(ThreadedTask, self).__init__()
self.id = id
self.queue = queue
self.run_event = run_event
self.callbacks = callbacks
def run(self):
queue = self.queue
while not self.run_event.is_set():
try:
item = queue.get(timeout=0.1)
except Queue.Empty:
pass
else:
for callback in self.callbacks:
callback(self.id, item)
queue.task_done()
def main():
queue = Queue.Queue()
run_event = threading.Event()
workers = []
def stop():
run_event.set()
for worker in workers:
# Allow worker threads to shut down completely
worker.join()
def sigint_handler(signum, frame):
print '\nShutting down...'
stop()
sys.exit(0)
signal.signal(signal.SIGINT, sigint_handler)
callbacks = [callback]
for id in range(1, 11):
worker = ThreadedTask(id, queue, run_event, callbacks)
workers.append(worker)
for worker in workers:
worker.start()
with open('list.txt') as fp:
for line in fp:
line = line.strip()
queue.put(line)
while not queue.empty():
time.sleep(0.1)
# Update: Added this to gracefully shut down threads after all
# items are consumed from the queue.
stop()
if __name__ == '__main__':
main()
Am very much new to python. So what i might be asking may not be correct. What am looking to do is. Create a thread from mainss and start the thread. When the thread is started i want it to access the variable of the mainss class from where thread is created and Modify the variable values. And i want the execution of mainss to sleep until thread modifies one of its variable value. How can i achieve this? Here is the code i tried below. Comment in the code of mythread.py class is where I need to modify the value of count variable of mainss class
main.py
#!/usr/bin/python
import time
from myThread import myThread
class mainss():
def __init__(self):
print "s"
def callThread(self):
global count
count = 1
# Create new threads
thread1 = myThread(1, "Thread-1", 1, count)
thread1.start()
# time.sleep(10) until count value is changed by thread to 3
print "Changed Count value%s " % count
print "Exiting"
m = mainss()
m.callThread()
myThread.py
#!/usr/bin/python
import threading
import time
exitFlag = 0
class myThread (threading.Thread):
def __init__(self, threadID, name, counter, count):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
self.count = count
def run(self):
print_time(self.name, 1, 5, self.count)
def print_time(threadName, delay, counter, count):
from main import mainss
while counter:
if exitFlag:
threadName.exit()
time.sleep(delay)
count = count + 1
print "count %s" % (count)
# here i want to modify count of mainss class
counter -= 1
Thanks in advance
Using Multiprocessing, a Manager dictionary is used to communicate between or to processes https://pymotw.com/3/multiprocessing/communication.html#managing-shared-state Note that the Manager dictionary can be changed while the process is running. Multiprocessing also has a wait feature https://pymotw.com/3/multiprocessing/communication.html#signaling-between-processes
I would use a threading.Event and a Queue
Something like this, (Please note that i didn't test this myself and obviously you gonna have to make some changes.)
main.py
import Queue
import threading
from myThread import myThread
class mainss:
def __init__(self):
self.queue = Queue.Queue()
self.event = threading.Event()
def callThread(self):
self.queue.put(1) # Put a value in the queue
t = myThread(self.queue, self.event)
t.start()
self.event.wait() # Wait for the value to update
count = self.queue.get()
print "Changed Count value %s" % count
if __name__ == '__main__':
m = mainss()
m.callThread()
myThread.py
import threading
class myThread(threading.Thread):
def __init__(self, queue, event):
super(myThread, self).__init__()
self.queue = queue
self.event = event
def run(self):
while True:
count = self.queue.get() # Get the value (1)
count += 1
print "count %s" % (count)
self.queue.put(count) # Put updated value
self.event.set() # Notify main thread
break
I'm having problem with QThreads in python.
I would like to start my multi QThread when I push on button Run.
But the compiler outputs following error:
"QThread: Destroyed while thread is still running"
I don't know what is wrong with my code.
Any help would be appreciated.
Here is my code:
# -*- coding: utf-8 -*-
from PySide import QtCore, QtGui
from Ui_MainWindow import Ui_MainWindow
from queue import Queue
import sys, re, random
import time, random, re, urllib.request
from urllib.parse import urljoin
class Worker(QtCore.QThread):
def __init__(self,threadID, name, q, delay):
QtCore.QThread.__init__(self)
self.threadID = threadID
self.name = name
self.q = q
self.delay = delay
self._running = False
def run(self):
self._running = True
print ("start - %s" %self.name)
while self._running:
req = self.request(self.name, self.q, self.delay)
def stop(self, wait=False):
print (self.name)
self._running = False
def request(self, threadName, q1, delay):
while not self.q.empty():
time.sleep(delay)
q = q1.get()
print ("%s: %s %s %s" % (threadName, time.ctime(time.time()), q, delay))
if self.q.empty():
print ("queue empty")
self.stop()
class MainWindow(QtGui.QMainWindow):
def __init__(self, parent=None):
QtGui.QMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.backend = Queue()
self.connect(self.ui.actionStart, QtCore.SIGNAL('triggered()'),self.start)
def start(self):
try :
f1 = open('./payload/backend.log')
except FileNotFoundError as e:
return
threadList = ["Thread-1", "Thread-2", "Thread-3", "Thread-4", "Thread-5"]
self.url = "http://test.com"
self.threads = []
threadID = 1
for payload in f1.read().splitlines() :
full_url = urljoin(self.url, payload)
self.backend_dir.put(full_url)
for tName in threadList:
ran_int = random.randint(1, 2)
downloader = Worker(threadID, tName, self.backend, ran_int)
downloader.start()
self.threads.append(downloader)
if __name__ == "__main__":
app = QtGui.QApplication(sys.argv)
window = MainWindow()
window.show()
sys.exit(app.exec_())
log
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
You are trying to do something that is a built-in of Qt: a QThreadPool.
I would advice you to read the doc and use it instead.
If you really want to use QThread:
You should not subclass QThread. Instead you should subclass the basic QObject to create your worker and use the moveToThread method:
class WorkerSignals(QObject):
finished = pyqtSignal()
class Worker(QObject):
def __init__():
self.signal = WorkerSignals()
def run(self):
# Your stuff
print('running')
self.signal.finished.emit()
Then somewhere else:
thread = QThread()
worker = Worker(...)
worker.moveToThread(thread)
thread.started.connect(worker.run)
worker.finished.connect(thread.quit)
worker.finished.connect(worker.deleteLater)
thread.finished(thread.deleteLater)
thread.start()
The solution is a rough translation of this one in C++:
https://mayaposch.wordpress.com/2011/11/01/how-to-really-truly-use-qthreads-the-full-explanation/
Hope this helps!
The problem is caused by the way you are using the queue.
All the threads start and begin their tasks normally, up until the queue becomes empty. At that point, the first thread to finish terminates, but the other four threads are left waiting for an item to be returned from the queue, which never comes.
This is because you use get with no arguments, which will block indefinitely until an item becomes available. Instead, you should use get_nowait, and also call stop() unconditionally at the end of request():
from queue import Queue, Empty
...
class Worker(QtCore.QThread):
...
def request(self, threadName, q1, delay):
while not q1.empty():
time.sleep(delay)
try:
q = q1.get_nowait()
except Empty:
break
else:
print ("%s: %s %s %s" % (threadName, time.ctime(time.time()), q, delay))
print ("queue empty")
self.stop()
I believe that you need to call self.threads.append(downloader) before downloader.start() so that the thread doesn't go out of scope and get garbage collected.
Below is the code that I have that downloads various URLS into each separate thread, I was in attempt to make some changes before I implement the thread pool but with this change the queue is coming to be empty and download is not beginning.
import Queue
import urllib2
import os
import utils as _fdUtils
import signal
import sys
import time
import threading
class ThreadedFetch(threading.Thread):
""" docstring for ThreadedFetch
"""
def __init__(self, queue, out_queue):
super(ThreadedFetch, self).__init__()
self.queueItems = queue.get()
self.__url = self.queueItems[0]
self.__saveTo = self.queueItems[1]
self.outQueue = out_queue
def run(self):
fileName = self.__url.split('/')[-1]
path = os.path.join(DESKTOP_PATH, fileName)
file_size = int(_fdUtils.getUrlSizeInBytes(self.__url))
while not STOP_REQUEST.isSet():
urlFh = urllib2.urlopen(self.__url)
_log.info("Download: %s" , fileName)
with open(path, 'wb') as fh:
file_size_dl = 0
block_sz = 8192
while True:
buffer = urlFh.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
fh.write(buffer)
status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
status = status + chr(8)*(len(status)+1)
sys.stdout.write('%s\r' % status)
time.sleep(.05)
sys.stdout.flush()
if file_size_dl == file_size:
_log.info("Download Completed %s%% for file %s, saved to %s",
file_size_dl * 100. / file_size, fileName, DESKTOP_PATH)
below is the main function that does the call and initiation.
def main(appName):
args = _fdUtils.getParser()
urls_saveTo = {}
# spawn a pool of threads, and pass them queue instance
# each url will be downloaded concurrently
for i in range(len(args.urls)):
t = ThreadedFetch(queue, out_queue)
t.daemon = True
t.start()
try:
for url in args.urls:
urls_saveTo[url] = args.saveTo
# urls_saveTo = {urls[0]: args.saveTo, urls[1]: args.saveTo, urls[2]: args.saveTo}
# populate queue with data
for item, value in urls_saveTo.iteritems():
queue.put([item, value])
# wait on the queue until everything has been processed
queue.join()
print '*** Done'
except (KeyboardInterrupt, SystemExit):
lgr.critical('! Received keyboard interrupt, quitting threads.')
You create the queue and then the first thread which immediately tries to fetch an item from the still empty queue. The ThreadedFetch.__init__() method isn't run asynchronously, just the run() method when you call start() on a thread object.
Store the queue in the __init__() and move the get() into the run() method. That way you can create all the threads and they are blocking in their own thread, giving you the chance to put items into the queue in the main thread.
class ThreadedFetch(threading.Thread):
def __init__(self, queue, out_queue):
super(ThreadedFetch, self).__init__()
self.queue = queue
self.outQueue = out_queue
def run(self):
url, save_to = self.queue.get()
# ...
For this example the queue is unnecessary by the way as every thread gets exactly one item from the queue. You could pass that item directly to the thread when creating the thread object:
class ThreadedFetch(threading.Thread):
def __init__(self, url, save_to, out_queue):
super(ThreadedFetch, self).__init__()
self.url = url
self.save_to = save_to
self.outQueue = out_queue
def run(self):
# ...
And when the ThreadedFetch class really just consists of the __init__() and run() method you may consider moving the run() method into a function and start that asynchronously.
def fetch(url, save_to, out_queue):
# ...
# ...
def main():
# ...
thread = Thread(target=fetch, args=(url, save_to, out_queue))
thread.daemon = True
thread.start()