We Keep Coding

Send file pointer to python thread and update file pointer

I have a python program with a thread and the thread should write into a file. I will spawn a thread from the main program. Now on new day trigger I will change the file pointer in the main program and I want the thread also to take the new file to write the data to the file.
I have a code which will take global variable and do this task. But is there any other better way of doing this?
#!/usr/bin/env python
import sys
import threading
import time
filePtr = None
import time
def fileWriteTh():
global filePtr
time.sleep(2)
filePtr.write("from the thrread this should in file 2")
def main():
global filePtr
filePtr = open("test1.txt","ab")
fileThread = threading.Thread(target=fileWriteTh)
fileThread.start()
if new_day_trigger:
filePtr.close()
filePtr = open("test2.txt","ab")
fileThread.join()
if __name__ == "__main__":
main()
This is the new code that is written:
#!/usr/bin/env python
import sys
import threading
import time
class SendPacket(object):
fileDesc = None
def __init__(self, fd):
super(SendPacket, self).__init__()
SendPacket.fileDesc = fd
def printFromInstance(self,var):
print var
SendPacket.fileDesc.write(var)
time.sleep(3)
print var
SendPacket.fileDesc.write(var)
def startabc(self, someVar):
self.printFromInstance(someVar)
#classmethod
def printVar(cls, printStr):
print printStr
cls.fileDesc.write(printStr)
#classmethod
def changeClsFile(cls, newFd):
cls.fileDesc = newFd
def main():
filePtr = open("test1.txt","ab")
sendPack_inst = SendPacket(filePtr)
fileThread = threading.Thread(target=sendPack_inst.startabc, args=("test1",))
fileThread.start()
time.sleep(2)
filePtr.close()
filePtr = open("test2.txt","ab")
SendPacket.changeClsFile(filePtr)
fileThread.join()
filePtr.close()
if __name__ == "__main__":
main()

Like this:
#!/usr/bin/env python
import sys
import thread
import time
class _fileACT :
def __init__(self):
self.trigger = 0
self.flag = True
self.msg = ""
self.files = (open("test1.txt","ab"),open("test2.txt","ab"))
def run(self,pssrg):
while self.flag :
if self.msg != "" :
self.files[self.trigger].write(self.msg)
self.msg = ""
def test(self,pssrg):
for i in range(20):
time.sleep(1)
if i %2 != 0 :
self.trigger = 0
elif i %2 != 1:
self.trigger = 1
self.msg = "%0.3d test-1,asdasdasd\n"%i
time.sleep(0.5)
print "wait..."
self.flag = False
for e in self.files : e.close()
print "can exit !"
if __name__ == "__main__":
fileACT = _fileACT()
thread.start_new_thread(fileACT.run,(None,))
thread.start_new_thread(fileACT.test,(None,))
We have three variables, filename, last opened file name and message. Two files, only False and True will be sufficient (of course you can use index for multiple files). We've written a test function into the class because we don't want our main cycle to freeze. The file selection is done with ' trigger ', but the previous and next file name is not the same, the previous closes.
The important point in the thread is that the time delay is strictly unavailable! The time delay is always applied to the trigger. The time delay cannot be placed in the main loop. An instance of access from outside the class is also attached. I hope it helps.

Python - "can't pickle thread.lock" error when creating a thread under a multiprocess in Windows

I'm getting stuck on what I think is a basic multiprocess and threading issue. I've got a multiprocess set up, and within this a thread. However, when I set up the thread class within the init function, I get the following error:
"TypeError: can't pickle thread.lock objects".
However, this does not happen if the thread is set up outside of the init function. Does anyone know why this is happening? Note I'm using Windows.
Some code is below to illustrate the issue. As typed below, it runs fine. However if print_hello() is called from within the DoStuff init def, then the error occurs, if it's called within the multi-process run() def then it's fine.
Can anyone point me in the right direction so it runs fine when called from init? thanks!
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("starting DoStuff")
# This works fine if the line below is uncommented and __init__ self.print_hello() is commented...
self.dostuff.print_hello()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
# If the following is uncommented, the error occurs...
# Note it also occurs if the lines in start_thead are pasted here...
# self.print_hello()
def print_hello(self):
print "hello"
self.start_thread()
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("Starting MyThread")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
# mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

It looks like there is no simple answer, and it appears to be a restriction of Windows (Win 7, python 3.6 in my case); on Windows it looks like you need to start the process before you can start the worker thread inside the owned object.
There appears to be no such restriction on Unix (CentOS 7, python 2.7.5).
As an experiment I modified your code as follows; this version checks the OS and starts either the process first, or the thread first:
import multiprocessing
import threading
import time
import os
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff(self)
def run(self):
print("MyProcess.run()")
print("MyProcess.ident = " + repr(self.ident))
if os.name == 'nt':
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, owner, **kwargs):
super(DoStuff, self).__init__(**kwargs)
self.owner = owner
if os.name != 'nt':
self.start_thread()
def start_thread(self):
print("DoStuff.start_thread()")
self.my_thread_instance = MyThread(self)
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self, owner):
super(MyThread, self).__init__()
self.owner = owner
def run(self):
print("MyThread.run()")
print("MyThread.ident = " + repr(self.ident))
print("MyThread.owner.owner.ident = " + repr(self.owner.owner.ident))
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
... and got the following on Windows, where the process starts first:
MyProcess.run()
MyProcess.ident = 14700
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 14220
MyThread.owner.owner.ident = 14700
... and the following on Linux, where the thread is started first:
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 140316342347520
MyThread.owner.owner.ident = None
MyProcess.run()
MyProcess.ident = 4358
If it were my code I'd be tempted to always start the process first, then create the thread within that process; the following version works fine for me across both platforms:
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("MyProcess.run()")
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("MyThread.run()")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

Modifying and acessing of variables of a different class by thread in python

Am very much new to python. So what i might be asking may not be correct. What am looking to do is. Create a thread from mainss and start the thread. When the thread is started i want it to access the variable of the mainss class from where thread is created and Modify the variable values. And i want the execution of mainss to sleep until thread modifies one of its variable value. How can i achieve this? Here is the code i tried below. Comment in the code of mythread.py class is where I need to modify the value of count variable of mainss class
main.py
#!/usr/bin/python
import time
from myThread import myThread
class mainss():
def __init__(self):
print "s"
def callThread(self):
global count
count = 1
# Create new threads
thread1 = myThread(1, "Thread-1", 1, count)
thread1.start()
# time.sleep(10) until count value is changed by thread to 3
print "Changed Count value%s " % count
print "Exiting"
m = mainss()
m.callThread()
myThread.py
#!/usr/bin/python
import threading
import time
exitFlag = 0
class myThread (threading.Thread):
def __init__(self, threadID, name, counter, count):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
self.count = count
def run(self):
print_time(self.name, 1, 5, self.count)
def print_time(threadName, delay, counter, count):
from main import mainss
while counter:
if exitFlag:
threadName.exit()
time.sleep(delay)
count = count + 1
print "count %s" % (count)
# here i want to modify count of mainss class
counter -= 1
Thanks in advance

Using Multiprocessing, a Manager dictionary is used to communicate between or to processes https://pymotw.com/3/multiprocessing/communication.html#managing-shared-state Note that the Manager dictionary can be changed while the process is running. Multiprocessing also has a wait feature https://pymotw.com/3/multiprocessing/communication.html#signaling-between-processes

I would use a threading.Event and a Queue
Something like this, (Please note that i didn't test this myself and obviously you gonna have to make some changes.)
main.py
import Queue
import threading
from myThread import myThread
class mainss:
def __init__(self):
self.queue = Queue.Queue()
self.event = threading.Event()
def callThread(self):
self.queue.put(1) # Put a value in the queue
t = myThread(self.queue, self.event)
t.start()
self.event.wait() # Wait for the value to update
count = self.queue.get()
print "Changed Count value %s" % count
if __name__ == '__main__':
m = mainss()
m.callThread()
myThread.py
import threading
class myThread(threading.Thread):
def __init__(self, queue, event):
super(myThread, self).__init__()
self.queue = queue
self.event = event
def run(self):
while True:
count = self.queue.get() # Get the value (1)
count += 1
print "count %s" % (count)
self.queue.put(count) # Put updated value
self.event.set() # Notify main thread
break

How can I stop a long-running function when it is called multiple times?

Below I have an example program. When the button is pressed, it takes a second before it can calculate the value to show. If the user presses the button in rapid succession they end up waiting a long time to see the last answer, which is the only answer they care about. In the code, you can see that the _dataCruncher function needs to know self._count, but self._count does not depend on the output of _dataCruncher.
My question, therefore, is how can I interrupt the normal execution of _dataCruncher on subsequent calls in order to keep the GUI free to do other stuff, and to not waste processing time when it is not needed? I realize that I will likely need to use a thread to run _dataCruncher and some sort of Queue to get the appropriate val to display, but I do not understand how to put this all together.
from PyQt4 import QtGui, QtCore
import sys
import time
import random
import random
class MainWindow(QtGui.QMainWindow):
def __init__(self):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _dataCruncher(self, val):
time.sleep(1) # takes a long time to process data using val
return val * random.randint(1,10)
def _buttonClicked(self):
self.count += 1
val = self._dataCruncher(self.count)
self.text.setText('Value {}'.format(val))
def startup(self):
self.show()
result = self.app.exec_()
sys.exit(result)
if __name__ == '__main__':
random.seed()
myWindow = MainWindow()
myWindow.startup()

So, finding an answer to this was more complicated than I thought. As #MTset mentions in one of the comments, python does not offer any means by which to cancel the execution of a Thread. So, what I did was create a 'threadHandler' class that, well, handles thread. It keeps track of the last thread that was created and offers a means by which to get the result from the execution of the last thread.
I am posting an modified version of the test code from the original post as well as the threadHandler code in full in case anyone has use for it.
File 1 here
# tester.py, run this file
from PyQt4 import QtGui, QtCore
import random, sys, time
from threadHandler import MyThreadHandler
class MyModel(object):
def dataCruncher(self, val):
delay = random.randint(1,5)
print('{} sleeping for {}'.format(val, delay))
time.sleep(delay) # takes a long time to process data using val
print('{} done sleeping'.format(val))
return val
class MainWindow(QtGui.QMainWindow):
def __init__(self, threadHandler):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
self.button_clicked_events = Event()
self.threadHandler = threadHandler
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _buttonClicked(self):
self.count += 1
self.button_clicked_events(self.count)
def setLabel(self, val):
self.text.setText(str(val))
def startup(self):
self.show()
result = self.app.exec_()
return result
class Event(list):
"""Event subscription.
A list of callable objects. Calling an instance of this will cause a
call to each item in the list in ascending order by index.
Example Usage:
>>> def f(x):
... print 'f(%s)' % x
>>> def g(x):
... print 'g(%s)' % x
>>> e = Event()
>>> e()
>>> e.append(f)
>>> e(123)
f(123)
>>> e.remove(f)
>>> e()
>>> e += (f, g)
>>> e(10)
f(10)
g(10)
>>> del e[0]
>>> e(2)
g(2)
"""
def __init__(self):
self.output = {}
def __call__(self, *args, **kwargs):
for f,key in self:
output = f(*args, **kwargs)
self.output[key] = output
return self.output
def __repr__(self):
return "Event({})".format(list.__repr__(self))
if __name__ == '__main__':
def checker(handler, window):
if handler.isLastDone():
val = handler.getLastResult()
window.setLabel(val)
else:
window.setLabel('calculating...')
random.seed()
model = MyModel()
threadHandler = MyThreadHandler()
myWindow = MainWindow(threadHandler)
threadHandler.createTimer(1, checker, threadHandler, myWindow)
def getData(count):
threadHandler.createOneShot(model.dataCruncher, count)
myWindow.button_clicked_events.append((getData, 'dt'))
result = myWindow.startup()
print('ending')
threadHandler.end()
print('ended')
sys.exit(result)
File 2 below
#threadHandler.py, save this file in the same folder as tester.py
import threading, time
class MyThreadHandler(object):
def __init__(self):
self.oneShots = []
self.timers = []
self.oldOneShots = []
self.latest = None
self.cleaning = False
self._startCleaner()
def _startCleaner(self):
print('-'*20+'Starting cleaner'+'-'*20)
self.cleaner = self.createTimer(1, self._cleanupThreads)
def _stopCleaner(self):
print('-'*20+'Stopping cleaner'+'-'*20)
self.cleaner.stop()
def getNumThreads(self):
return len(self.oneShots)
def getNumOldThreads(self):
return len(self.oldOneShots)
def end(self):
for i,timer in enumerate(self.timers):
timer.stop()
self.timers.pop(i)
def createTimer(self, interval, func, *args, **kwargs):
timer = myTimer(interval, func, args, kwargs)
self.timers.append(timer)
return timer
def createOneShot(self, func, *args, **kwargs):
oneshot = myOneShot(func, args, kwargs)
self.oneShots.append(oneshot)
self.latest = oneshot
def isLastDone(self):
if not self.latest is None:
return not self.latest.running()
else:
return None
def getLastResult(self):
if self.latest is None:
raise ValueError('There have not been any oneshots created.')
while self.latest.running():
pass
result = self.latest.getResult()
if len(self.oneShots) > 0:
self.oldOneShots.append(myOneShot(self._cleanAll, (self.oneShots,)))
self.oneShots = []
return result
def _cleanAll(self, toClean):
# loop through toClean and pop up anything that's done. this DOES lock
while len(toClean) > 0:
toClean = self._cleanup(toClean)
def _cleanup(self, toCleanup):
while not self.cleaning:
self.cleaning = True
for i, thread in enumerate(toCleanup):
if not thread.running():
toCleanup.pop(i)
self.cleaning = False
return toCleanup
def _cleanupThreads(self):
# check each of these lists and pop out any threads that are done. This
# does not lock. This function should really only be called by the
# cleaner, which is set up in __init__
self.oneShots = self._cleanup(self.oneShots)
self.timers = self._cleanup(self.timers)
self.oldOneShots = self._cleanup(self.oldOneShots)
class myTimer(object):
def __init__(self, delay, func, args=tuple(), kwargs={}):
self.delay = delay
self.func = func
self.loop = True
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
while self.loop:
self.output = self.func(*self.args, **self.kwargs)
if self.delay > 0.1:
count = 0
while count <= self.delay:
count += 0.1
time.sleep(0.1)
else:
time.sleep(self.delay)
def stop(self):
self.loop = False
def running(self):
return self.loop
def getResult(self):
return self.output
class myOneShot(object):
def __init__(self, func, args=tuple(), kwargs={}):
self.func = func
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
self.output = self.func(*self.args, **self.kwargs)
def running(self):
return self.thread.is_alive()
def getResult(self):
return self.output
if __name__ == '__main__':
import random
random.seed()
def longFunc(num):
delay = random.randint(5,8)
if num in (3, 6):
delay = 2
print('-'*30+'func {} has sleep {}'.format(num, delay))
time.sleep(delay)
print('-'*30+'func {} is done'.format(num))
return num
def checker(handler):
if handler.isLastDone():
return handler.getLastResult()
else:
return None
myHandler = MyThreadHandler()
# The 'checker' function simulates something in my program that uses the
# data generated by the 'longFunc'. It waits until there are no more threads
# in the threadHandler, as that would indicate that the user is done
# switching back-and-forth between different values
checkTimer = myHandler.createTimer(1, checker, myHandler)
# create 10 one-shot threads that take a 'long' time. The delay is to keep
# them in order, as this loop is meant to simulate a user switching between
# items using a keyboard or mouse, which I imagine they couldn't do any
# faster than every 1/10th of a second
start = time.time()
for i in range(4):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
# wait until there are no more threads executing
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
for i in range(4, 7):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
while myHandler.getNumOldThreads() >0 or myHandler.getNumThreads() > 0:
pass
myHandler.end()
print('done ending')

You could disable the button after it's pressed until an answer is ready using:
setEnabled(False)
Then reset it just before providing the result.

Cancellable threading.Timer in Python

I am trying to write a method that counts down to a given time and unless a restart command is given, it will execute the task. But I don't think Python threading.Timer class allows for timer to be cancelable.
import threading
def countdown(action):
def printText():
print 'hello!'
t = threading.Timer(5.0, printText)
if (action == 'reset'):
t.cancel()
t.start()
I know the above code is wrong somehow. Would appreciate some kind guidance over here.

You would call the cancel method after you start the timer:
import time
import threading
def hello():
print "hello, world"
time.sleep(2)
t = threading.Timer(3.0, hello)
t.start()
var = 'something'
if var == 'something':
t.cancel()
You might consider using a while-loop on a Thread, instead of using a Timer.
Here is an example appropriated from Nikolaus Gradwohl's answer to another question:
import threading
import time
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
self.count = 10
def run(self):
while self.count > 0 and not self.event.is_set():
print self.count
self.count -= 1
self.event.wait(1)
def stop(self):
self.event.set()
tmr = TimerClass()
tmr.start()
time.sleep(3)
tmr.stop()

I'm not sure if I understand correctly. Do you want to write something like in this example?
>>> import threading
>>> t = None
>>>
>>> def sayHello():
... global t
... print "Hello!"
... t = threading.Timer(0.5, sayHello)
... t.start()
...
>>> sayHello()
Hello!
Hello!
Hello!
Hello!
Hello!
>>> t.cancel()
>>>

The threading.Timer class does have a cancel method, and although it won't cancel the thread, it will stop the timer from actually firing. What actually happens is that the cancel method sets a threading.Event, and the thread actually executing the threading.Timer will check that event after it's done waiting and before it actually executes the callback.
That said, timers are usually implemented without using a separate thread for each one. The best way to do it depends on what your program is actually doing (while waiting for this timer), but anything with an event loop, like GUI and network frameworks, all have ways to request a timer that is hooked into the eventloop.

Im not sure if best option but for me is woking like this:
t = timer_mgr(.....) append to list "timers.append(t)" and then after all created you can call:
for tm in timers:#threading.enumerate():
print "********", tm.cancel()
my timer_mgr() class is this:
class timer_mgr():
def __init__(self, st, t, hFunction, id, name):
self.is_list = (type(st) is list)
self.st = st
self.t = t
self.id = id
self.hFunction = hFunction
self.thread = threading.Timer(t, self.handle_function, [id])
self.thread.name = name
def handle_function(self, id):
if self.is_list:
print "run_at_time:", datetime.now()
self.hFunction(id)
dt = schedule_fixed_times(datetime.now(), self.st)
print "next:", dt
self.t = (dt-datetime.now()).total_seconds()
else:
self.t = self.st
print "run_every", self.t, datetime.now()
self.hFunction(id)
self.thread = threading.Timer(self.t, self.handle_function, [id])
self.thread.start()
def start(self):
self.thread.start()
def cancel(self):
self.thread.cancel()

Inspired by above post.
Cancelable and Resetting Timer in Python. It uses thread.
Features: Start, Stop, Restart, callback function.
Input: Timeout, sleep_chunk values, and callback_function.
Can use or inherit this class in any other program. Can also pass arguments to the callback function.
Timer should respond in middle also. Not just after completion of full sleep time. So instead of using one full sleep, using small chunks of sleep and kept checking event object in loop.
import threading
import time
class TimerThread(threading.Thread):
def __init__(self, timeout=3, sleep_chunk=0.25, callback=None, *args):
threading.Thread.__init__(self)
self.timeout = timeout
self.sleep_chunk = sleep_chunk
if callback == None:
self.callback = None
else:
self.callback = callback
self.callback_args = args
self.terminate_event = threading.Event()
self.start_event = threading.Event()
self.reset_event = threading.Event()
self.count = self.timeout/self.sleep_chunk
def run(self):
while not self.terminate_event.is_set():
while self.count > 0 and self.start_event.is_set():
# print self.count
# time.sleep(self.sleep_chunk)
# if self.reset_event.is_set():
if self.reset_event.wait(self.sleep_chunk): # wait for a small chunk of timeout
self.reset_event.clear()
self.count = self.timeout/self.sleep_chunk # reset
self.count -= 1
if self.count <= 0:
self.start_event.clear()
#print 'timeout. calling function...'
self.callback(*self.callback_args)
self.count = self.timeout/self.sleep_chunk #reset
def start_timer(self):
self.start_event.set()
def stop_timer(self):
self.start_event.clear()
self.count = self.timeout / self.sleep_chunk # reset
def restart_timer(self):
# reset only if timer is running. otherwise start timer afresh
if self.start_event.is_set():
self.reset_event.set()
else:
self.start_event.set()
def terminate(self):
self.terminate_event.set()
#=================================================================
def my_callback_function():
print 'timeout, do this...'
timeout = 6 # sec
sleep_chunk = .25 # sec
tmr = TimerThread(timeout, sleep_chunk, my_callback_function)
tmr.start()
quit = '0'
while True:
quit = raw_input("Proceed or quit: ")
if quit == 'q':
tmr.terminate()
tmr.join()
break
tmr.start_timer()
if raw_input("Stop ? : ") == 's':
tmr.stop_timer()
if raw_input("Restart ? : ") == 'r':
tmr.restart_timer()