We Keep Coding

Handling kill events for python multiprocessing processes

For a program that should run both on Linux and Windows (python 2.7), I'm trying to update values of a given object using multiprocessing.Process (while the main program is running, I'm calling the update class by a separate process).
Sometimes it takes too long before my object is updated, so I want to be able to kill my update process, and to continue with the main program. "Too long" is not strictly defined here, but rather a subjective perception of the user.
For a single queue (as in the MyFancyClass example in http://pymotw.com/2/multiprocessing/communication.html) I can kill the update process and the main program continues as I want.
However, when I make a second queue to retrieve the updated object, ending the update process does not allow me to continue in the main program.
What I have so far is:
import multiprocessing
import time, os
class NewParallelProcess(multiprocessing.Process):
def __init__(self, taskQueue, resultQueue, processName):
multiprocessing.Process.__init__(self)
self.taskQueue = taskQueue
self.resultQueue = resultQueue
self.processName = processName
def run(self):
print "pid %s of process that could be killed" % os.getpid()
while True:
next_task = self.taskQueue.get()
if next_task is None:
# poison pill for terminate
print "%s: exiting" % self.processName
self.taskQueue.task_done()
break
print "%s: %s" % (self.processName, next_task)
answer = next_task()
self.taskQueue.task_done()
self.resultQueue.put(answer)
return
class OldObject(object):
def __init__(self):
self.accurate = "OldValue"
self.otherValue = "SomeOtherValue"
class UpdateObject(dict):
def __init__(self, objectToUpdate):
self.objectToUpdate = objectToUpdate
def __call__(self):
returnDict = {}
returnDict["update"] = self.updateValue("NewValue")
return returnDict
def __str__(self):
return "update starting"
def updateValue(self, updatedValue):
for i in range(5):
time.sleep(1) # updating my object - time consuming with possible pid kill
print "working... (pid=%s)" % os.getpid()
self.objectToUpdate.accurate = updatedValue
return self.objectToUpdate
if __name__ == '__main__':
taskQueue = multiprocessing.JoinableQueue()
resultQueue = multiprocessing.Queue()
newProcess = NewParallelProcess(taskQueue, resultQueue, processName="updateMyObject")
newProcess.start()
myObject = OldObject()
taskQueue.put(UpdateObject(myObject))
# poison pill for NewParallelProcess loop and wait to finish
taskQueue.put(None)
taskQueue.join()
# get back results
results = resultQueue.get()
print "Values have been updated"
print "---> %s became %s" % (myObject.accurate, results["update"].accurate)
Any suggestions on how to kill the newProcess and to continue in the main program?
Well, made some modifications, and this does what I want. Not sure whether it is the most efficient, so any improvements are always welcome :)
import multiprocessing
import time, os
class NewParallelProcess(multiprocessing.Process):
def __init__(self, taskQueue, resultQueue, processName):
multiprocessing.Process.__init__(self)
self.taskQueue = taskQueue
self.resultQueue = resultQueue
self.name = processName
def run(self):
print "Process %s (pid = %s) added to the list of running processes" % (self.name, self.pid)
next_task = self.taskQueue.get()
self.taskQueue.task_done()
self.resultQueue.put(next_task())
return
class OldObject(object):
def __init__(self):
self.accurate = "OldValue"
self.otherValue = "SomeOtherValue"
class UpdateObject(dict):
def __init__(self, objectToUpdate, valueToUpdate):
self.objectToUpdate = objectToUpdate
self.valueToUpdate = valueToUpdate
def __call__(self):
returnDict = {}
returnDict["update"] = self.updateValue(self.valueToUpdate)
return returnDict
def updateValue(self, updatedValue):
for i in range(5):
time.sleep(1) # updating my object - time consuming with possible pid kill
print "working... (pid=%s)" % os.getpid()
self.objectToUpdate.accurate = updatedValue
return self.objectToUpdate
if __name__ == '__main__':
# queue for single process
taskQueue = multiprocessing.JoinableQueue()
resultQueue = multiprocessing.Queue()
newProcess = NewParallelProcess(taskQueue, resultQueue, processName="updateMyObject")
newProcess.start()
myObject = OldObject()
taskQueue.put(UpdateObject(myObject, "NewValue"))
while True:
# check if newProcess is still alive
time.sleep(5)
if newProcess.is_alive() is False:
print "Process %s (pid = %s) is not running any more (exit code = %s)" % (newProcess.name, newProcess.pid, newProcess.exitcode)
break
if newProcess.exitcode == 0:
print "ALL OK"
taskQueue.join()
# get back results
print "NOT KILLED"
results = resultQueue.get()
print "Values have been updated"
print "---> %s became %s" % (myObject.accurate, results["update"].accurate)
elif newProcess.exitcode == 1:
print "ended with error in function"
print "KILLED"
for i in range(5):
time.sleep(1)
print "i continue"
elif newProcess.exitcode == -15 or newProcess.exitcode == -9:
print "ended with kill signal %s" % newProcess.exitcode
print "KILLED"
for i in range(5):
time.sleep(1)
print "i continue"
else:
print "no idea what happened"
print "KILLED"
for i in range(5):
time.sleep(1)
print "i continue"

Python: Threading does not let the rest of the program run

In my program, when a thread starts, only the thread runs while the rest of the program waits for it. Here is the program:
import socket
from time import sleep
import threading
import string
class IRC:
"IRC Module for python"
def __init__(self, HOST, PORT, NICK, REALNAME, IDENT):
print "New Connection"
self.s=socket.socket( )
self.s.connect((HOST, PORT))
self.s.send("USER %s %s bla :%s\r\n" % (IDENT, HOST, REALNAME))
self.s.send("NICK %s\r\n" % NICK)
self.rb="ReadBuffer"
t = threading.Thread(target=self.ping())
t.start()
print "started thread"
def read(self):
readbuffer=readbuffer+self.s.recv(1024)
temp=string.split(readbuffer, "\n")
readbuffer=temp.pop( )
return temp
def ping(self):
"Handles Pinging"
readbuffer=""
print "Started pinging"
while True:
readbuffer=readbuffer+self.s.recv(1024)
temp=string.split(self.rb, "\n")
readbuffer=temp.pop( )
for line in temp:
line=string.rstrip(line)
line=string.split(line)
if(line[0]=='PING'):
self.s.send("PONG %s\r\n" % line[1])
print("Ponged")
self.pinged=True
print "ran"
def send(self, message, channel):
self.s.send("PRIVMSG %s :%s\r\n" % (channel, message))

Since the comment already solved your problem, there's not much more to say. Here's what doc says:
class threading.Thread(group=None, target=None, name=None, args=(),
kwargs={}, *, daemon=None)
This constructor should always be called
with keyword arguments. Arguments are:
group should be None; reserved for future extension when a ThreadGroup
class is implemented.
target is the callable object to be invoked by the run() method.
Defaults to None, meaning nothing is called.
So you should set target to a callable, which is the self.ping method, rather then call it.

Twisted spawnProcess, send output of one process to input of another

I am trying to use twisted spawnProcess to replicate the behavior of something like this:
cat <input.txt | wc -w
This is just an example of two commands, in reality I have my own processes (say python or bash scripts or external programs) where each process reads from stdin and writes to stdout. Just like the above example, I want to pipe stdout from one process to stdin of another, and I want to do this using spawnProcess. I used some hints here:
Twisted pipe two processes with spawnProcess
but I can't get it to work. It just hangs when reading from stdin on the second spawnProcess protocol. My code is below.What am I doing wrong? How exactly can I achieve this objective? Is it better to call the second spawnProcess from within the first?
#!/usr/bin/env python
from twisted.internet import protocol
from twisted.internet import reactor
import re
import os
import sys
class CatPP(protocol.ProcessProtocol):
def __init__(self,input_data):
self.input_data=input_data
self.data = ""
def connectionMade(self):
print "connectionMade in CatPP! Now writing to stdin of cat"
print " writing this data: %s" % self.input_data
self.transport.write(self.input_data+'\n')
print " closing stdin"
self.transport.closeStdin() # tell them we're done
print " stdin closed"
def outReceived(self, data):
print "outReceived from cat! with %d bytes!" % len(data)
self.data = self.data + data
print " received this: %s" % self.data
def errReceived(self, data):
print "errReceived from cat! with %d bytes!" % len(data)
def inConnectionLost(self):
print "inConnectionLost for cat! stdin is closed! (we probably did it)"
def outConnectionLost(self):
print "outConnectionLost for cat! The child closed their stdout!"
# now is the time to examine what they wrote
print "I saw cat write this:", self.data
def errConnectionLost(self):
print "errConnectionLost for cat! The child closed their stderr."
def processExited(self, reason):
print "processExited for cat, status %d" % (reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for cat, status %d" % (reason.value.exitCode,)
class WcPP(protocol.ProcessProtocol):
def __init__(self):
self.data = ""
def connectionMade(self):
print "connectionMade! Now reading from pipe to get stdin for wp"
print " reading from stdin"
txt = sys.stdin.read()
print " Read this from stdin: %s" % (txt,)
self.transport.write(txt)
self.transport.closeStdin() # tell them we're done
def outReceived(self, data):
print "outReceived from cat! with %d bytes!" % len(data)
self.data = self.data + data
def errReceived(self, data):
print "errReceived from cat! with %d bytes!" % len(data)
def inConnectionLost(self):
print "inConnectionLost for cat! stdin is closed! (we probably did it)"
def outConnectionLost(self):
print "outConnectionLost for cat! The child closed their stdout!"
# now is the time to examine what they wrote
print "Final output:", self.data
#(dummy, lines, words, chars, file) = re.split(r'\s+', self.data)
#print "I saw %s lines" % lines
def errConnectionLost(self):
print "errConnectionLost for cat! The child closed their stderr."
def processExited(self, reason):
print "processExited for cat, status %d" % (reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for cat, status %d" % (reason.value.exitCode,)
reactor.stop()
readPipe, writePipe = os.pipe()
handle=open('junkin.txt','r')
cat_txt=handle.read()
handle.close()
pp1 = CatPP(cat_txt)
pp2 = WcPP()
reactor.spawnProcess(pp1, "cat", ["cat"], {}, childFDs={1: writePipe})
reactor.spawnProcess(pp2, "wc", ["wc", "-w"], {},childFDs={0: readPipe})
reactor.run()
try:
os.close(readPipe)
except:
print "Exception closing readPipe"
try:
os.close(writePipe)
except:
print "Exception closing writePipe"

Here is a working example.
In the case of piping through cat | wc, spawnProcess duplicates the pipes so you need to close them.
from twisted.internet import protocol
from twisted.internet import reactor
import os
class Writer(protocol.ProcessProtocol):
def __init__(self, data):
self.data = data
def connectionMade(self):
print "Writer -- connection made"
self.transport.writeToChild(0, self.data)
self.transport.closeChildFD(0)
def childDataReceived(self, fd, data):
pass
def processEnded(self, status):
pass
class Reader(protocol.ProcessProtocol):
def __init__(self):
pass
def connectionMade(self):
print "Reader -- connection made"
pass
def childDataReceived(self, fd, data):
print "Reader -- childDataReceived"
self.received = data
def processEnded(self, status):
print "process ended, got:", self.received
class WriteRead(protocol.ProcessProtocol):
def __init__(self, data):
self.data = data
def connectionMade(self):
self.transport.writeToChild(0, self.data)
self.transport.closeChildFD(0)
def childDataReceived(self, fd, data):
self.received = data
print "got data:", data
def processEnded(self, status):
print "process ended - now what?"
def test1(data):
# just call wc
p2 = reactor.spawnProcess(WriteRead(data), "wc", ["wc"], env=None, childFDs={0: "w", 1: "r"})
reactor.run()
def test2(data):
rfd, wfd = os.pipe()
p1 = reactor.spawnProcess(Writer(data), "cat", ["cat"], env=None, childFDs={0:"w", 1: wfd })
p2 = reactor.spawnProcess(Reader(), "wc", ["wc", "-w"], env=None, childFDs={0: rfd, 1: "r"})
os.close(rfd)
os.close(wfd)
reactor.run()
test2("this is a test")

Ok I figured out how to do it but without using the pipes approach from Twisted pipe two processes with spawnProcess, which I never got to work. Instead I used a class ChainableProcessProtocol that takes another ChainableProcessProtocol as an argument.This way you can chain these together, with the output of a prior protocol writing to the stdin of the next protocol. This chaining stops when the next_protocol is None. The final output is the data from the final protocol. Here is my example:
#!/usr/bin/env python
from twisted.internet import protocol
from twisted.internet import reactor, defer
import re
import os
import sys
import json
def shutdown(x):
print "Shutdown called"
reactor.stop()
class ChainableProcessProtocol(protocol.ProcessProtocol):
def __init__(self,cmd,cmdargs,input_data,next_protocol):
self.cmd=cmd
self.cmdargs=cmdargs
self.input_data=input_data
self.next_protocol=next_protocol
self.data = ""
def set_input_data(self,new_input_data):
self.input_data=new_input_data
def connectionMade(self):
print "connectionMade in %s! Now writing to stdin of cat" % self.cmd
print " writing this data: %s" % self.input_data
self.transport.write(self.input_data+'\n')
print " closing stdin"
self.transport.closeStdin() # tell them we're done
print " stdin closed"
def outReceived(self, data):
print "outReceived from %s! with %d bytes!" % (self.cmd,len(data))
self.data = self.data + data
print " received this: %s" % self.data
def errReceived(self, data):
print "errReceived from %s! with %d bytes!" % (self.cmd,len(data))
def inConnectionLost(self):
print "inConnectionLost for %s! stdin is closed! (we probably did it)" %(self.cmd,)
def outConnectionLost(self):
print "outConnectionLost for %s! The child closed their stdout!" %(self.cmd,)
# now is the time to examine what they wrote
print "I saw %s write this: %s" % (self.cmd,self.data)
#
# cmd is done, now write to next_protocol if set
#
if self.next_protocol:
print "Calling chained protocol"
self.next_protocol.set_input_data(self.data)
npcmd=self.next_protocol.cmd
npcmdargs=self.next_protocol.cmdargs
print "npcmd is %s" % (npcmd,)
print "npcmdargs is %s" % (json.dumps(npcmdargs),)
reactor.spawnProcess(self.next_protocol, npcmd, npcmdargs, {})
else:
print "No chained protocol"
def errConnectionLost(self):
print "errConnectionLost for %s! The child closed their stderr." % (self.cmd,)
def processExited(self, reason):
print "processExited for %s, status %d" % (self.cmd,reason.value.exitCode,)
def processEnded(self, reason):
print "processEnded for %s, status %d" % (self.cmd,reason.value.exitCode,)
handle=open('junkin.txt','r')
in_txt=handle.read()
handle.close()
#
# Create the last protocol first because earlier protocol(s) need it
#
pp2 = ChainableProcessProtocol("wc",["wc","-w"],'',None)
#
# The first process takes an instance of the second process as an argument
#
pp1 = ChainableProcessProtocol("cat",["cat"],in_txt,pp2)
#
# before using spawnProcess, lets create a deferred to shut down the reactor in 2 seconds
# This should give us enough time. This is only so our test script does not run forever
#
d=defer.Deferred()
d.addCallback(shutdown)
reactor.callLater(2, d.callback, '')
#
# Now spawn the first process
#
reactor.spawnProcess(pp1, pp1.cmd, pp1.cmdargs, {})
reactor.run()
print "Final output is data in pp2: %s" % (pp2.data.strip(),)
print "Done!"

Max threads Python and how kill Thread?

I make daemon which creates N threads. N beetwen 1 to 500.
When thread end handling data it must die. When thread start it create log file and write to it. When it die - file should be remove. When threads started its check LOCK file. If file removed it should be die. But sometimes not. Why thread does not die immidiatly?
class HandlerWorker(threading.Thread):
q_reader = None
q_writer = None
q_cmd = None
id = ''
task_id = 0
def __init__(self,id,task_id,q_reader,q_writer,q_cmd):
'''
Constructor
'''
#print "Worker %d started" % int(id)
self.id = id
self.task_id = task_id
self.q_cmd = q_cmd
self.q_reader = q_reader
self.q_writer = q_writer
threading.Thread.__init__(self)
def __del__(self):
print "Destroy worker %d %d" % (int(self.task_id),int(self.id))
def isPid(self):
is_pid = True
try:
cfg = ConfigReader('config.json')
general = cfg.getGeneral()
pidfile = "%s%d" % (str(general['pids']),int(self.task_id))
f=open(pidfile,'r')
f.close()
except:
is_pid = False
return is_pid
def run(self):
'''
Handle email
'''
is_run = True
cfg = ConfigReader('config.json')
general = cfg.getGeneral()
logpath = str(general['pids'])+"../logs/%d_%d" % (int(self.task_id),int(self.id))
f = open(logpath,"w+")
while is_run:
if not self.q_reader.empty():
msg = self.q_reader.get()
# convert message
self.q_writer.put(msg)
log_str = "Date: %s Email:%s Status:%d\n" % (str(time.asctime()),str(msg),int(status))
f.write(log_str);
f.flush()
is_run = self.isPid()
time.sleep(1)
f.close()
try:
os.remove(logpath)
except:
print "Can't remove LOG file: %s" %logpath
print "Stop thread %d %d" % (int(self.task_id),int(self.id))

How to create connection timeout with python SocketServer

Good day! I was writen simple server:
class SingleTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(1024)
self.request.close()
class SimpleServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
def running():
server = SimpleServer((settings.host, settings.port), SingleTCPHandler)
try:
server.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
How to set connection timeout. I want when the client not send me data and is not active in 30 seconds, server will close connection.
P.S. sorry for my english.
UPDATE
#!/usr/bin/env python
# -*- coding: utf8 -*-
import sys
import time
import SocketServer
import datetime
import settings
import os
from signal import SIGTERM, SIGCHLD, signal, alarm
import socket
import subprocess
from threading import Thread
import MySQLdb
import re
class SingleTCPHandler(SocketServer.BaseRequestHandler):
"One instance per connection. Override handle(self) to customize action."
def handle(self):
alarm(30)
data = self.request.recv(1024)
# Some code
self.request.close()
class SimpleServer(SocketServer.ForkingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
def running():
server = SimpleServer((settings.host, settings.port), SingleTCPHandler)
try:
server.serve_forever()
except KeyboardInterrupt:
sys.exit(0)
def deamonize(stdout='/dev/null', stderr=None, stdin='/dev/null', pidfile=None, startmsg='started with pid %s'):
try:
pid = os.fork()
if (pid > 0):
sys.exit(0)
except OSError, e:
sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
os.chdir(settings.place)
os.umask(0)
os.setsid()
try:
pid = os.fork()
if (pid > 0):
sys.exit(0)
except OSError, e:
sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
if (not stderr):
stderr = stdout
print stdin, stdout, stderr
si = file(stdin, 'r')
so = file(stdout, 'a+')
se = file(stderr, 'a+', 0)
pid = str(os.getpid())
sys.stderr.write("\n%s\n" % startmsg % pid)
sys.stderr.flush()
if pidfile: file(pidfile, 'w+').write("%s\n" % pid)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())
def startstop(stdout='/dev/null', stderr=None, stdin='/dev/null', pidfile='pid.txt', startmsg='started with pid %s'):
if len(sys.argv) > 1:
action = sys.argv[1]
try:
pf = open(pidfile)
pid = int(pf.read().strip())
pf.close()
except IOError:
pid = None
if ((action == 'stop') or (action == 'restart')):
if (not pid):
mess = "Не могу остановить, pid файл '%s' отсутствует.\n"
sys.stderr.write(mess % pidfile)
sys.exit(1)
try:
while 1:
os.kill(pid, SIGTERM)
time.sleep(1)
except OSError, err:
err = str(err)
if err.find("No such process") > 0:
os.remove(pidfile)
if 'stop' == action:
sys.exit(0)
action = 'start'
pid = None
else:
print str(err)
sys.exit(1)
if ('start' == action):
if (pid):
mess = "Старт отменен — pid файл '%s' существует.\n"
sys.stderr.write(mess % pidfile)
sys.exit(1)
deamonize(stdout, stderr, stdin, pidfile, startmsg)
return
print "Синтакс запуска: %s start|stop|restart" % sys.argv[0]
sys.exit(2)
if (__name__ == "__main__"):
startstop(stdout=settings.log, pidfile=settings.pid)
running()

If you use StreamRequestHandler instead of BaseRequestHandler, you just need to override the timeout variable there, and it will be set. If you want to learn how to do it yourself, just look at the SocketServer.py
Here's an example, this will kill any connections that aren't done in 5 seconds:
#!/usr/bin/env python
import SocketServer
class myHandler(SocketServer.StreamRequestHandler):
timeout = 5
def handle(self):
recvdata = ""
while True:
tmp = self.request.recv(16384)
recvdata = recvdata + tmp.strip()
if (len(tmp) < 16384):
break;
self.request.send("Received: {0}".format(recvdata))
class myApp(SocketServer.TCPServer):
def __init__(self):
SocketServer.TCPServer.__init__(self, ("localhost", 5555), myHandler)
print self.server_address
try:
self.serve_forever()
except KeyboardInterrupt:
print "Got keyboard interrupt, shutting down"
self.shutdown()
if __name__ == "__main__":
app = myApp()
This uses the python's socket settimeout() call.
I don't think your alarm() solution will work with threading or forking.

Please look at it:
import sys
import SocketServer
class SingleTCPHandler(SocketServer.BaseRequestHandler):
def handle(self):
data = self.request.recv(1024)
self.request.close()
class SimpleServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
timeout = 30
daemon_threads = True
allow_reuse_address = True
def __init__(self, server_address, RequestHandlerClass):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
def handle_timeout(self):
print 'Timeout!'
def running():
server = SimpleServer(('localhost', 6666), SingleTCPHandler)
try:
#server.serve_forever()
server.handle_request()
except KeyboardInterrupt:
sys.exit(0)
if __name__ == '__main__':
running()
# vim: filetype=python syntax=python expandtab shiftwidth=4 softtabstop=4 encoding=utf8
If you want to handle more than one request you need to execute server.handle_request() again.