We Keep Coding

Timeout subprocess while reading stdout realtime

from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
cmd = 'for x in $(seq 1 3); do echo stage $x; sleep 1; done'
proc = Popen(cmd, shell=True, close_fds=True,
stdout=PIPE, stderr=STDOUT,
universal_newlines=True, start_new_session=True)
for line in proc.stdout:
print(line.strip())
Default behavior of communicate is to spit out all stdout after the proc. is terminated. However I need access to stdout while proc. is still running, but terminate it, if it's running too long.
EDIT
The following seem to work the way I want, not sure if that's correct approach?
from os import killpg
from signal import SIGKILL
from concurrent import futures
from subprocess import Popen, PIPE, STDOUT, TimeoutExpired
cmd = 'for x in $(seq 1 3); do echo stage $x; sleep 1; done'
proc = Popen(cmd, shell=True, close_fds=True,
stdout=PIPE, stderr=STDOUT,
universal_newlines=True, start_new_session=True)
def handler(proc, timeout=None):
try:
proc.wait(timeout)
except TimeoutExpired as err:
print(err)
killpg(proc.pid, SIGKILL)
exe = futures.ThreadPoolExecutor(max_workers=1)
exe.submit(handler, proc, 2)
for line in proc.stdout:
print(line.strip())

Python interact with subprocess

I want to interact with a process.
I can start the process and print out the first two lines (something like 'process successfully started').
Now I want to send a new command to the process which should return again something like 'command done' but nothing happens.
Please help me.
import subprocess
def PrintAndPraseOutput(output, p):
print(output)
if 'sucessfully' in output:
p.stdin.write('command')
cmd = ["./programm"]
p = subprocess.Popen(cmd, universal_newlines=True, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
while p.poll() is None:
output = p.stdout.readline()
PrintAndPraseOutput(output, p)
Update:
same problem, no output after 'process successfully started'
import subprocess
def print_and_parse_output(output, p):
print(output)
if 'successfully' in output:
p.stdin.write('command\n')
with subprocess.Popen(["./programm"], universal_newlines=True, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE) as proc:
while proc.poll() is None:
output = proc.stdout.readline()
print_and_parse_output(output, proc)

Your I/O should be line buffered, so PrintAndPraseOutput should send a '\n' at the end of the string.
BTW, you have a couple of spelling errors. That function should be named print_and_parse_output to conform to PEP-0008, and "successfully" has 2 c's.
def print_and_parse_output(output, p):
print(output)
if 'successfully' in output:
p.stdin.write('command\n')
When using subprocess like this it's a good idea to put it in a with statement. From the subprocess.Popen` docs:
Popen objects are supported as context managers via the with
statement: on exit, standard file descriptors are closed, and the
process is waited for.
with Popen(["ifconfig"], stdout=PIPE) as proc:
log.write(proc.stdout.read())

Output from subprocess not saved

When I use stdout in my Popen with stdin my process does not run.
def program():
p = subprocess.Popen(link, shell=True, stdin=subprocess.PIPE, stderr=subprocess.STDOUT)
out = p.communicate(longquery)
print out
When I run this my out comes out as (None, None) although I do have an output that shows up on the console. Why is this happening and how can I save the output that is coming out?

You need to pipe the output data per the manual:
"to get anything other than None in the result tuple, you need to give stdout=PIPE and/or stderr=PIPE too."
def program():
p = subprocess.Popen(link, shell=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
out = p.communicate(longquery)
print out

You need to set stdout=PIPE as well.
Change your code to:
from subprocess import Popen, PIPE, STDOUT
def program():
p = Popen(link, shell=True, stdin=PIPE, stderr=STDOUT, stdout=PIPE)
out = p.communicate(longquery)
print out

Python popen python.exe

I'm rather puzzled by why the code below doesn't print stdout and exit, instead it hangs (on windows). Any reason why?
import subprocess
from subprocess import Popen
def main():
proc = Popen(
'C:/Python33/python.exe',
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE
)
proc.stdin.write(b'exit()\r\n')
proc.stdin.flush()
print(proc.stdout.read(1))
if __name__=='__main__':
main()

Replace the following:
proc.stdin.flush()
with:
proc.stdin.close()
Otherwise the subprocess python.exe will wait forever stdin to be closed.
Alternative: using communicate()
proc = Popen(...)
out, err = proc.communicate(b'exit()\r\n')
print(out) # OR print(out[:1]) if you want only the first byte to be print.

subprocess.communicate() hangs on Windows 8 if parent process creates some child

I have a simple code which works fine on Win 2003:
proc = subprocess.Popen('<some python script which runs another process>', stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = subprocess.PIPE)
out = proc.communicate()[0]
But on Windows 8 this part; out = proc.communicate()[0], hangs.
Have anybody seeen this issue?
I've checked that process is really ternimated (PID is absent when child process has been started)
It's also a problem to make proc.stdout.readlines(), it hangs too. How to check that stdout has EOF?
When I stop child process proc.communicate() works fine.
Here is the simplest example:
import subprocess
proc = subprocess.Popen([sys.executable, "D:\\test.py"], stdout = subprocess.PIPE)
print 'PID', proc.pid #When this PID is printed I see in the taskbr that process is already finished
print 'Output', proc.communicate() # but this part is hangs
And code od test.py:
import os, time
from subprocess import Popen, PIPE
CREATE_NEW_PROCESS_GROUP = 0x00000200 # note: could get it from subprocess
DETACHED_PROCESS = 0x00000008 # 0x8 | 0x200 == 0x208
p = Popen("start /B notepad", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE,
creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
print 'Done'
exit()
Is it valid scenario?

To allow .communicate() to return without waiting for the grandchild (notepad) to exit, you could try in test.py:
import sys
from subprocess import Popen, PIPE
CREATE_NEW_PROCESS_GROUP = 0x00000200
DETACHED_PROCESS = 0x00000008
p = Popen('grandchild', stdin=PIPE, stdout=PIPE, stderr=PIPE,
creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)
See Popen waiting for child process even when the immediate child has terminated.