I want to use subprocess to call other py files and get output.
The colors they output are the same, How can i distinguish between stdout and stderr in subprocess?
run.py
def run():
for i in range(3):
print('Processing {}.'.format(i))
time.sleep(1)
print(1/0)
run()
main.py
import subprocess
import sys
def byte2str(b):
return str(b, encoding='utf-8')
if __name__ == '__main__':
cmd = [sys.executable, 'temp/run.py']
p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while p.poll() is None:
line = p.stdout.readline()
if line:
print(byte2str(line), end='')
Related
How can I get string as return in this script?
main.py
from subprocess import Popen, PIPE
import os
import sys
child = os.path.join(os.path.dirname(__file__), "child.py")
command = [sys.executable, child, "test"]
process = Popen(command, stdout=PIPE, stdin=PIPE)
process.communicate()
print(process.poll())
child.py
import sys
def main(i):
return i*3
if __name__ == '__main__':
main(*sys.argv[1:])
I get only 0.
I think get response from print() and process.communicate() not the best way.
Processes can't return values in the same sense a function can.
They can only set an exit code (which is the 0 you get).
You can, however, use stdin and stdout to communicate between the main script and child.py.
To "return" something from a child, just print the value you want to return.
# child.py
print("Hello from child")
The parent would do something like this:
process = Popen(command, stdout=PIPE, stdin=PIPE)
stdout, stderr = Popen.communicate()
assert stdout == "Hello from child"
I'm trying to create a python script that allows you to open 2 terminals simultaneously, it works fine, but if I try to insert it into a loop the 2nd terminal doesn't open and it skips to the next statement, here is the code that works fine:
from subprocess import Popen, PIPE, CREATE_NEW_CONSOLE
import multiprocessing as mp
import threading as th
def run(command):
cmd = Popen(command, PIPE, creationflags=CREATE_NEW_CONSOLE)
com, err = cmd.communicate()
print(com,err)
if __name__ == "__main__":
mp.freeze_support()
command = f"python testFile.py" # or command = f"testFile.exe"
process = mp.Process(target=run, args= (command,))
process.start()
input("Wait ")
as I said this works fine, but if I try to insert it in a loop the 2nd terminal doesn't open:
from subprocess import Popen, PIPE, CREATE_NEW_CONSOLE
import multiprocessing as mp
import threading as th
def run(command):
cmd = Popen(command, PIPE, creationflags=CREATE_NEW_CONSOLE)
com, err = cmd.communicate()
print(com,err)
while True:
if __name__ == "__main__":
mp.freeze_support()
command = f"python testFile.py" # or command = f"testFile.exe"
process = mp.Process(target=run, args= (command,))
process.start()
input("Wait ")
Let's say I have myscript.py:
while True:
print("hi")
time.sleep(1)
Now launching it in mainscript.py as subprocess:
sub = subprocess.Popen([sys.executable, "-u", myscript.py], creationflags=CREATE_NEW_CONSOLE, stdout=PIPE, bufsize=1)
Now I read the output of myscript.py:
while sub.poll() == None:
subm = sub.stdout.readline()
print (subm) #print for checking only
But this only works, when I don't create a new console. But I need this console window.
How can I catch/access the output of another console window?
Any help will be appreciated. Thanks!
This works for me.
Is this what you are searching for?
import sys, subprocess
from subprocess import Popen, PIPE, CREATE_NEW_CONSOLE
p = subprocess.Popen([sys.executable, "-u", os.path.dirname(__file__) + '/myscript.py'], creationflags=CREATE_NEW_CONSOLE, bufsize=1)
if p.stdout:
for line in iter(p.stdout.readline, b''):
print line,
p.wait()
Well, I have two scripts. The a.py which prints the output of the b.py script as follows:
#a.py
from subprocess import Popen, PIPE, STDOUT
p = Popen(['/Users/damian/Desktop/b.py'], shell=False, stdout=PIPE, stderr=STDOUT)
while p.poll() is None:
print p.stdout.readline()
#b.py
#!/usr/bin/env python
import time
while 1:
print 'some output'
#time.sleep(1)
This works.But,
Why do my scripts deadlock when I uncomment the time.sleep() line?
Your output is probably buffered. Add a .flush() for stdout to clear it:
import sys
import time
while 1:
print 'someoutput'
sys.stdout.flush()
time.sleep(1)
If you add -u to the call in a.py (make the output unbuffered) then you don't need to modify b.py script:
import sys
from subprocess import Popen, PIPE, STDOUT
p = Popen([sys.executable, '-u', '/Users/damian/Desktop/b.py'],
stdout=PIPE, stderr=STDOUT, close_fds=True)
for line in iter(p.stdout.readline, ''):
print line,
p.stdout.close()
if p.wait() != 0:
raise RuntimeError("%r failed, exit status: %d" % (cmd, p.returncode))
See more ways to get output from a subprocess.
I'm running into some difficulties getting output from a subprocess stdout pipe. I'm launching some third party code via it, in order to extract log output. Up until a recent update of the third party code, everything worked fine. After the update, python has started blocking indefinitely, and not actually showing any output. I can manually launch the third party app fine and see output.
A basic version of the code I'm using:
import subprocess, time
from threading import Thread
def enqueue_output(out):
print "Hello from enqueue_output"
for line in iter(out.readline,''):
line = line.rstrip("\r\n")
print "Got %s" % line
out.close()
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, bufsize=1)
thread = Thread(target=enqueue_output, args=(proc.stdout,))
thread.daemon = True
thread.start()
time.sleep(30)
This works perfectly if I substitute third_party.exe for this script:
import time, sys
while True:
print "Test"
sys.stdout.flush()
time.sleep(1)
So I'm unclear as to magic needs to be done to get this working with the original command.
These are all variants of the subprocess.Popen line I've tried with no success:
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, bufsize=0)
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, shell=True)
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, creationflags=subprocess.CREATE_NEW_CONSOLE)
si = subprocess.STARTUPINFO()
si.dwFlags = subprocess.STARTF_USESTDHANDLES | subprocess.STARTF_USESHOWWINDOW
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, startupinfo=si)
Edit 1:
I can't actually use .communicate() in this case. The app I'm launching remains running for long periods of time (days to weeks). The only way I could actually test .communicate() would be to kill the app shortly after it launches, which I don't feel would give me valid results.
Even the non-threaded version of this fails:
import subprocess, time
from threading import Thread
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print "App started, reading output..."
for line in iter(proc.stdout.readline,''):
line = line.rstrip("\r\n")
print "Got: %s" % line
Edit 2:
Thanks to jdi, the following works okay:
import tempfile, time, subprocess
w = "test.txt"
f = open("test.txt","a")
p = subprocess.Popen("third_party.exe", shell=True, stdout=f,
stderr=subprocess.STDOUT, bufsize=0)
time.sleep(30)
with open("test.txt", 'r') as r:
for line in r:
print line
f.close()
First I would recommend that you simplify this example to make sure you can actually read anything. Remove the complication of the thread from the mix:
proc = subprocess.Popen("third_party.exe", stdout=subprocess.PIPE, bufsize=1)
print proc.communicate()
If that works, great. Then you are having problems possibly with how you are reading the stdout directly or possibly in your thread.
If this does not work, have you tried piping stderr to stdout as well?
proc = subprocess.Popen("third_party.exe",
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, bufsize=1)
Update
Since you say communicate() is deadlocking, here is another approach you can try to see if its a problem with the internal buffer of subprocess...
import tempfile
import subprocess
w = tempfile.NamedTemporaryFile()
p = subprocess.Popen('third_party.exe', shell=True, stdout=w,
stderr=subprocess.STDOUT, bufsize=0)
with open(w.name, 'r') as r:
for line in r:
print line
w.close()
args = ['svn','log','-v']
def foo(info=''):
import logging
import subprocess
import tempfile
try:
pipe = subprocess.Popen(args,bufsize = 0,\
stdout = subprocess.PIPE,\
stderr=subprocess.STDOUT)
except Exception as e:
logging.error(str(e))
return False
while 1:
s = pipe.stdout.read()
if s:
print s,
if pipe.returncode is None:
pipe.poll()
else:
break
if not 0 == pipe.returncode:
return False
return True
print foo()
This one should works,not thread,temp file magic.