My main goal is to run an external python script (client script) by subprocess in another python script (caller script). The console of the caller script displays all output from the client script except the tqdm output - so it is not a general problem of displaying output by subprocess, but a specific problem related to subprocess interacting with tqdm.
My secondary goal is that I'd like to understand it :). So thoughtful explanations are much appreciated.
The client script (train.py) contains several tqdm calls. So far, I haven't seen much difference in outputs between various tqdm argument configurations, so let's use the simplest one.
In train.py:
...
from tqdm import tqdm
with tqdm(total = 10, ncols = 80,
file=sys.stdout, position = 0, leave = True,
desc='f5b: pbar.set_postfix') as pbar:
for i in range(10):
pbar.update(1)
postfix = {'loss': '{0:.4f}'.format(1+i)}
pbar.set_postfix(**postfix)
sleep(0.1)
The caller script experiment.py executes the function execute_experiment which calls train.py by the argument command_list:
def execute_experiment(command_list):
tic = time.time()
try:
process = subprocess.Popen(
command_list, shell=False,
encoding='utf-8',
bufsize=0,
stdin=subprocess.DEVNULL,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
# Poll process for new output until finished
# Source: https://stackoverflow.com/q/37401654/7769076
while process.poll() is None:
nextline = process.stdout.readline()
sys.stdout.write(nextline)
sys.stdout.flush()
except CalledProcessError as err:
print("CalledProcessError: {0}".format(err))
sys.exit(1)
except OSError as err:
print("OS error: {0}".format(err))
sys.exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
raise
if (process.returncode == 0):
toc = time.time()
time1 = str(round(toc - tic))
return time1
else:
return 1
This script call to the above code snipped of train.py does return output but the tqdm output is stopped after 0 seconds and looks like this:
f5b: pbar.set_postfix: 0%| | 0/10 [00:00<?, ?it/s]
f5b: pbar.set_postfix: 10%|█▊ | 1/10 [00:00<00:00, 22310.13it/s]
The script call to the original code of train.py returns all output except tqdm output:
Training default configuration
train.py data --use-cuda ...
device: cuda
...
Comments:
shell = False: As python script calls python script. When shell=True, the client script is not called at all
bufsize=0: To prevent buffering
The train.py call is preceded with sys.executable to ensure that the python interpreter of the corresponding conda environment is called when on local machine.
Questions:
Does tqdm.set_postfix prevent passing the progress bar output upstream? I know this happens when tqdm.set_description is invoked, e.g. by:
pbar.set_description('processed: %d' %(1 + i))
This code contains it:
def train(self, dataloader, max_batches=500, verbose=True, **kwargs):
with tqdm(total=max_batches, disable=not verbose, **kwargs) as pbar:
for results in self.train_iter(dataloader, max_batches=max_batches):
pbar.update(1)
postfix = {'loss': '{0:.4f}'.format(results['mean_outer_loss'])}
if 'accuracies_after' in results:
postfix['accuracy'] = '{0:.4f}'.format(
np.mean(results['accuracies_after']))
pbar.set_postfix(**postfix)
# for logging
return results
Is the nested function call the reason why the progress bar is not shown?
The order of calls is experiment.py > train.py > nested.py.
train.py calls the train function in nested.py by:
for epoch in range(args.num_epochs):
results_metatraining = metalearner.train(meta_train_dataloader,
max_batches=args.num_batches,
verbose=args.verbose,
desc='Training',
# leave=False
leave=True
)
Alternatives tried out with no success:
### try2
process = subprocess.Popen(command_list, shell=False, encoding='utf-8',
stdin=DEVNULL, stdout=subprocess.PIPE)
while True:
output = process.stdout.readline().strip()
print('output: ' + output)
if output == '' and process.poll() is not None: # end of output
break
if output: # print output in realtime
print(output)
else:
output = process.communicate()
process.wait()
### try6
process = subprocess.Popen(command_list, shell=False,
stdout=subprocess.PIPE, universal_newlines=True)
for stdout_line in iter(process.stdout.readline, ""):
yield stdout_line
process.stdout.close()
return_code = process.wait()
print('return_code' + str(return_code))
if return_code:
raise subprocess.CalledProcessError(return_code, command_list)
### try7
with subprocess.Popen(command_list, stdout=subprocess.PIPE,
bufsize=1, universal_newlines=True) as p:
while True:
line = p.stdout.readline()
if not line:
break
print(line)
exit_code = p.poll()
I think readline is waiting for '\n', and tqdm is not creating new lines, maybe this could help (I did not try):
import io
def execute_experiment(command_list):
tic = time.time()
try:
process = subprocess.Popen(
command_list, shell=False,
encoding='utf-8',
bufsize=1,
stdin=subprocess.DEVNULL,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
# Poll process for new output until finished
# Source: https://stackoverflow.com/q/37401654/7769076
reader = io.TextIOWrapper(process.stdout, encoding='utf8')
while process.poll() is None:
char = reader.read(1)
sys.stdout.write(char)
sys.stdout.flush()
except CalledProcessError as err:
print("CalledProcessError: {0}".format(err))
sys.exit(1)
except OSError as err:
print("OS error: {0}".format(err))
sys.exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
raise
if (process.returncode == 0):
toc = time.time()
time1 = str(round(toc - tic))
return time1
else:
return 1
Related
Hi I am trying to get continuous output from airodump-ng mon0
For that reason I was trying to read the output of airodump-ng mon0 after certain time with Popen.communicate but still cannot get anything.
import subprocess
airodump = subprocess.Popen(['airodump-ng', 'mon0'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
o_airodump, unused_stderr = airodump.communicate(timeout=15)
except subprocess.TimeoutExpired as e:
airodump.kill()
o_airodump, unused_stderr = airodump.communicate()
print(o_airodump)
print(unused_stderr)
When I run this it gets stuck at:
o_airodump, unused_stderr = airodump.communicate()
I am totally stuck now. And unable to find any other ways. Please help.
I used the following code to retreive the list of avaible wifi networks after 60 seconds:
def find_wifi(interface):
table = ''
stdout = []
timeout = 60
table_start = re.compile('\sCH')
start_time = time.time()
airodump = subprocess.Popen(['airodump-ng', interface], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, bufsize=1)
while time.time() < start_time + timeout:
line = airodump.stdout.readline()
if table_start.match(line):
table = ''.join(stdout)
stdout = []
stdout.append(line)
airodump.terminate()
print(table)
This is my code:
def uploadByRSync(host, user, passwd, src, dst, statusManager):
try:
os.environ["RSYNC_PASSWORD"] = passwd
print host, user, passwd, src, dst
parameters = ["rsync", "-azP", "--partial", src ,"{3}#{0}::{2}/{1}".format(host, dst, user, user)]
print " ".join(parameters)
process = subprocess.Popen(parameters, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
for line in unbuffered(process):
if "%" in line:
spl = line.split()
statusManager.uploadSpeed = spl[2]
statusManager.uploaded = spl[1]
return not process.wait()
except Exception as ex:
print ex
return False
newlines = ['\n', '\r\n', '\r']
def unbuffered(proc, stream='stdout'):
stream = getattr(proc, stream)
with contextlib.closing(stream):
while True:
out = []
last = stream.read(1)
# Don't loop forever
if last == '' and proc.poll() is not None:
break
while last not in newlines:
# Don't loop forever
if last == '' and proc.poll() is not None:
break
out.append(last)
last = stream.read(1)
out = ''.join(out)
print out
yield out
When running with the py2app version I can never get an output. When running as script everything works just fine. ps: this code runs on a separated thread of a Qt app. Does anyone have any idea why this is happening?
Most likely you have a stream buffering issue. Here is how you can output all lines of your process in real time:
import subprocess
import select
p = subprocess.Popen(parameters,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
bufsize=0)
poll = [p.stdout.fileno(), p.stderr.fileno()]
while True:
# check if process is still running and read remaining data
if p.poll() is not None:
for l in p.stdout.readlines():
print(l)
for l in p.stderr.readlines():
print(l)
break
# blocks until data is being recieved
ret = select.select(poll, [], [])
for fd in ret[0]:
line = p.stdout.readline() if fd == p.stdout.fileno() else p.stderr.readline()
print(line)
Just made a test changing the Popen call by a simple 'ls',but I still cannot get the output when running py2app version. It works just fine when running python script. When I kill the py2app version app the output is just printed.
process = subprocess.Popen(["ls"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
For example:
#!/usr/bin/env python3
# cmd.py
import time
for i in range(10):
print("Count %d" % i)
time.sleep(1)
#!/usr/bin/env python3
import subprocess
# useCmd.py
p = subprocess.Popen(['./cmd.py'], stdout=subprocess.PIPE)
out, err = p.communicate()
out = out.decode()
print(out)
In useCmd.py I can print out the output of cmd.py, but only after it's finished outputting. How can I print out it in realtime and still have it stored in a string? (sort of like tee in bash.)
If you don't have to deal with stdin, you could avoid using communicate that is blocking, and read directly from the process stdout until your stdout ends:
p = subprocess.Popen(['python', 'cmd.py'], stdout=subprocess.PIPE)
# out, err = p.communicate()
while True:
line = p.stdout.readline()
if line != '':
print line,
else:
break
related
I want to check if a subprocess has finished execution successfully or failed. Currently I have come up with a solution but I am not sure if it is correct and reliable. Is it guaranteed that every process outputs its errors only to stderr respectfully to stdout:
Note: I am not interested in just redirecting/printing out the output. That I know already how to do.
pipe = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
if "" == pipe.stdout.readline():
print("Success")
self.isCommandExectutionSuccessful = True
if not "" == pipe.stderr.readline():
print("Error")
self.isCommandExectutionSuccessful = True
alternatively:
if "" == pipe.stdout.readline():
print("Success")
self.isCommandExectutionSuccessful = True
else:
print("Error")
self.isCommandExectutionSuccessful = False
and:
if not "" == pipe.stderr.readline():
print("Success")
self.isCommandExectutionSuccessful = True
else:
print("Error")
self.isCommandExectutionSuccessful = False
Do you need to do anything with the output of the process?
The check_call method might be useful here. See the python docs here: https://docs.python.org/2/library/subprocess.html#subprocess.check_call
You can then use this as follows:
try:
subprocess.check_call(command)
except subprocess.CalledProcessError:
# There was an error - command exited with non-zero code
However, this relies on command returning an exit code of 0 for succesful completion and a non-zero value for an error.
If you need to capture the output as well, then the check_output method may be more appropriate. It is still possible to redirect the standard error if you need this as well.
try:
proc = subprocess.check_output(command, stderr=subprocess.STDOUT)
# do something with output
except subprocess.CalledProcessError:
# There was an error - command exited with non-zero code
See the docs here: https://docs.python.org/2/library/subprocess.html#subprocess.check_output
Complete solution with check on return code, stdout and stderr:
import subprocess as sp
# ok
pipe = sp.Popen( 'ls /bin', shell=True, stdout=sp.PIPE, stderr=sp.PIPE )
# res = tuple (stdout, stderr)
res = pipe.communicate()
print("retcode =", pipe.returncode)
print("res =", res)
print("stderr =", res[1])
for line in res[0].decode(encoding='utf-8').split('\n'):
print(line)
# with error
pipe = sp.Popen( 'ls /bing', shell=True, stdout=sp.PIPE, stderr=sp.PIPE )
res = pipe.communicate()
print("retcode =", pipe.returncode)
print("res =", res)
print("stderr =", res[1])
Prints:
retcode = 0
res = (b'bash\nbunzip2\nbusybox\nbzcat\n...zmore\nznew\n', b'')
stderr = b''
bash
bunzip2
busybox
bzcat
...
zmore
znew
retcode = 2
res = (b'', b"ls: cannot access '/bing': No such file or directory\n")
stderr = b"ls: cannot access '/bing': No such file or directory\n"
output,error=pipe.communicate()
This will wait for command to finish and give you output or error depending on the state of command.
You can check return code of the process using check_call() method.
In case if process returned non-zero value CalledProcessError will be raised.
There are situations where using check_call is not a possibility. For example when you need to communicate with the process (e.g., passing input with communicate).
In this case, a simple solution is to mimic check_call manually. We can look at the Python source code to see what check_call is doing here and you can see it's simply checking the return code is 0 and if not it raises a CalledProcessError. It couldn't be simpler.
You may notice that check_call is not including stdout or stderr in the CalledProcessError even though it can accept them. This is because the process may not have captured them unless it was given subprocess.PIPE for stdout and stderr Popen parameters.
video_url = "http://.../sample/BigBuckBunny.mp4"
p = subprocess.Popen(
[
"ffplay",
"-i", "-"
],
stdin=subprocess.PIPE
)
p.communicate(video_url.encode())
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, p.args)
The above is an example scenario where we need to PIPE input data (in this case an URL) to the process. It is not possible to write to stdin with check_call.
We simply mimic check_call with the last 2 lines.
This is how I did it finally:
# Call a system process
try:
# universal_newlines - makes manual decoding of subprocess.stdout unnecessary
output = subprocess.check_output(command,
stderr=subprocess.STDOUT,
universal_newlines=True)
# Print out command's standard output (elegant)
self.textEdit_CommandLineOutput.insertPlainText(output)
self.isCommandExecutionSuccessful = True
except subprocess.CalledProcessError as error:
self.isCommandExecutionSuccessful = False
errorMessage = ">>> Error while executing:\n"\
+ command\
+ "\n>>> Returned with error:\n"\
+ str(error.output)
self.textEdit_CommandLineOutput.append(errorMessage)
QMessageBox.critical(None,
"ERROR",
errorMessage)
print("Error: " + errorMessage)
except FileNotFoundError as error:
errorMessage = error.strerror
QMessageBox.critical(None,
"ERROR",
errorMessage)
print("Error: ", errorMessage)
I hope it will be useful to someone else.
I am a Python newbie writing a Python (2.7) script that needs to exec a number of external applications, one of which writes a lot of output to its stderr stream. What I am trying to figure out is a concise and succinct way (in Python) to get the last N lines from that subprocess' stderr output stream.
Currently, I am running that external application from my Python script like so:
p = subprocess.Popen('/path/to/external-app.sh', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print "ERROR: External app did not complete successfully (error code is " + str(p.returncode) + ")"
print "Error/failure details: ", stderr
status = False
else:
status = True
I'd like to capture the last N lines of output from its stderr stream so that they can be written to a log file or emailed, etc.
N = 3 # for 3 lines of output
p = subprocess.Popen(['/path/to/external-app.sh'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if p.returncode != 0:
print ("ERROR: External app did not complete successfully "
"(error code is %s)" % p.returncode)
print "Error/failure details: ", '\n'.join(stderr.splitlines()[-N:])
status = False
else:
status = True
If the whole output can't be stored in RAM then:
import sys
from collections import deque
from subprocess import Popen, PIPE
from threading import Thread
ON_POSIX = 'posix' in sys.builtin_module_names
def start_thread(func, *args):
t = Thread(target=func, args=args)
t.daemon = True
t.start()
return t
def consume(infile, output):
for line in iter(infile.readline, ''):
output(line)
infile.close()
p = Popen(['cat', sys.argv[1]], stdout=PIPE, stderr=PIPE,
bufsize=1, close_fds=ON_POSIX)
# preserve last N lines of stdout, print stderr immediately
N = 100
queue = deque(maxlen=N)
threads = [start_thread(consume, *args)
for args in (p.stdout, queue.append), (p.stderr, sys.stdout.write)]
for t in threads: t.join() # wait for IO completion
print ''.join(queue), # print last N lines
retcode = p.wait()