Related
The following script (which should take the output from p1 and pipe it to p2, and then output the result in the terminal) doesn't seem to work as expected.
Code as follows :
#!/binr/bin/python
# Script to lauch mosquitto_sub to monitor a mqtt topic -- SYNTAX : mosk topic
import sys
import subprocess
total = len(sys.argv)
cmdargs = str(sys.argv)
print ("The total numbers of args passed to the script: %d " % total)
print ("Args list: %s " % cmdargs)
# Pharsing args one by one
print ("Script name: %s" % str(sys.argv[0]))
print ("First argument: %s" % str(sys.argv[1]))
path = str(sys.argv[1])
print (path)
p1 = subprocess.Popen(['mosquitto_sub','-h','192.168.1.100','-t',path,'-v'], shell=False, stdout=subprocess.PIPE)
p2 = subprocess.Popen(['ts'], stdin=p1.stdout, shell=False, stdout=subprocess.PIPE)
for line in p2.stdout:
sys.stdout.write(line)
with an input as follows "./mosk2.py test/+" and whilst publishing MQTT via mosquitto on the relevant topics, I never get the expected output in the terminal
Solved - I ended up stepping neatly around the problem (cheating) as follows :
cmd = "mosquitto_sub -h 192.168.1.100 -v -t " + topic + " | ts"
print "The command which was executed is : " , cmd
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
while True:
output = process.stdout.readline()
if output == '' and process.poll() is not None:
break
if output:
print output.strip()
rc = process.poll()
return rc
run_command(cmd) #This is the lauch of the actual command
this is the function i am creating, i have one argument that can tell to print real time or not since some of the process take like an hour. and since i am subprocesing several at the same time, another argument to raise an error and stop everything, or just let the main script run.
but if i do print_real_time True, i loose the p.communicate()
i could store all the prints from the iter in a variable and return that, but how do i put in order the std out, and the stderr, and get the return value to see if did fail or not?
def launch_subprocess_cmd(command_to_lunch, cwd=None, print_real_time=False, raise_errors=True):
"""
for a given command line will lunch that as a subprocess
:param command_to_lunch: string
:param print_real_time: boolean
:param cwd: the folder path from where the command should be run.
:param raise_errors: boolean if the return code of the subprocess is different than 0 raise an error an stop all scripts.
else the main script will keep running and can access the third return value of this function and decide what to do with it.
:return: list com return the stdout and the stderr of the Popen subprocess.
"""
if cwd is None:
p = subprocess.Popen(command_to_lunch, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
else:
p = subprocess.Popen(command_to_lunch, cwd=cwd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
if print_real_time is True:
for i in iter(p.stdout.readline, b''):
print i
com = p.communicate()
if raise_errors is True:
if p.returncode != 0:
raise ValueError("\n\nSubprocess fail: \n" + "Error captures: \n" + "stdout:\n" + com[0] + "\nstderr:\n" + com[1] + "\n")
# com[0] is std_out, com[1] is std_err and p.return code is if the subprocess was successful or not with a int number
return com[0], com[1], p.returncode
thanks guys =)
A possible solution to your problem is to store the stdout stream in a list when print_real_time is True and then use the content of the list to generate the stdout data string. If print_real_time is not True, then use the content from com[0] instead.
def launch_subprocess_cmd(cmd, cwd=None, print_real_time=False, raise_errors=True):
"""
for a given command line will lunch that as a subprocess
:param cmd: string
:param print_real_time: boolean
:param cwd: the folder path from where the command should be run.
:param raise_errors: boolean if the return code of the subprocess is different
than 0 raise an error an stop all scripts else
the main script will keep running and can access the third
return value of this function and decide what to do with it.
:return: list com return the stdout and the stderr of the Popen subprocess.
"""
if cwd is None:
p = subprocess.Popen(cmd, stderr=subprocess.PIPE,
stdout=subprocess.PIPE, shell=True)
else:
p = subprocess.Popen(cmd, cwd=cwd, stderr=subprocess.PIPE,
stdout=subprocess.PIPE, shell=True)
stdout_list = []
if print_real_time is True:
for i in iter(p.stdout.readline, b''):
stdout_list.append(i)
print i
com = p.communicate()
stdout_data = "".join(stdout_list) if print_real_time is True else com[0]
if raise_errors is True:
if p.returncode != 0:
raise ValueError("\n\nSubprocess fail: \n" + "Error captures: \n" +
"stdout:\n" + stdout_data + "\nstderr:\n" +
com[1] + "\n")
# stdout_data is stdout, com[1] is stderr and
# p.return code is if the subprocess was successful or not with a int number
return stdout_data, com[1], p.returncode
As a side note, I would also urge you to try to rewrite the program to not use shell=True in your Popen calls. It may require that you preprocess the input cmd into a list of base command and arguments, but it is generally considered a bad idea to use shell=True.
launch_subprocess_cmd(command_to_lunch, cwd=None, print_real_time=False, raise_errors=True):
if cwd is None:
p = subprocess.Popen(command_to_lunch, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
else:
p = subprocess.Popen(command_to_lunch, cwd=cwd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
stdout_list = []
errout_list = []
if print_real_time is True:
for i in iter(p.stdout.readline, b''):
stdout_list.append(i)
print i
for j in iter(p.stderr.readline, b''):
errout_list.append(j)
print j
com = p.communicate()
if print_real_time is True:
stdout_data = "".join(stdout_list)
std_err_data = "".join(errout_list)
else:
stdout_data = com[0]
std_err_data = com[1]
if raise_errors is True:
if p.returncode != 0:
raise ValueError("\n\npopen fail:\n" + command_to_lunch + "\nError:\n" + "Error captures:\n" + "stdout:\n" + stdout_data + "\nstderr:\n" + std_err_data + "\n")
# com[0] is out, com[1] is errors and p.return code is if it fail or not
return stdout_data, std_err_data, p.returncode
I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx
I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.
I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output
shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH
"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)
Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)
I am trying to capture the output of a tcpdump/grep pipeline from Python. I am using Python 2.6 on Mac OS 10.6.7.
When I try it with dmesg/grep, the caller receives output from the subprocesses, as expected.
When I try it with tcpdump/grep, select never returns anything.
What am I doing wrong?
#! /usr/bin/python
def tcpdump():
import subprocess, fcntl, os
# This works
# cmd1 = ['sudo', 'dmesg']
# cmd2 = ['grep', '-E', '.*']
# This doesn't work
# sudo tcpdump -i en0 -n -s 0 -w - | grep -a -o -E "Host\: .*|GET \/.*"
cmd1 = ['sudo', 'tcpdump', '-i', 'en0', '-n', '-s', '0', '-w', '-']
cmd2 = ['grep', '-a', '-o', '-E', 'Host\: .*|GET \/.*']
p1 = subprocess.Popen(cmd1, stdout=subprocess.PIPE)
p2 = subprocess.Popen(cmd2, stdout=subprocess.PIPE, stdin=p1.stdout)
# set stdout file descriptor to nonblocking
flags = \
fcntl.fcntl(p2.stdout.fileno(), fcntl.F_GETFL)
fcntl.fcntl(p2.stdout.fileno(), fcntl.F_SETFL, (flags | os.O_NDELAY | os.O_NONBLOCK))
return p2
def poll_tcpdump(proc):
import select
txt = None
while True:
# wait 1/10 of a second and check whether proc has written anything to stdout
readReady, _, _ = select.select([proc.stdout.fileno()], [], [], 0.1)
if not len(readReady):
break
for line in iter(proc.stdout.readline, ""):
if txt is None:
txt = ''
txt += line
break
return txt
proc = tcpdump()
while True:
text = poll_tcpdump(proc)
if text:
print '>>>> ' + text
Try
cmd2 = ['grep', '--line-buffered', '-a', '-o', '-E', 'Host\: .*|GET \/.*']
I have command like this.
wmctrl -lp | awk '/gedit/ { print $1 }'
And I want its output within python script, i tried this code
>>> import subprocess
>>> proc = subprocess.Popen(["wmctrl -lp", "|","awk '/gedit/ {print $1}"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
>>> proc.stdout.readline()
'0x0160001b -1 6504 beer-laptop x-nautilus-desktop\n'
>>> proc.stdout.readline()
'0x0352f117 0 6963 beer-laptop How to get output from external command combine with Pipe - Stack Overflow - Chromium\n'
>>> proc.stdout.readline()
'0x01400003 -1 6503 beer-laptop Bottom Expanded Edge Panel\n'
>>>
It seem my code is wrong only wmctrl -lp was execute, and | awk '{print $1}' is omitted
My expect output would like 0x03800081
$ wmctrl -lp | awk '/gedit/ {print $1}'
0x03800081
Does one please help.
With shell=True, you should use a single command line instead of an array, otherwise your additional arguments are interpreted as shell arguments. From the subprocess documentation:
On Unix, with shell=True: If args is a string, it specifies the command string to execute through the shell. If args is a sequence, the first item specifies the command string, and any additional items will be treated as additional shell arguments.
So your call should be:
subprocess.Popen("wmctrl -lp | sed /gedit/ '{print $1}'", shell=True, ...
I think you may also have an unbalanced single quote in there.
Because you are passing a sequence in for the program, it thinks that the pipe is an argument to wmcrtrl, such as if you did
wmctrl -lp "|"
and thus the actual pipe operation is lost.
Making it a single string should indeed give you the correct result:
>>> import subprocess as s
>>> proc = s.Popen("echo hello | grep e", shell=True, stdout=s.PIPE, stderr=s.PIPE)
>>> proc.stdout.readline()
'hello\n'
>>> proc.stdout.readline()
''
After some research, I have the following code which works very well for me. It basically prints both stdout and stderr in real time. Hope it helps someone else who needs it.
stdout_result = 1
stderr_result = 1
def stdout_thread(pipe):
global stdout_result
while True:
out = pipe.stdout.read(1)
stdout_result = pipe.poll()
if out == '' and stdout_result is not None:
break
if out != '':
sys.stdout.write(out)
sys.stdout.flush()
def stderr_thread(pipe):
global stderr_result
while True:
err = pipe.stderr.read(1)
stderr_result = pipe.poll()
if err == '' and stderr_result is not None:
break
if err != '':
sys.stdout.write(err)
sys.stdout.flush()
def exec_command(command, cwd=None):
if cwd is not None:
print '[' + ' '.join(command) + '] in ' + cwd
else:
print '[' + ' '.join(command) + ']'
p = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
)
out_thread = threading.Thread(name='stdout_thread', target=stdout_thread, args=(p,))
err_thread = threading.Thread(name='stderr_thread', target=stderr_thread, args=(p,))
err_thread.start()
out_thread.start()
out_thread.join()
err_thread.join()
return stdout_result + stderr_result
When needed, I think it's easy to collect the output or error in a string and return.