Related
In the Python 2.7 documentation of subprocess module, it provides a simple solution to replace the shell pipeline:
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
output = p2.communicate()[0]
Source: https://docs.python.org/2/library/subprocess.html#replacing-shell-pipeline
I want to have a wrapper function that can support multiple | in command.
Yes, the easiest solution is to set Shell=True which makes the program vulnerable to shell injection.
I'm wondering if there is a neat way to support both multiple | and keep Shell=False
I'm wondering if there is a neat way to support both multiple | and keep shell=False
Yes, the two-command example from the documentation works for any number of commands in the pipeline:
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
p1.stdout.close()
p3 = Popen(["grep", "-v", "audio"], stdin=p2.stdout, stdout=PIPE)
p2.stdout.close()
output = p3.communicate()[0]
For clarity and convenience, this can be extracted to a utility function:
import subprocess
def pipe(*arglists):
prev_proc = subprocess.Popen(arglists[0], stdout=subprocess.PIPE)
for args in arglists[1:]:
proc = subprocess.Popen(args, stdin=prev_proc.stdout, stdout=subprocess.PIPE)
prev_proc.stdout.close()
prev_proc = proc
return prev_proc
>>> p = pipe(["dmesg"], ["grep", "hda"], ["grep", "-v", "audio"])
>>> p.communicate()
('[ 2.654606] snd_hda_intel 0000:0f:00.1: Disabling MSI\n', None)
I am currently using
p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE)
count = int(p3.stdout.read())
if count == 2:
print count
print "yes"
else:
print "fail"
For checking to see if my python is currently running already or not.
It works fine.
However, I want to know if I can combine p1 p2 and p3 into one line.
I tried
p = subprocess.Popen(['ps ax | grep bash | wc -l'], stdout=subprocess.PIPE)
and few more but its not working.Is there a way to combine all these 3 into one line?
You can only use shell-style pipes (|) if you pass the shell=True keyword argument to Popen:
p = subprocess.Popen('ps ax | grep bash | wc -l', stdout=subprocess.PIPE, shell=True)
Otherwise, the entire string you pass is treated like the name of the executable, which will fail, since you don't have a program named ps ax | grep bash | wc -l.
Additionally, you shouldn't pass the command as a list with shell=True, you just pass a string.
One other note: You probably need to adjust your command for this to work reliably, so that the grep bash command itself isn't counted in the output of ps ax | grep bash. One trick to do this is to use ps ax | grep [b]ash. Also, using shell=True might end up starting a bash instance to run the command, so you might need to use if count == 3 instead of if count == 2.
Putting it altogether:
p = subprocess.Popen('ps ax | grep [b]ash | wc -l', stdout=subprocess.PIPE, shell=True)
count = int(p.stdout.read())
if count == 2: # Or maybe 3?
print count
print "yes"
else:
print "fail"
Edit:
Here's output from running different versions of the code on my machine
>>> p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
>>> p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
>>> p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE); p3.stdout.read()
'42\n'
>>> p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
>>> p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
>>> p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE); p3.stdout.read()
'42\n'
>>> p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
>>> p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
>>> p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE); p3.stdout.read()
'42\n'
>>> p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
>>> p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
>>> p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE); p3.stdout.read()
'42\n'
>>> p1 = subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE)
>>> p2 = subprocess.Popen(['grep', 'bash'], stdin=p1.stdout, stdout=subprocess.PIPE)
>>> p3 = subprocess.Popen(['wc', '-l'], stdin=p2.stdout, stdout=subprocess.PIPE); p3.stdout.read()
'42\n'
>>> p = subprocess.Popen('ps -ef | grep bash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
44
>>> p = subprocess.Popen('ps -ef | grep bash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
44
>>> p = subprocess.Popen('ps -ef | grep bash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
44
>>> p = subprocess.Popen('ps -ef | grep [b]ash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
42
>>> p = subprocess.Popen('ps -ef | grep [b]ash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
42
>>> p = subprocess.Popen('ps -ef | grep [b]ash | wc -l ', stdout=subprocess.PIPE, shell=True) ; print(''.join(p.stdout.readlines()))
42
Here is the obvious answer. All I did was replace the second p1 and p2 with the first assignment.
p3 = subprocess.Popen(['wc', '-l'], stdin=subprocess.Popen(['grep', 'bash'], \
stdin=subprocess.Popen(['ps', 'ax'], stdout=subprocess.PIPE).stdout, \
stdout=subprocess.PIPE).stdout, stdout=subprocess.PIPE)
Here is the code:
from subprocess import Popen, PIPE
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p2 = Popen(["grep", "net.ipv4.icmp_echo_ignore_all"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
print output
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p3 = Popen(["grep", "net.ipv4.icmp_echo_ignore_broadcasts"], stdin=p1.stdout, stdout=PIPE)
output1 = p3.communicate()[0]
print output1
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p4 = Popen(["grep", "net.ipv4.ip_forward"], stdin=p1.stdout, stdout=PIPE)
output2 = p4.communicate()[0]
print output2
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p5 = Popen(["grep", "net.ipv4.tcp_syncookies"], stdin=p1.stdout, stdout=PIPE)
output3 = p5.communicate()[0]
print output3
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p6 = Popen(["grep", "net.ipv4.conf.all.rp_filter"], stdin=p1.stdout, stdout=PIPE)
output4 = p6.communicate()[0]
print output4
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p7 = Popen(["grep", "net.ipv4.conf.all.log.martians"], stdin=p1.stdout, stdout=PIPE)
output5 = p7.communicate()[0]
print output5
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p8 = Popen(["grep", "net.ipv4.conf.all.secure_redirects"], stdin=p1.stdout, stdout=PIPE)
output6 = p8.communicate()[0]
print output6
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p9 = Popen(["grep", "net.ipv4.conf.all.send_redirects"], stdin=p1.stdout, stdout=PIPE)
output7 = p9.communicate()[0]
print output7
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p10 = Popen(["grep", "net.ipv4.conf.all.accept_source_route"], stdin=p1.stdout, stdout=PIPE)
output8 = p10.communicate()[0]
print output8
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p11 = Popen(["grep", "net.ipv4.conf.all.accept_redirects"], stdin=p1.stdout, stdout=PIPE)
output9 = p11.communicate()[0]
print output9
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p12 = Popen(["grep", "net.ipv4.tcp_max_syn_backlog"], stdin=p1.stdout, stdout=PIPE)
output10 = p12.communicate()[0]
print output10
current_kernel_para = dict() #new dictionary to store the above kernel parameters
The output of above program is:
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.ip_forward = 0
net.ipv4.tcp_syncookies = 1
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.secure_redirects = 1
net.ipv4.conf.all.send_redirects = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.accept_redirects = 1
net.ipv4.tcp_max_syn_backlog = 512
I want to store these values in a dictionary "current_kernel_para". The desired output is:
{net.ipv4.icmp_echo_ignore_all:0, net.ipv4.icmp_echo_ignore_broadcasts:1} etc.
Please help. Thanks in advance.
you could just split the string at the "=" and use the first as key and second token as value.
split the output on the '='
x = output.split(' = ')
This would give:
['net.ipv4.conf.all.send_redirects', '1']
You can then add all these lists together and use:
x = ['net.ipv4.icmp_echo_ignore_all', '0', 'net.ipv4.conf.all.send_redirects', '1'...]
dict_x = dict(x[i:i+2] for i in range(0, len(x), 2))
current_kernel_para = {}
paras = ["net.ipv4.icmp_echo_ignore_all",
"net.ipv4.icmp_echo_ignore_broadcasts", #...
]
for para in paras:
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p2 = Popen(["grep", para], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
output.strip()
k, v = map(strip, output.split('='))
current_kernel_para[k] = v
Rather than assigning each output to a new variable, just collect it as a list. Remember to strip the trailing \n
outputs = []
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p2 = Popen(["grep", "net.ipv4.icmp_echo_ignore_all"], stdin=p1.stdout, stdout=PIPE)
outputs.append(p2.communicate()[0].strip('\n'))
p1 = Popen(["sysctl", "-a"], stdout=PIPE)
p3 = Popen(["grep", "net.ipv4.icmp_echo_ignore_broadcasts"], stdin=p1.stdout, stdout=PIPE)
outputs.append(p3.communicate()[0].strip('\n'))
This gives
>>> outputs
['net.ipv4.icmp_echo_ignore_all = 0', 'net.ipv4.icmp_echo_ignore_broadcasts = 1']
Then you can split each string in the list at = and collect the result as a list of lists.
outputs_list=[x.split('=') for x in outputs]
Gives
>>> outputs_list
[['net.ipv4.icmp_echo_ignore_all ', ' 0'], ['net.ipv4.icmp_echo_ignore_broadcasts ', ' 1']]
Good, but it has leading/trailing spaces. Lets strip those out too
outputs_list=[[y.strip() for y in x.split('=')] for x in outputs]
This gives
>>> outputs_list
[['net.ipv4.icmp_echo_ignore_all', '0'], ['net.ipv4.icmp_echo_ignore_broadcasts', '1']]
Pass it too the dict() constructor to form a dictionary
>>> dict(outputs_list)
{'net.ipv4.icmp_echo_ignore_all': '0', 'net.ipv4.icmp_echo_ignore_broadcasts': '1'}
This question already has answers here:
How do I use subprocess.Popen to connect multiple processes by pipes?
(9 answers)
Closed 8 years ago.
I'd like to use subprocess on the following line:
convert ../loxie-orig.png bmp:- | mkbitmap -f 2 -s 2 -t 0.48 | potrace -t 5 --progress -s -o ../DSC00232.svg
I found thank to other posts the subprocess documentation but in the example we use only twice pipe.
So, I try for two of the three commands and it works
p1 = subprocess.Popen(['convert', fileIn, 'bmp:-'], stdout=subprocess.PIPE)
# p2 = subprocess.Popen(['mkbitmap', '-f', '2', '-s', '2', '-t', '0.48'], stdout=subprocess.PIPE)
p3 = subprocess.Popen(['potrace', '-t' , '5', '-s' , '-o', fileOut], stdin=p1.stdout,stdout=subprocess.PIPE)
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p3 exits.
output = p3.communicate()[0]
Can you help me for the third command?
Thank you very much.
Just add a third command following the same example:
p1 = subprocess.Popen(['convert', fileIn, 'bmp:-'], stdout=subprocess.PIPE)
p2 = subprocess.Popen(['mkbitmap', '-f', '2', '-s', '2', '-t', '0.48'],
stdin=p1.stdout, stdout=subprocess.PIPE)
p1.stdout.close()
p3 = subprocess.Popen(['potrace', '-t' , '5', '-s' , '-o', fileOut],
stdin=p2.stdout,stdout=subprocess.PIPE)
p2.stdout.close()
output = p3.communicate()[0]
def runPipe(cmds):
try:
p1 = subprocess.Popen(cmds[0].split(' '), stdin = None, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
prev = p1
for cmd in cmds[1:]:
p = subprocess.Popen(cmd.split(' '), stdin = prev.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
prev = p
stdout, stderr = p.communicate()
p.wait()
returncode = p.returncode
except Exception, e:
stderr = str(e)
returncode = -1
if returncode == 0:
return (True, stdout.strip().split('\n'))
else:
return (False, stderr)
Then execute it like:
runPipe(['ls -1','head -n 2', 'head -n 1'])
Use subprocess.Popen() with the option shell=True, and you can pass it your entire command as a single string.
This is the simplest solution and makes it possible to embed a complicated pipeline in python without head-scratching; but in some cases it might not work, e.g. (as #torek commented) if there are spaces in the filenames passed for input or output. In that case, take the trouble to build up the robust solution in the accepted answer.
I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx
I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.
I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output
shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH
"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)
Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)