We Keep Coding

python subprocess.call and pipes [duplicate]

This question already has answers here:
How do I use subprocess.Popen to connect multiple processes by pipes?
(9 answers)
Closed 1 year ago.
I have a script in which I am trying to use subprocess.call to execute a series of shell commands, but which appears to have some commands omitted when executed.
Specifically:
#!/usr/bin/python
import tempfile
import subprocess
import os
import re
grepfd, grepfpath = tempfile.mkstemp(suffix=".xx")
sedfd, sedfpath = tempfile.mkstemp(suffix=".xx")
# grepoutfile = open( grepfpath, 'w')
sedoutfile = open( sedfpath, 'w' )
subprocess.call(['cp','/Users/bobby/Downloads/sample.txt', grepfpath])
sedcmd = [ 'sort',
grepfpath,
'|',
'uniq',
'|',
'sed',
'-e',
'"s/bigstring of word/ smaller /"',
'|',
'column',
'-t',
'-s',
'"=>"' ]
print "sedcmd = ", sedcmd
subprocess.call( ['ls', grepfpath ] )
subprocess.call( ['sort', '|', 'uniq' ], stdin = grepfd )
subprocess.call( sedcmd, stdout = sedoutfile )
And it generates this as output:
python d3.py
sedcmd = ['sort', /var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx', '|', 'uniq', '|', 'sed', '-e', '"s/bigstring of word/ smaller /"', '|', 'column', '-t', '-s', '"=>"']
/var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx
sort: open failed: |: No such file or directory
sort: invalid option -- e
Try `sort --help' for more information.
The first 'sort: open failed: |:No such file... is from the first subprocess call ['sort','|','uniq'], stdin = grepfd )
The 'sort: invalid option -- e .. is from the second subprocess call (sedcmd).
I have seen a lot of examples that use pipes in this context -- so what am I doing wrong?
Thanks!

This is a class that will run a command with an arbitrary number of pipes:
pipeline.py
import shlex
import subprocess
class Pipeline(object):
def __init__(self, command):
self.command = command
self.command_list = self.command.split('|')
self.output = None
self.errors = None
self.status = None
self.result = None
def run(self):
process_list = list()
previous_process = None
for command in self.command_list:
args = shlex.split(command)
if previous_process is None:
process = subprocess.Popen(args, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(args,
stdin=previous_process.stdout,
stdout=subprocess.PIPE)
process_list.append(process)
previous_process = process
last_process = process_list[-1]
self.output, self.errors = last_process.communicate()
self.status = last_process.returncode
self.result = (0 == self.status)
return self.result
This example shows how to use the class:
harness.py
from pipeline import Pipeline
if __name__ == '__main__':
command = '|'.join([
"sort %s",
"uniq",
"sed -e 's/bigstring of word/ smaller /'",
"column -t -s '=>'"
])
command = command % 'sample.txt'
pipeline = Pipeline(command)
if not pipeline.run():
print "ERROR: Pipeline failed"
else:
print pipeline.output
I created this sample file to for testing:
sample.txt
word1>word2=word3
list1>list2=list3
a>bigstring of word=b
blah1>blah2=blah3
Output
a smaller b
blah1 blah2 blah3
list1 list2 list3
word1 word2 word3

So if in a command you want to use shell pipes you can add shell=True in subprocess:
so it will be like this:
sedcmd = 'sort /var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx | uniq | sed -e "s/bigstring of word/ smaller /" | column -t -s "=>" '
subprocess.call(sedcmd, shell=True)
But be carefull with shell=True, it's strongly discouraged to use it : subprocess official documentation
So if you want to use pipes without shell=True you can use subprocees.PIPE in the stdout , and here's an example on how to do it: stackoveflow answer

Multiple line shell commands using Python commands module

I'm trying to write a Python function that transforms a given coordinate system to another using gdal. Problem is that I'm trying to execute the command as one string, but in shell, I have to press enter before entering the coordinates.
x = 1815421
y = 557301
ret = []
tmp = commands.getoutput( 'gdaltransform -s_srs \'+proj=lcc +lat_1=34.03333333333333
+lat_2=35.46666666666667 +lat_0=33.5 +lon_0=-118 +x_0=2000000 +y_0=500000 +ellps=GRS80
+units=m +no_defs\' -t_srs epsg:4326 \n' + str(x) + ' ' + str(y) )
I tried it using '\n', but that doesn't work.

My guess is that you run gdaltransform by pressing Enter and the coordinates are read by the program itself from its stdin, not the shell:
from subprocess import Popen, PIPE
p = Popen(['gdaltransform', '-s_srs', ('+proj=lcc '
'+lat_1=34.03333333333333 '
'+lat_2=35.46666666666667 '
'+lat_0=33.5 '
'+lon_0=-118 +x_0=2000000 +y_0=500000 +ellps=GRS80 '
'+units=m +no_defs'), '-t_srs', 'epsg:4326'],
stdin=PIPE, stdout=PIPE, universal_newlines=True) # run the program
output = p.communicate("%s %s\n" % (x, y))[0] # pass coordinates

from subprocess import *
c = 'command 1 && command 2 && command 3'
# for instance: c = 'dir && cd C:\\ && dir'
handle = Popen(c, stdin=PIPE, stderr=PIPE, stdout=PIPE, shell=True)
print handle.stdout.read()
handle.flush()
If i'm not mistaken, the commands will be executed over a "session" and thus keeping whatever niformation you need in between the commands.
More correctly, using shell=True (from what i've been tought) is that it's supposed to be used if given a string of commands rather than a list. If you'd like to use a list suggestions are to do as follows:
import shlex
c = shlex.split("program -w ith -a 'quoted argument'")
handle = Popen(c, stdout=PIPE, stderr=PIPE, stdin=PIPE)
print handle.stdout.read()
And then catch the output, Or you could work with a open stream and use handle.stdin.write() but it's a bit tricky.
Unless you only want to execute, read and die, .communicate() is perfect, or just .check_output(<cmd>)
Good information n how Popen works can be found here (altho different topic): python subprocess stdin.write a string error 22 invalid argument
Solution
Anyway, this should work (you have to redirect STDIN and STDOUT):
from subprocess import *
c = 'gdaltransform -s_srs \'+proj=lcc +lat_1=34.03333333333333 +lat_2=35.46666666666667 +lat_0=33.5 +lon_0=-118 +x_0=2000000 +y_0=500000 +ellps=GRS80 +units=m +no_defs\' -t_srs epsg:4326 \n' + str(x) + ' ' + str(y) + '\n'
handle = Popen(c, stdin=PIPE, stderr=PIPE, stdout=PIPE, shell=True)
print handle.stdout.read()
handle.flush()

awk from Python: wrong subprocess arguments?

I need to run the following (working) command in Python:
ip route list dev eth0 | awk ' /^default/ {print $3}'
Using subprocess, I would have to do the following:
first = "ip route list dev eth0"
second = "awk ' /^default/ {print $3}'"
p1 = subprocess.Popen(first.split(), stdout=subprocess.PIPE)
p2 = subprocess.Popen(second.split(), stdin=p1.stdout, stdout=subprocess.PIPE)
p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits.
output = p2.communicate()[0]
Something went wrong with p2. I get:
>>> awk: cmd. line:1: '
awk: cmd. line:1: ^ invalid char ''' in expression
What should I do? On a terminal it works perfectly.

split splits on any whitespace, including that inside single-quoted arguments. If you really have to, use shlex.split:
import shlex
p2 = subprocess.Popen(shlex.split(second), stdin=p1.stdout, stdout=subprocess.PIPE)
However it usually makes more sense to specify the commands directly:
first = ['ip', 'route', 'list', 'dev', 'eth0']
second = ['awk', ' /^default/ {print $3}']
p1 = subprocess.Popen(first, stdout=subprocess.PIPE)
p2 = subprocess.Popen(second, stdin=p1.stdout, stdout=subprocess.PIPE)

Not the best solution, but while you are waiting for the best answer, you can still do this :
cmd = "ip route list dev eth0 | awk ' /^default/ {print $3}'"
p2 = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)

Python: Closing a for loop by reading stdout

import os
dictionaryfile = "/root/john.txt"
pgpencryptedfile = "helloworld.txt.gpg"
array = open(dictionaryfile).readlines()
for x in array:
x = x.rstrip('\n')
newstring = "echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile
os.popen(newstring)
I need to create something inside the for loop that will read gpg's output. When gpg outputs this string gpg: WARNING: message was not integrity protected, I need the loop to close and print Success!
How can I do this, and what is the reasoning behind it?
Thanks Everyone!

import subprocess
def check_file(dictfile, pgpfile):
# Command to run, constructed as a list to prevent shell-escaping accidents
cmd = ["gpg", "--passphrase-fd", "0", pgpfile]
# Launch process, with stdin/stdout wired up to `p.stdout` and `p.stdin`
p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
# Read dictfile, and send contents to stdin
passphrase = open(dictfile).read()
p.stdin.write(passphrase)
# Read stdout and check for message
stdout, stderr = p.communicate()
for line in stdout.splitlines():
if line.strip() == "gpg: WARNING: message was not integrity protected":
# Relevant line was found
return True
# Line not found
return False
Then to use:
not_integrity_protected = check_file("/root/john.txt", "helloworld.txt.gpg")
if not_integrity_protected:
print "Success!"
If the "gpg: WARNING:" message is actually on stderr (which I would suspect it is), change the subprocess.Popen line to this:
p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stderr = subprocess.PIPE)
..and the for loop from stdout to stderr, like this:
for line in stderr.splitlines():

Use subprocess.check_output to call gpg and break the loop based on its output.
Something like this (untested since I don't know anything about gpg):
import subprocess
dictionaryfile = "/root/john.txt"
pgpencryptedfile = "helloworld.txt.gpg"
with open(dictionaryfile, 'r') as f:
for line in f:
x = line.rstrip('\n')
cmd = ["echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile]
output = subprocess.check_output(cmd, shell=True)
if 'gpg: WARNING: message was not integrity protected' in output:
break

You could use the subprocess module which allows you to use:
subprocess.call(args, *, stdin, stdout, stderr, shell)
(See the Python Documentation for how to use the parameters.)
This is good because you can easily read in the exit code of whatever program you call.
For example if you change 'newstring' to:
"echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile | grep 'gpg: WARNING: message was not integrity protected'
grep will then return 0 if there is a match and a 1 if not matches are found. (Source)
This exit code from grep will be returned from the subprocess.call() function and you can easily store it in a variable and use an if statement.
Edit: As Matthew Adams mentions below, you could also read the exit code of gpg itself.

link several Popen commands with pipes

I know how to run a command using cmd = subprocess.Popen and then subprocess.communicate.
Most of the time I use a string tokenized with shlex.split as 'argv' argument for Popen.
Example with "ls -l":
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
However, pipes seem not to work... For instance, the following example returns noting:
import subprocess
import shlex
print subprocess.Popen(shlex.split(r'ls -l | sed "s/a/b/g"'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()[0]
Can you tell me what I am doing wrong please?
Thx

I think you want to instantiate two separate Popen objects here, one for 'ls' and the other for 'sed'. You'll want to pass the first Popen object's stdout attribute as the stdin argument to the 2nd Popen object.
Example:
p1 = subprocess.Popen('ls ...', stdout=subprocess.PIPE)
p2 = subprocess.Popen('sed ...', stdin=p1.stdout, stdout=subprocess.PIPE)
print p2.communicate()
You can keep chaining this way if you have more commands:
p3 = subprocess.Popen('prog', stdin=p2.stdout, ...)
See the subprocess documentation for more info on how to work with subprocesses.

I've made a little function to help with the piping, hope it helps. It will chain Popens as needed.
from subprocess import Popen, PIPE
import shlex
def run(cmd):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
output, err, exit_code = run("ls -lha /var/log | grep syslog | grep gz")
if exit_code != 0:
print "Output:"
print output
print "Error:"
print err
# Handle error here
else:
# Be happy :D
print output

shlex only splits up spaces according to the shell rules, but does not deal with pipes.
It should, however, work this way:
import subprocess
import shlex
sp_ls = subprocess.Popen(shlex.split(r'ls -l'), stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_sed = subprocess.Popen(shlex.split(r'sed "s/a/b/g"'), stdin = sp_ls.stdout, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
sp_ls.stdin.close() # makes it similiar to /dev/null
output = sp_ls.communicate()[0] # which makes you ignore any errors.
print output
according to help(subprocess)'s
Replacing shell pipe line
-------------------------
output=`dmesg | grep hda`
==>
p1 = Popen(["dmesg"], stdout=PIPE)
p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE)
output = p2.communicate()[0]
HTH

"""
Why don't you use shell
"""
def output_shell(line):
try:
shell_command = Popen(line, stdout=PIPE, stderr=PIPE, shell=True)
except OSError:
return None
except ValueError:
return None
(output, err) = shell_command.communicate()
shell_command.wait()
if shell_command.returncode != 0:
print "Shell command failed to execute"
return None
return str(output)

Thank #hernvnc, #glglgl, and #Jacques Gaudin for the answers. I fixed the code from #hernvnc. His version will cause hanging in some scenarios.
import shlex
from subprocess import PIPE
from subprocess import Popen
def run(cmd, input=None):
"""Runs the given command locally and returns the output, err and exit_code."""
if "|" in cmd:
cmd_parts = cmd.split('|')
else:
cmd_parts = []
cmd_parts.append(cmd)
i = 0
p = {}
for cmd_part in cmd_parts:
cmd_part = cmd_part.strip()
if i == 0:
if input:
p[i]=Popen(shlex.split(cmd_part),stdin=PIPE, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=None, stdout=PIPE, stderr=PIPE)
else:
p[i]=Popen(shlex.split(cmd_part),stdin=p[i-1].stdout, stdout=PIPE, stderr=PIPE)
i = i +1
# close the stdin explicitly, otherwise, the following case will hang.
if input:
p[0].stdin.write(input)
p[0].stdin.close()
(output, err) = p[i-1].communicate()
exit_code = p[0].wait()
return str(output), str(err), exit_code
# test case below
inp = b'[ CMServer State ]\n\nnode node_ip instance state\n--------------------------------------------\n1 linux172 10.90.56.172 1 Primary\n2 linux173 10.90.56.173 2 Standby\n3 linux174 10.90.56.174 3 Standby\n\n[ ETCD State ]\n\nnode node_ip instance state\n--------------------------------------------------\n1 linux172 10.90.56.172 7001 StateFollower\n2 linux173 10.90.56.173 7002 StateLeader\n3 linux174 10.90.56.174 7003 StateFollower\n\n[ Cluster State ]\n\ncluster_state : Normal\nredistributing : No\nbalanced : No\ncurrent_az : AZ_ALL\n\n[ Datanode State ]\n\nnode node_ip instance state | node node_ip instance state | node node_ip instance state\n------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n1 linux172 10.90.56.172 6001 P Standby Normal | 2 linux173 10.90.56.173 6002 S Primary Normal | 3 linux174 10.90.56.174 6003 S Standby Normal'
cmd = "grep -E 'Primary' | tail -1 | awk '{print $3}'"
run(cmd, input=inp)