We Keep Coding

Python: Returning 0 in min(list) with no min of 0

I'm creating a command that will take a partial or whole name match of a process and kill its lowest pid - and thus the rest of the processes spawned from it. My code returns a min(list_of_process_ids) of 0, of which there is no min of 0. Please enlighten me as to why this is happening. Thank you.
#!/usr/bin/env python
"""Kill proceses by partial name matching"""
import os, sys
def usage():
return ("pskill.py process_name")
def pids(proc):
""" Find the processes"""
procs = []
procs = os.system("ps -ef|grep -i " + proc + "|grep -v grep|grep -v pfind|awk '{print $2}'")
procs = [int(x) for x in str(procs)]
return procs
def kill(procs):
ppid = min(procs)
os.system("kill " + str(ppid))
return ("Processes Killed...")
def main():
if len(sys.argv) != 2:
print (usage())
else:
proc = sys.argv[1]
pids(proc)
kill(pids(proc))
main()

You aren't grabbing the stdout, so you aren't actually getting anything other that the exist status of the command. Which you can be glad is 0 :-)
Try using the subprocess module. Specifically with the stdout option of piping the result to your python console...

python subprocess.call and pipes [duplicate]

This question already has answers here:
How do I use subprocess.Popen to connect multiple processes by pipes?
(9 answers)
Closed 1 year ago.
I have a script in which I am trying to use subprocess.call to execute a series of shell commands, but which appears to have some commands omitted when executed.
Specifically:
#!/usr/bin/python
import tempfile
import subprocess
import os
import re
grepfd, grepfpath = tempfile.mkstemp(suffix=".xx")
sedfd, sedfpath = tempfile.mkstemp(suffix=".xx")
# grepoutfile = open( grepfpath, 'w')
sedoutfile = open( sedfpath, 'w' )
subprocess.call(['cp','/Users/bobby/Downloads/sample.txt', grepfpath])
sedcmd = [ 'sort',
grepfpath,
'|',
'uniq',
'|',
'sed',
'-e',
'"s/bigstring of word/ smaller /"',
'|',
'column',
'-t',
'-s',
'"=>"' ]
print "sedcmd = ", sedcmd
subprocess.call( ['ls', grepfpath ] )
subprocess.call( ['sort', '|', 'uniq' ], stdin = grepfd )
subprocess.call( sedcmd, stdout = sedoutfile )
And it generates this as output:
python d3.py
sedcmd = ['sort', /var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx', '|', 'uniq', '|', 'sed', '-e', '"s/bigstring of word/ smaller /"', '|', 'column', '-t', '-s', '"=>"']
/var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx
sort: open failed: |: No such file or directory
sort: invalid option -- e
Try `sort --help' for more information.
The first 'sort: open failed: |:No such file... is from the first subprocess call ['sort','|','uniq'], stdin = grepfd )
The 'sort: invalid option -- e .. is from the second subprocess call (sedcmd).
I have seen a lot of examples that use pipes in this context -- so what am I doing wrong?
Thanks!

This is a class that will run a command with an arbitrary number of pipes:
pipeline.py
import shlex
import subprocess
class Pipeline(object):
def __init__(self, command):
self.command = command
self.command_list = self.command.split('|')
self.output = None
self.errors = None
self.status = None
self.result = None
def run(self):
process_list = list()
previous_process = None
for command in self.command_list:
args = shlex.split(command)
if previous_process is None:
process = subprocess.Popen(args, stdout=subprocess.PIPE)
else:
process = subprocess.Popen(args,
stdin=previous_process.stdout,
stdout=subprocess.PIPE)
process_list.append(process)
previous_process = process
last_process = process_list[-1]
self.output, self.errors = last_process.communicate()
self.status = last_process.returncode
self.result = (0 == self.status)
return self.result
This example shows how to use the class:
harness.py
from pipeline import Pipeline
if __name__ == '__main__':
command = '|'.join([
"sort %s",
"uniq",
"sed -e 's/bigstring of word/ smaller /'",
"column -t -s '=>'"
])
command = command % 'sample.txt'
pipeline = Pipeline(command)
if not pipeline.run():
print "ERROR: Pipeline failed"
else:
print pipeline.output
I created this sample file to for testing:
sample.txt
word1>word2=word3
list1>list2=list3
a>bigstring of word=b
blah1>blah2=blah3
Output
a smaller b
blah1 blah2 blah3
list1 list2 list3
word1 word2 word3

So if in a command you want to use shell pipes you can add shell=True in subprocess:
so it will be like this:
sedcmd = 'sort /var/folders/3h/_0xwt5bx0hx8tgx06cmq9h_4f183ql/T/tmp5Gp0ff.xx | uniq | sed -e "s/bigstring of word/ smaller /" | column -t -s "=>" '
subprocess.call(sedcmd, shell=True)
But be carefull with shell=True, it's strongly discouraged to use it : subprocess official documentation
So if you want to use pipes without shell=True you can use subprocees.PIPE in the stdout , and here's an example on how to do it: stackoveflow answer

Resource.getrusage() always returns 0

At the end of a script, I would like to return the peak memory usage. After reading other questions, here is my script:
#!/usr/bin/env python
import sys, os, resource, platform
print platform.platform(), platform.python_version()
os.system("grep 'VmRSS' /proc/%s/status" % os.getpid())
print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
dat = [x for x in xrange(10000000)]
os.system("grep 'VmRSS' /proc/%s/status" % os.getpid())
print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
and here is what I get:
$ test.py
Linux-2.6.18-194.26.1.el5-x86_64-with-redhat-5.5-Final 2.7.2
VmRSS: 4472 kB
0
VmRSS: 322684 kB
0
Why is resource.getrusage always returning me 0?
The same thing happens interactively in a terminal. Can this be due to the way Python was specifically installed on my machine? (It's a computer cluster I'm using with others and managed by admins.)
Edit: same thing happen when I use subprocess; executing this script
#!/usr/bin/env python
import sys, os, resource, platform
from subprocess import Popen, PIPE
print platform.platform(), platform.python_version()
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
print p.communicate()
print "resource:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
dat = [x for x in xrange(10000000)]
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
print p.communicate()
print "resource:", resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
gives this:
$ test.py
Linux-2.6.18-194.26.1.el5-x86_64-with-redhat-5.5-Final 2.7.2
('VmRSS:\t 4940 kB\n', None)
resource: 0
('VmRSS:\t 323152 kB\n', None)
resource: 0

Here's a way to replace the ´os.system´ call
In [131]: from subprocess import Popen, PIPE
In [132]: p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()], shell=False, stdout=PIPE)
In [133]: p.communicate()
Out[133]: ('VmRSS:\t 340832 kB\n', None)
I also have no issue running the line you felt you have problems with:
In [134]: print resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
340840
Edit
The rusage issue could well be a kernel dependent issue and simply not available on your red hat dist http://bytes.com/topic/python/answers/22489-getrusage
You could of course have a separate thread in your code looking at the current usage and storing throughout the execution of the code and store the highest value observed
Edit 2
Here's a full solution skipping resource and monitoring usages via Popen. The frequency of checking must of course be relevant but not frequent so that it eats all cpu.
#!/usr/bin/env python
import threading
import time
import re
import os
from subprocess import Popen, PIPE
maxUsage = 0
keepThreadRunning = True
def memWatch(freq=20):
global maxUsage
global keepThreadRunning
while keepThreadRunning:
p = Popen(["grep", "VmRSS", "/proc/%s/status" % os.getpid()],
shell=False, stdout=PIPE)
curUsage = int(re.search(r'\d+', p.communicate()[0]).group())
if curUsage > maxUsage:
maxUsage = curUsage
time.sleep(1.0 / freq)
if __name__ == "__main__":
t = threading.Thread(target=memWatch)
t.start()
print maxUsage
[p for p in range(1000000)]
print maxUsage
[str(p) for p in range(1000000)]
print maxUsage
keepThreadRunning = False
t.join()
The memWatch function can be optimized by calculating the sleep time once, not reformatting the path to the process each loop and compiling the regular expression before entering the while loop. But in all I hope that was the functionality you sought.

Python: Closing a for loop by reading stdout

import os
dictionaryfile = "/root/john.txt"
pgpencryptedfile = "helloworld.txt.gpg"
array = open(dictionaryfile).readlines()
for x in array:
x = x.rstrip('\n')
newstring = "echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile
os.popen(newstring)
I need to create something inside the for loop that will read gpg's output. When gpg outputs this string gpg: WARNING: message was not integrity protected, I need the loop to close and print Success!
How can I do this, and what is the reasoning behind it?
Thanks Everyone!

import subprocess
def check_file(dictfile, pgpfile):
# Command to run, constructed as a list to prevent shell-escaping accidents
cmd = ["gpg", "--passphrase-fd", "0", pgpfile]
# Launch process, with stdin/stdout wired up to `p.stdout` and `p.stdin`
p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE)
# Read dictfile, and send contents to stdin
passphrase = open(dictfile).read()
p.stdin.write(passphrase)
# Read stdout and check for message
stdout, stderr = p.communicate()
for line in stdout.splitlines():
if line.strip() == "gpg: WARNING: message was not integrity protected":
# Relevant line was found
return True
# Line not found
return False
Then to use:
not_integrity_protected = check_file("/root/john.txt", "helloworld.txt.gpg")
if not_integrity_protected:
print "Success!"
If the "gpg: WARNING:" message is actually on stderr (which I would suspect it is), change the subprocess.Popen line to this:
p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stderr = subprocess.PIPE)
..and the for loop from stdout to stderr, like this:
for line in stderr.splitlines():

Use subprocess.check_output to call gpg and break the loop based on its output.
Something like this (untested since I don't know anything about gpg):
import subprocess
dictionaryfile = "/root/john.txt"
pgpencryptedfile = "helloworld.txt.gpg"
with open(dictionaryfile, 'r') as f:
for line in f:
x = line.rstrip('\n')
cmd = ["echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile]
output = subprocess.check_output(cmd, shell=True)
if 'gpg: WARNING: message was not integrity protected' in output:
break

You could use the subprocess module which allows you to use:
subprocess.call(args, *, stdin, stdout, stderr, shell)
(See the Python Documentation for how to use the parameters.)
This is good because you can easily read in the exit code of whatever program you call.
For example if you change 'newstring' to:
"echo " + x + " | gpg --passphrase-fd 0 " + pgpencryptedfile | grep 'gpg: WARNING: message was not integrity protected'
grep will then return 0 if there is a match and a 1 if not matches are found. (Source)
This exit code from grep will be returned from the subprocess.call() function and you can easily store it in a variable and use an if statement.
Edit: As Matthew Adams mentions below, you could also read the exit code of gpg itself.

How to get output from external command combine with Pipe

I have command like this.
wmctrl -lp | awk '/gedit/ { print $1 }'
And I want its output within python script, i tried this code
>>> import subprocess
>>> proc = subprocess.Popen(["wmctrl -lp", "|","awk '/gedit/ {print $1}"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
>>> proc.stdout.readline()
'0x0160001b -1 6504 beer-laptop x-nautilus-desktop\n'
>>> proc.stdout.readline()
'0x0352f117 0 6963 beer-laptop How to get output from external command combine with Pipe - Stack Overflow - Chromium\n'
>>> proc.stdout.readline()
'0x01400003 -1 6503 beer-laptop Bottom Expanded Edge Panel\n'
>>>
It seem my code is wrong only wmctrl -lp was execute, and | awk '{print $1}' is omitted
My expect output would like 0x03800081
$ wmctrl -lp | awk '/gedit/ {print $1}'
0x03800081
Does one please help.

With shell=True, you should use a single command line instead of an array, otherwise your additional arguments are interpreted as shell arguments. From the subprocess documentation:
On Unix, with shell=True: If args is a string, it specifies the command string to execute through the shell. If args is a sequence, the first item specifies the command string, and any additional items will be treated as additional shell arguments.
So your call should be:
subprocess.Popen("wmctrl -lp | sed /gedit/ '{print $1}'", shell=True, ...
I think you may also have an unbalanced single quote in there.

Because you are passing a sequence in for the program, it thinks that the pipe is an argument to wmcrtrl, such as if you did
wmctrl -lp "|"
and thus the actual pipe operation is lost.
Making it a single string should indeed give you the correct result:
>>> import subprocess as s
>>> proc = s.Popen("echo hello | grep e", shell=True, stdout=s.PIPE, stderr=s.PIPE)
>>> proc.stdout.readline()
'hello\n'
>>> proc.stdout.readline()
''

After some research, I have the following code which works very well for me. It basically prints both stdout and stderr in real time. Hope it helps someone else who needs it.
stdout_result = 1
stderr_result = 1
def stdout_thread(pipe):
global stdout_result
while True:
out = pipe.stdout.read(1)
stdout_result = pipe.poll()
if out == '' and stdout_result is not None:
break
if out != '':
sys.stdout.write(out)
sys.stdout.flush()
def stderr_thread(pipe):
global stderr_result
while True:
err = pipe.stderr.read(1)
stderr_result = pipe.poll()
if err == '' and stderr_result is not None:
break
if err != '':
sys.stdout.write(err)
sys.stdout.flush()
def exec_command(command, cwd=None):
if cwd is not None:
print '[' + ' '.join(command) + '] in ' + cwd
else:
print '[' + ' '.join(command) + ']'
p = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
)
out_thread = threading.Thread(name='stdout_thread', target=stdout_thread, args=(p,))
err_thread = threading.Thread(name='stderr_thread', target=stderr_thread, args=(p,))
err_thread.start()
out_thread.start()
out_thread.join()
err_thread.join()
return stdout_result + stderr_result
When needed, I think it's easy to collect the output or error in a string and return.