We Keep Coding

Python,Shell: How to extract data and store in a iterable varible(specifically in list)?

I am working on my own project. In which these steps have to be performed:
Connect to remote server.
Get pid, process name, cpu usage, swap memory usage by each running process on remote server daily on some specific time(say at 4'0 clock).
I have to compare every day's result with previous day's result (e.g. day1-pid with day2 pid and day1 process name with day2 process name etc.)
So far I have done up to step-2. Now I want to know that how to extract the pid, process name, cpu usage, swap memory usage from remote server and store it in some iterable variable. So that I can compare it for checking memory spike?
Any other way apart from my idea will be appreciable.
My code sample is like this:
import paramiko
import re
import psutil
class ShellHandler:
def __init__(self, host, user, psw):
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(host, username=user, password=psw, port=22)
channel = self.ssh.invoke_shell()
self.stdin = channel.makefile('wb')
self.stdout = channel.makefile('r')
def __del__(self):
self.ssh.close()
#staticmethod
def _print_exec_out(cmd, out_buf, err_buf, exit_status):
print('command executed: {}'.format(cmd))
print('STDOUT:')
for line in out_buf:
print(line, end="")
print('end of STDOUT')
print('STDERR:')
for line in err_buf:
print(line, end="")
print('end of STDERR')
print('finished with exit status: {}'.format(exit_status))
print('------------------------------------')
#print(psutil.pids())
pass
def execute(self, cmd):
"""
:param cmd: the command to be executed on the remote computer
:examples: execute('ls')
execute('finger')
execute('cd folder_name')
"""
cmd = cmd.strip('\n')
self.stdin.write(cmd + '\n')
finish = 'end of stdOUT buffer. finished with exit status'
echo_cmd = 'echo {} $?'.format(finish)
self.stdin.write(echo_cmd + '\n')
shin = self.stdin
self.stdin.flush()
shout = []
sherr = []
exit_status = 0
for line in self.stdout:
if str(line).startswith(cmd) or str(line).startswith(echo_cmd):
# up for now filled with shell junk from stdin
shout = []
elif str(line).startswith(finish):
# our finish command ends with the exit status
exit_status = int(str(line).rsplit(maxsplit=1)[1])
if exit_status:
# stderr is combined with stdout.
# thus, swap sherr with shout in a case of failure.
sherr = shout
shout = []
break
else:
# get rid of 'coloring and formatting' special characters
shout.append(re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[#-~]').sub('', line).replace('\b', '').replace('\r', ''))
# first and last lines of shout/sherr contain a prompt
if shout and echo_cmd in shout[-1]:
shout.pop()
if shout and cmd in shout[0]:
shout.pop(0)
if sherr and echo_cmd in sherr[-1]:
sherr.pop()
if sherr and cmd in sherr[0]:
sherr.pop(0)
self._print_exec_out(cmd=cmd, out_buf=shout, err_buf=sherr, exit_status=exit_status)
return shin, shout, sherr
obj=ShellHandler('Servername','username','password')
pID=[]
## I want this(pid, cmd, swap memory) to store in a varible which would be iterable.
pID=ShellHandler.execute(obj,"ps -eo pid,cmd,lstart,%mem,%cpu|awk '{print $1}'")
print(pID[0])##---------------------------------Problem not giving any output.

Your ShellHandler's execute method returns three items, the first of which is the input you sent to it.
You should probably call it directly like this, anyway:
obj = ShellHandler('Servername','username','password')
in, out, err = obj.execute("ps -eo pid,lstart,%mem,%cpu,cmd")
for line in out.split('\n'):
pid, lstartwd, lstartmo, lstartdd, lstartm, lstartyy, mem, cpu, cmd = line.split(None, 8)
I moved cmd last because it might contain spaces. The lstart value also contains multiple space-separated fields. Here's what the output looks like in Debian:
19626 Tue Jan 15 15:03:57 2019 0.0 0.0 less filename
There are many questions about how to parse ps output in more detail; I'll refer you to them for figuring out how to handle the results from split exactly.
Splitting out the output of ps using Python
Is there any way to get ps output programmatically?

ps aux command should have all the info you need (pid, process name, cpu, memory)

How to solve this argparse freeze issue?

I would like to create a very simple terminal menu that uses files and directories so I tried the following:
if __name__ == '__main__':
p = argparse.ArgumentParser(NAME)
p.add_argument('-data:', '--input_data', default=sys.stdin)
p.add_argument('-labels:', '--targets', default=sys.stdin)
p.add_argument('-input_directory:', '--input', default=sys.stdin)
p.add_argument('-output_data:', '--output', default=sys.stdout)
opts = p.parse_args()
if not any([opts.input, opts.output, opts.targets,opts.input_data]):
p.print_usage()
quit()
print '\npress -h to see the help\n'
elif any([opts.targets,opts.input_data]):
print '\ncompleted action\n'
p.print_usage()
#Recibe (datos, etiquetas)
perform_stuff(opts.input_data, opts.targets)
quit()
#Here is the problem:
elif any([opts.input, opts.output]):
print '\ncompleted action\n'
retrive(opts.input)
write(opts.output)
Note that for simplicity I omited the perform_stuff(path/in/file, path/out/file), retrive(opts.input) and write(opts.output) implementation. The problem is that the only command that works is:
if not any([opts.input, opts.output, opts.targets,opts.input_data]):
p.print_usage()
quit()
print '\npress -h to see the help\n'
elif any([opts.targets,opts.input_data]):
print '\ncompleted action\n'
p.print_usage()
#Recibe (datos, etiquetas)
perform_stuff(opts.input_data, opts.targets)
quit()
On the other hand, when I try to run:
#Here is the problem:
elif any([opts.input, opts.output]):
print '\ncompleted action\n'
retrive(opts.input)
write(opts.output)
My simple program just freeze. I am new with argparse, I guess that the problem is correlated with the proper use of if/else statements. Therefore, How should I create the menu of my simple program?

Here I've cleaned up and condensed your code, with a focus on getting the if/else tree working.
In the parser I rewrote the short options as single characters. I also left the defaults as None. That makes if tests easier. (what should it do if opts.input is stdin and you test if not opts.input:?).
import argparse
def quit():
import sys
sys.exit(1)
def perform_stuff(*args):
print 'stuff', args
def retrive(*args):
print 'retrieve', args
if __name__ == '__main__':
p = argparse.ArgumentParser()
# single letter `-d` string; use default None
p.add_argument('-d', '--input_data')
p.add_argument('-t', '--targets')
p.add_argument('-i', '--input')
p.add_argument('-o', '--output')
opts = p.parse_args()
print 'opts', opts
if not any([opts.input, opts.output, opts.targets, opts.input_data]):
p.print_usage()
print 'press -h to see the help'
quit()
elif any([opts.targets,opts.input_data]):
perform_stuff(opts.input_data, opts.targets)
quit()
elif any([opts.input, opts.output]):
retrive(opts.input, opts.output)
else:
print 'else'
Sample runs:
1814:~/mypy$ python stack36877714.py
opts Namespace(input=None, input_data=None, output=None, targets=None)
usage: stack36877714.py [-h] [-d INPUT_DATA] [-t TARGETS] [-i INPUT]
[-o OUTPUT]
press -h to see the help
1815:~/mypy$ python stack36877714.py -d input_data
opts Namespace(input=None, input_data='input_data', output=None, targets=None)
stuff ('input_data', None)
1815:~/mypy$ python stack36877714.py -t targets
opts Namespace(input=None, input_data=None, output=None, targets='targets')
stuff (None, 'targets')
1815:~/mypy$ python stack36877714.py -i input
opts Namespace(input='input', input_data=None, output=None, targets=None)
retrieve ('input', None)
1816:~/mypy$ python stack36877714.py -o output
opts Namespace(input=None, input_data=None, output='output', targets=None)
retrieve (None, 'output')
Your functions could be written like:
def retrieve(in_arg, out_arg):
if in_arg is None:
data = sys.stdin.read()
# cannot open/close stdin
else:
with open(in_arg,'r') as f:
data = f.read()
if out_arg is None:
# write to sys.stdout
else:
# write to open(out_arg,'w')

Gtk.Vte and feed_child: command sent two time

i'm building a tabbed python terminal to manage multiple ssh connection (like superputty or putty manager on windows), so i use feed_child to send command to terminal, but there is a little problem: the command is written 2 time in the terminal, on time before everything.. code is better:
I use this to send the command (in this case for example the command is uptime)
if self.command != "":
self.length_command = len(self.command) + 1
self.hbox.term.feed_child(self.command + "\n", self.length_command)
but in the terminal i get this result
uptime
maurelio#myhost1 ~ $ uptime
19:04:18 up 15 days, 14:32, 2 users, load average: 0,04, 0,07, 0,14
maurelio#myhost1
As you can see, uptime appears in terminal like normal text when i start the program or open a new tab, then appear the prompt and the command is correctly executed. i'd like to get just this:
maurelio#myhost1 ~ $ uptime
19:04:18 up 15 days, 14:32, 2 users, load average: 0,04, 0,07, 0,14
maurelio#myhost1
Any idea?
Update
Here there is a complete code (that should works) :-)
#!/usr/bin/env python
# -*- coding: utf-8; -*-
from gi.repository import Gtk, Vte, GLib
import os
class Terminal(Vte.Terminal):
"""Just create a standard terminal with some configuration
"""
def __init__(self):
super(Terminal, self).__init__()
self.configure_terminal()
def configure_terminal(self):
self.set_audible_bell(False)
self.set_visible_bell(False)
self.set_sensitive(True)
self.fork_command_full(Vte.PtyFlags.DEFAULT, os.environ['HOME'], ["/bin/bash"],
[], GLib.SpawnFlags.DO_NOT_REAP_CHILD, None, None)
class Window(Gtk.Window):
def __init__(self):
Gtk.Window.__init__(self, title="Terminal")
self.box = Gtk.Box(spacing=6)
self.add(self.box)
self.term = Terminal()
self.box.pack_start(self.term, True, True, 0)
self.term.set_audible_bell(False)
self.term.set_visible_bell(False)
self.term.set_sensitive(True)
self.term.fork_command_full(Vte.PtyFlags.DEFAULT, os.environ['HOME'], ["/bin/bash"],
[], GLib.SpawnFlags.DO_NOT_REAP_CHILD, None, None)
self.command = 'uptime'
self.length_command = len(self.command) + 1
self.term.feed_child(self.command + "\n", self.length_command)
self.term.connect("child-exited", Gtk.main_quit)
win = Window()
win.connect("delete-event", Gtk.main_quit)
win.show_all()
Gtk.main()
If you run this, you will see the problem.
Thanks.

You forgot to add the Vte terminal to a ScrolledWindow first:
Gtk.Window.__init__(self, title="Terminal")
self.set_default_size(600, 300)
self.box = Gtk.Box(spacing=6)
self.add(self.box)
self.term = Terminal()
self.sw = Gtk.ScrolledWindow()
self.sw.add(self.term)
self.box.pack_start(self.sw, True, True, 0)

How can I automate this sequence of lldb commands?

In order to work around a bug in Apple's lldb (rdar://13702081) I very frequently need to type two commands in sequence, like this:
(lldb) p todo.matA
(vMAT_Array *) $2 = 0x000000010400b5a0
(lldb) po $2.dump
$3 = 0x0000000100503ce0 <vMAT_Int8Array: 0x10400b5a0; size: [9 1]> =
1
1
1
1
1
1
1
1
1
Is it possible to write a new lldb command using the Python library (or something) that could combine those steps for me? Ideally to something like:
(lldb) pmat todo
$3 = 0x0000000100503ce0 <vMAT_Int8Array: 0x10400b5a0; size: [9 1]> =
1
1
1
1
1
1
1
1
1
Solution
Thanks to Jason Molenda here is output from a working lldb command script:
(lldb) pmat Z
$0 = 0x0000000100112920 <vMAT_DoubleArray: 0x101880c20; size: [9 3]> =
7 9 0.848715
3 5 0.993378
0 1 1.11738
4 12 1.2013
11 13 1.20193
6 10 1.29206
14 15 1.53283
8 16 1.53602
2 17 1.68116
I did have to tweak the script provided in the answer below very slightly, using Jason's suggestions for working around the lldb bug with overly-complex expressions. Here is my final script:
# import this into lldb with a command like
# command script import pmat.py
import lldb
import shlex
import optparse
def pmat(debugger, command, result, dict):
# Use the Shell Lexer to properly parse up command options just like a
# shell would
command_args = shlex.split(command)
parser = create_pmat_options()
try:
(options, args) = parser.parse_args(command_args)
except:
return
target = debugger.GetSelectedTarget()
if target:
process = target.GetProcess()
if process:
frame = process.GetSelectedThread().GetSelectedFrame()
if frame:
var = frame.FindVariable(args[0])
if var:
array = var.GetChildMemberWithName("matA")
if array:
id = array.GetValueAsUnsigned (lldb.LLDB_INVALID_ADDRESS)
if id != lldb.LLDB_INVALID_ADDRESS:
debugger.HandleCommand ('po [0x%x dump]' % id)
def create_pmat_options():
usage = "usage: %prog"
description='''Print a dump of a vMAT_Array instance.'''
parser = optparse.OptionParser(description=description, prog='pmat',usage=usage)
return parser
#
# code that runs when this script is imported into LLDB
#
def __lldb_init_module (debugger, dict):
# This initializer is being run from LLDB in the embedded command interpreter
# Make the options so we can generate the help text for the new LLDB
# command line command prior to registering it with LLDB below
# add pmat
parser = create_pmat_options()
pmat.__doc__ = parser.format_help()
# Add any commands contained in this module to LLDB
debugger.HandleCommand('command script add -f %s.pmat pmat' % __name__)

You can do this either with a regex command or by creating your own python command and loading it in to lldb. In this specific instance the regex command won't help you because you'll hit the same crasher you're hitting. But just for fun, I'll show both solutions.
First, python. This python code gets the currently selected frame on the currently selected thread. It looks for a variable whose name is provided on the command argument. It finds a child of that variable called matA and it runs GetObjectDescription() on that SBValue object.
# import this into lldb with a command like
# command script import pmat.py
import lldb
import shlex
import optparse
def pmat(debugger, command, result, dict):
# Use the Shell Lexer to properly parse up command options just like a
# shell would
command_args = shlex.split(command)
parser = create_pmat_options()
try:
(options, args) = parser.parse_args(command_args)
except:
return
target = debugger.GetSelectedTarget()
if target:
process = target.GetProcess()
if process:
frame = process.GetSelectedThread().GetSelectedFrame()
if frame:
var = frame.FindVariable(args[0])
if var:
child = var.GetChildMemberWithName("matA")
if child:
print child.GetObjectDescription()
def create_pmat_options():
usage = "usage: %prog"
description='''Call po on the child called "matA"'''
parser = optparse.OptionParser(description=description, prog='pmat',usage=usage)
return parser
#
# code that runs when this script is imported into LLDB
#
def __lldb_init_module (debugger, dict):
# This initializer is being run from LLDB in the embedded command interpreter
# Make the options so we can generate the help text for the new LLDB
# command line command prior to registering it with LLDB below
# add pmat
parser = create_pmat_options()
pmat.__doc__ = parser.format_help()
# Add any commands contained in this module to LLDB
debugger.HandleCommand('command script add -f %s.pmat pmat' % __name__)
In use,
(lldb) br s -p break
Breakpoint 2: where = a.out`main + 31 at a.m:8, address = 0x0000000100000eaf
(lldb) r
Process 18223 launched: '/private/tmp/a.out' (x86_64)
Process 18223 stopped
* thread #1: tid = 0x1f03, 0x0000000100000eaf a.out`main + 31 at a.m:8, stop reason = breakpoint 2.1
#0: 0x0000000100000eaf a.out`main + 31 at a.m:8
5 #autoreleasepool {
6 struct var myobj;
7 myobj.matA = #"hello there";
-> 8 printf ("%s\n", [(id)myobj.matA UTF8String]); // break here
9 }
10 }
(lldb) p myobj
(var) $0 = {
(void *) matA = 0x0000000100001070
}
(lldb) comm scri imp ~/lldb/pmat.py
(lldb) pmat myobj
hello there
(lldb)
You can put the command script import line in your ~/.lldbinit file if you want to use this.
It's easy to use the Python APIs once you have a general idea of how the debugger is structured. I knew that I would find the variable based on the frame, so I looked at the help for the SBFrame object with
(lldb) script help (lldb.SBFrame)
The method FindVariable returns an SBValue so then I looked at the lldb.SBValue help page, etc. There's a lot of boilerplate in my example python above - you're really looking at 4 lines of python that do all the work.
If this is still triggering the code path that is crashing your lldb process, you can do the last little bit of the script in two parts - get the address of the object and run po on that raw address. e.g.
child = var.GetChildMemberWithName("matA")
if child:
id = child.GetValueAsUnsigned (lldb.LLDB_INVALID_ADDRESS)
if id != lldb.LLDB_INVALID_ADDRESS:
debugger.HandleCommand ('po 0x%x' % id)
Second, using a command regex:
(lldb) br s -p break
Breakpoint 1: where = a.out`main + 31 at a.m:8, address = 0x0000000100000eaf
(lldb) r
Process 18277 launched: '/private/tmp/a.out' (x86_64)
Process 18277 stopped
* thread #1: tid = 0x1f03, 0x0000000100000eaf a.out`main + 31 at a.m:8, stop reason = breakpoint 1.1
#0: 0x0000000100000eaf a.out`main + 31 at a.m:8
5 #autoreleasepool {
6 struct var myobj;
7 myobj.matA = #"hello there";
-> 8 printf ("%s\n", [(id)myobj.matA UTF8String]); // break here
9 }
10 }
(lldb) command regex pmat 's/(.*)/po %1.matA/'
(lldb) pmat myobj
$0 = 0x0000000100001070 hello there
(lldb)
You can't use the simpler command alias in this instance - you have to use a regex alias - because you're calling a command which takes raw input. Specifically, po is really an alias to expression and you need to use regex command aliases to substitute values into those.

Terminal like app in PySide

I am making an application in PySide and I want to add a Console/Terminal like screen, where you have a prompt and you can type commands. How would I be able to accomplish this. I am guessing some combination of QPlainTextEdit/QTextEdit for the output and QLineEdit for the actual prompt. Is there a better way to do this?

You could check out Spyder. They use PyQt (which is similar) and have a terminal. I think you can import their terminal widget, but I haven't played with it.
https://code.google.com/p/spyderlib/
Also, it is my favorite python editor by far!
I've spent a fair deal of time trying to find something like this, but to no avail. Good luck!

I did this with a custom QPlainTextEdit and custom QLineEdit. I also had an indicator label that would display ">>>" on the terminal to show user input. It needs more work. The best way would be to create your own custom widget based off of a QTextEdit and your own io handler. Below is an example of my execute method with self.input being the QLineEdit and self.view being the QTextEdit. It should get you the general idea.
import io, subprocess, shlex, code, rlcompleter, platform
def execute(self, currentText=None):
"""Execute runs the command given based on the console type.
If the console type is "both" then execute will run python commands
unless the user give the input ::os::command or ("::(platform.syste())::command").
Otherwise the console type will determine the what the input will execute with.
Args:
currentText(str): Text to execute. None will run the text from the QLineEdit self.input.
"""
# Check for given text
if currentText is None:
currentText = self.input.text()
self.input.clear()
self.view.display(currentText, "input")
else:
cmd = re.search("^>>>", currentText) # search for start only
if cmd is None:
currentText = ">>>" + currentText
else:
self.view.display(currentText, "input")
# end
# Variables
self.completer.clear()
cmdString = re.sub("^>>>", "", currentText)
result = None
displayType = "output"
run, cmdString = self.getRunType(cmdString)
try:
# Check where the output is going
sys.stdout = self.out = io.StringIO()
sys.stderr = sys.stdout
if run == "python": # Run python command
result = self._runInterpreter(cmdString)
displayType = "python"
elif run == "os": # Run os command
result = self._runSubprocess(cmdString)
displayType = "os"
except Exception as err:
result = str(err)
displayType = "Error"
notFoundPython = "NameError:" in result and "is not defined" in result
notFoundWindows = "returned non-zero exit status" in result
if notFoundPython or notFoundWindows:
result = "Command not found"
finally:
sys.stdout = self.old_stdout
sys.stderr = self.old_stdout
self.display(result, displayType)
# end execute
def getRunType(self, cmdString):
run = self._consoleType
# Check the run type
if self._consoleType == "both":
if re.search("^::python::", cmdString) is not None:
cmdString = re.sub("^::[a-z]*::", "", cmdString)
run = "python"
elif re.search("^(::os::|::"+platform.system()+"::)", cmdString) is not None:
cmdString = re.sub("^::[a-z]*::", "", cmdString)
run = "os"
else:
run = "python"
# end
return run, cmdString
# end getRunType
def _runInterpreter(self, cmdString, outIO=None, run=None):
# Check for a proper console type
if(self._consoleType != "both" and self._consoleType != "python"):
return
# Get the IO
if outIO is None:
outIO = sys.stdout
# Run python command
self.interpreter.push(cmdString)
# Check outIO
result = "Unreadable buffer: Check python's sys.stdout"
if isinstance(outIO, io.StringIO):
result = outIO.getvalue()
else:
if outIO.readable():
result = str(outIO.readlines())
# Check for error
if re.search("^Traceback", result) or re.search("[a-zA-z]*Error:", result):
raise ValueError(result)
return result
# end _runInterpreter
def _runSubprocess(self, cmdString, run=None):
# Check for a proper console type
if(self._consoleType != "both" and self._consoleType != "os"):
return
# Run OS command
cmd = shlex.split(cmdString)
result = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode("utf-8")
# Check for error
if re.search("^Traceback", result) or re.search("[a-zA-z]*Error:", result):
raise ValueError(result)
return result
# end _runSubprocess