We Keep Coding

How to retrieve and validate subprocess.popen arguments when mocking a test?

In following example:
import subprocess
import mock
class MyArgs():
cmd = ''
cmd_args = ''
cmd_path = ''
def __init__(self):
pass
def set_args(self, c, a, p):
self.cmd = c
self.cmd_args = a
self.cmd_path = p
def get_command(self):
return ([self.cmd, self.cmd_args, self.cmd_path])
class Example():
args = MyArgs()
def __init__(self):
pass
def run_ls_command(self):
print 'run_ls_command command:' + str(self.get_command())
p = subprocess.Popen(self.get_command(), stdout=subprocess.PIPE)
out, err = p.communicate()
print out #to verify the mock is working, should output 'output' if the mock is called
return err
def set_args(self, c, a, p):
#this would be more complicated logic in
#future and likely not just one method, this is a MWE
self.args.set_args(c,a,p)
def get_command(self):
return self.args.get_command()
#mock.patch.object(subprocess, 'Popen', autospec=True)
def test_subprocess_popen(mock_popen):
mock_popen.return_value.returncode = 0
mock_popen.return_value.communicate.return_value = ("output", "Error")
e = Example()
e.set_args('ls', '-al', '/bin/foobar')
e.run_ls_command()
#todo: validate arguments called by the popen command for the test
test_subprocess_popen()
The longer term goal is being able to validate more complicated subprocess.Popen commands, which will be constructed by more manipulations on the Example object (though the concept will be the same as this example).
What I would like to do is somehow analyze the arguments sent to the p = subprocess.Popen(self.get_command(), stdout=subprocess.PIPE) command.
However I am not sure how to get those arguments - I know my mock is being called because my output matches expected for the mock.

How to make python scripts pipe-able both in bash and within python

Summary: I'd like to write python scripts that act like bash scripts on the command line, but then I'd also like to pipe them together easily in python. Where I'm having trouble is the glue to make the latter happen.
So imagine I wrote two scripts, script1.py and script2.py and I can pipe them together like so:
echo input_string | ./script1.py -a -b | ./script2.py -c -d
How do I get this behavior from within another python file?
Here's the way I know, but I don't like:
arg_string_1 = convert_to_args(param_1, param_2)
arg_string_2 = convert_to_args(param_3, param_4)
output_string = subprocess.check_output("echo " + input_string + " | ./script1.py " + arg_string_1 + " | ./script2.py " + arg_string_2)
If I didn't want to take advantage of multithreading, I could do something like this (?):
input1 = StringIO(input_string)
output1 = StringIO()
script1.main(param_1, param_2, input1, output1)
input2 = StringIO(output1.get_value())
output2 = StringIO()
script2.main(param_3, param_4, input2, output2)
Here's the approach I was trying, but I got stuck at writing the glue. I'd appreciate either learning how to finish my approach below, or suggestions for a better design/approach!
My approach: I wrote script1.py and script2.py to look like:
#!/usr/bin/python3
... # import sys and define "parse_args"
def main(param_1, param_2, input, output):
for line in input:
...
print(stuff, file=output)
if __name__ == "__main__":
parameter_1, parameter_2 = parse_args(sys.argv)
main(parameter_1, parameter_2, sys.stdin, sys.stdout)
Then I wanted to write something like this, but don't know how to finish:
pipe_out, pipe_in = ????
output = StringIO()
thread_1 = Thread(target=script1.main, args=(param_1, param_2, StreamIO(input_string), pipe_out))
thread_2 = Thread(target=script2.main, args=(param_3, param_4, pipe_in, output)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
output_str = output.get_value()

For the "pipe in", uses sys.stdin with the readlines() method. (Using method read() would read one character at a time.)
For passing information from one thread to another, you can use Queue. You must define one way to signal the end of data. In my example, since all data passed between threads are str, I simply use a None object to signal the end of data (since it cannot appear in the transmitted data).
One could also use more threads, or use different functions in threads.
I did not include the sys.argvin my example to keep it simple. Modifying it to get parameters (parameter1, ...) should be easy.
import sys
from threading import Thread
from Queue import Queue
import fileinput
def stdin_to_queue( output_queue ):
for inp_line in sys.stdin.readlines(): # input one line at at time
output_queue.put( inp_line, True, None ) # blocking, no timeout
output_queue.put( None, True, None ) # signal the end of data
def main1(input_queue, output_queue, arg1, arg2):
do_loop = True
while do_loop:
inp_data = input_queue.get(True)
if inp_data is None:
do_loop = False
output_queue.put( None, True, None ) # signal end of data
else:
out_data = arg1 + inp_data.strip('\r\n').upper() + arg2 # or whatever transformation...
output_queue.put( out_data, True, None )
def queue_to_stdout(input_queue):
do_loop = True
while do_loop:
inp_data = input_queue.get(True)
if inp_data is None:
do_loop = False
else:
sys.stdout.write( inp_data )
def main():
q12 = Queue()
q23 = Queue()
q34 = Queue()
t1 = Thread(target=stdin_to_queue, args=(q12,) )
t2 = Thread(target=main1, args=(q12,q23,'(',')') )
t3 = Thread(target=main1, args=(q23,q34,'[',']') )
t4 = Thread(target=queue_to_stdout, args=(q34,))
t1.start()
t2.start()
t3.start()
t4.start()
main()
Finally, I tested this program (python2) with a text file.
head sometextfile.txt | python script.py

Redirect the return value to stdout depending on whether the script is being run from the command line:
#!/usr/bin/python3
import sys
# Example function
def main(input):
# Do something with input producing stuff
...
return multipipe(stuff)
if __name__ == '__main__':
def multipipe(data):
print(data)
input = parse_args(sys.argv)
main(input)
else:
def multipipe(data):
return data
Each other script will have the same two definitions of multipipe. Now, use multipipe for output.
If you call all the scripts together from the command line $ ./scrip1.py | ./scrip2.py, each will have __name__ == '__main__' and so multipipe will print it all to stdout to be read as an argument by the next script (and return None, so each function returns None, but you're not looking at the return values anyway in this case).
If you call them within some other python script, each function will return whatever you passed to multipipe.
Effectively, you can use your existing functions, just replace print(stuff, file=output) with return multipipe(stuff). Nice and simple.
To use it with multithreading or multiprocessing, set the functions up so that each function returns a single thing, and plug them into a simple function that adds data to a multithreading queue. For an example of such a queueing system, see the sample at the bottom of the Queue docs. With that example, just make sure that each step in the pipeline puts None (or other sentinel value of your choice - I like ... for that since it's extremely rare that you'd pass the Ellipsis object for any reason other than as a marker for its singleton-ness) in the queue to the next one to signify done-ness.

There is a very simple solution using the standard Popen class.
Here's an example:
#this is the master python program
import subprocess
import sys
import os
#note the use of stdin and stdout arguments here
process1 = subprocess.Popen(['./script1.py'], stdin=sys.stdin, stdout=subprocess.PIPE)
process2 = subprocess.Popen(['./script2.py'], stdin=process1.stdout)
process1.wait()
process2.wait()
the two scripts are:
#!/usr/bin/env python
#script1.py
import sys
for line in sys.stdin:
print(line.strip().upper())
Here's the second one
#!/usr/bin/env python
#script2.py
import sys
for line in sys.stdin:
print("<{}>".format(line.strip()))

Python SSH via Pseudo TTY Clear

I wrote the following python module to handle ssh connections in my program:
#!/usr/bin/env python
from vxpty import VX_PTY
class SSHError(Exception):
def __init__(self, msg):
self.msg = msg
def __str__(self):
return repr(self.msg)
class SSHShell:
def __init__(self, host, port, user, password):
self.host = host
self.port = port
self.user = user
self.password = password
self.authenticated = False
def authenticate(self):
self.tty = VX_PTY(['/usr/bin/ssh', 'ssh', '-p'+str(self.port), self.user+'#'+self.host])
resp = self.tty.read()
if "authenticity of host" in resp:
self.tty.println('yes')
while 1:
resp = self.tty.read()
if "added" in resp:
break
resp = self.tty.read()
if "assword:" in resp:
self.tty.println(self.password)
tmp_resp = self.tty.read()
tmp_resp += self.tty.read()
if "denied" in tmp_resp or "assword:" in tmp_resp:
raise(SSHError("Authentication failed"))
else:
self.authenticated = True
self.tty.println("PS1=''")
return self.authenticated
def execute(self, os_cmd):
self.tty.println(os_cmd)
resp_buf = self.tty.read().replace(os_cmd+'\r\n', '')
return resp_buf
Which uses a pty module I wrote earlier:
#!/usr/bin/env python
import os,pty
class PTYError(Exception):
def __init__(self, msg):
self.msg = msg
def __str__(self):
return repr(self.msg)
class VX_PTY:
def __init__(self, execlp_args):
self.execlp_args = execlp_args
self.pty_execlp(execlp_args)
def pty_execlp(self, execlp_args):
(self.pid, self.f) = pty.fork()
if self.pid==0:
os.execlp(*execlp_args)
elif self.pid<0:
raise(PTYError("Failed to fork pty"))
def read(self):
data = None
try:
data = os.read(self.f, 1024)
except Exception:
raise(PTYError("Read failed"))
return data
def write(self, data):
try:
os.write(self.f, data)
except Exception:
raise(PTYError("Write failed"))
def fsync(self):
os.fsync(self.f)
def seek_end(self):
os.lseek(self.f, os.SEEK_END, os.SEEK_CUR)
def println(self, ln):
self.write(ln+'\n')
However, whenever I call the execute() method, I end up reading the output from the first line:
>>> import SSH;shell=SSH.SSHShell('localhost',22,'735tesla','notmypassword');shell.authenticate()
True
>>> shell.execute('whoami')
"\x1b[?1034hLaptop:~ 735Tesla$ PS1=''\r\n"
>>>
Then the second time I call read() I get the output:
>>> shell.tty.read()
'whoami\r\n735Tesla\r\n'
>>>
Removing whoami\r\n from the output is not problem but is there any way to clear the output so I don't have to call read twice with the first command?

I think your problem is deeper than you realize. Luckily, it's also easier to solve than you realize.
What you seem to want is for os.read to return the entirety of what the shell has to send to you in one call. That's not something you can ask for. Depending on several factors, including, but not limited to, the shell's implementation, network bandwidth and latency, and the behavior of the PTYs (yours and the remote host's), the amount of data you'll get back in each call to read can be as much as, well, everything, and as little as a single character.
If you want to receive just the output of your command, you should bracket it with unique markers, and don't worry about messing with PS1. What I mean is that you need to make the shell output a unique string before your command executes and another one after your command executes. Your tty.read method should then return all the text it finds in between these two marker strings. The easiest way to make the shell output these unique strings is just to use the echo command.
For multiline commands, you have to wrap the command in a shell function, and echo the markers before and after executing the function.
A simple implementation is as follows:
def execute(self, cmd):
if '\n' in cmd:
self.pty.println(
'__cmd_func__(){\n%s\n' % cmd +
'}; echo __"cmd_start"__; __cmd_func__; echo __"cmd_end"__; unset -f __cmd_func__'
)
else:
self.pty.println('echo __"cmd_start"__; %s; echo __"cmd_end"__' % cmd)
resp = ''
while not '__cmd_start__\r\n' in resp:
resp += self.pty.read()
resp = resp[resp.find('__cmd_start__\r\n') + 15:] # 15 == len('__cmd_start__\r\n')
while not '_cmd_end__' in resp:
resp += self.pty.read()
return resp[:resp.find('__cmd_end__')]

python: change sys.stdout print to custom print function

Im trying to understand how to create a custom print function.
(using python 2.7)
import sys
class CustomPrint():
def __init__(self):
self.old_stdout=sys.stdout #save stdout
def write(self, text):
sys.stdout = self.old_stdout #restore normal stdout and print
print 'custom Print--->' + text
sys.stdout= self # make stdout use CustomPrint on next 'print'
# this is the line that trigers the problem
# how to avoid this??
myPrint = CustomPrint()
sys.stdout = myPrint
print 'why you make 2 lines??...'
The code above prints this to console:
>>>
custom Print--->why you make 2 lines??...
custom Print--->
>>>
and i want to print only one line:
>>>
1custom Print--->why you make 2 lines??...
>>>
But cant figure out how to make this custom print work , i understand that there's some kind of recursion that triggers the second output to the console (i use self.write , to assign stdout to self.write himself !)
how can i make this work ? or is my approach just completely wrong...

It's not recursion. What happens is your write function is called twice, once with the text you expect, second time with just '\n'. Try this:
import sys
class CustomPrint():
def __init__(self):
self.old_stdout=sys.stdout
def write(self, text):
text = text.rstrip()
if len(text) == 0: return
self.old_stdout.write('custom Print--->' + text + '\n')
def flush(self):
self.old_stdout.flush()
What I do in the above code is I add the new line character to the text passed in the first call, and make sure the second call made by the print statement, the one meant to print new line, doesn't print anything.
Now try to comment out the first two lines and see what happens:
def write(self, text):
#text = text.rstrip()
#if len(text) == 0: return
self.old_stdout.write('custom Print--->' + text + '\n')

One solution may be to use a context manager if it's localised.
#!/usr/bin/env python
from __future__ import print_function
from contextlib import contextmanager
#############################
#contextmanager
def no_stdout():
import sys
old_stdout = sys.stdout
class CustomPrint():
def __init__(self, stdout):
self.old_stdout = stdout
def write(self, text):
if len(text.rstrip()):
self.old_stdout.write('custom Print--->' + text)
sys.stdout = CustomPrint(old_stdout)
try:
yield
finally:
sys.stdout = old_stdout
#############################
print("BEFORE")
with no_stdout():
print("WHY HELLO!\n")
print("DING DONG!\n")
print("AFTER")
The above produces:
BEFORE
custom Print--->WHY HELLO!
custom Print--->DING DONG!
AFTER
The code would need tidying up esp. around what the class should do WRT setting stdout back to what it was.

How about doing from __future__ import print_function. This way you will use Python3 print function instead of print statement from Python2. Then you can redefine the print function:
def print(*args, **kwargs):
__builtins__.print("Custom--->", *args, **kwargs)
There is a catch however, you will have to start using print function.

Creating interruptible process in python

I'm creating a python script of which parses a large (but simple) CSV.
It'll take some time to process. I would like the ability to interrupt the parsing of the CSV so I can continue at a later stage.
Currently I have this - of which lives in a larger class: (unfinished)
Edit:
I have some changed code. But the system will parse over 3 million rows.
def parseData(self)
reader = csv.reader(open(self.file))
for id, title, disc in reader:
print "%-5s %-50s %s" % (id, title, disc)
l = LegacyData()
l.old_id = int(id)
l.name = title
l.disc_number = disc
l.parsed = False
l.save()
This is the old code.
def parseData(self):
#first line start
fields = self.data.next()
for row in self.data:
items = zip(fields, row)
item = {}
for (name, value) in items:
item[name] = value.strip()
self.save(item)
Thanks guys.

If under linux, hit Ctrl-Z and stop the running process. Type "fg" to bring it back and start where you stopped it.

You can use signal to catch the event. This is a mockup of a parser than can catch CTRL-C on windows and stop parsing:
import signal, tme, sys
def onInterupt(signum, frame):
raise Interupted()
try:
#windows
signal.signal(signal.CTRL_C_EVENT, onInterupt)
except:
pass
class Interupted(Exception): pass
class InteruptableParser(object):
def __init__(self, previous_parsed_lines=0):
self.parsed_lines = previous_parsed_lines
def _parse(self, line):
# do stuff
time.sleep(1) #mock up
self.parsed_lines += 1
print 'parsed %d' % self.parsed_lines
def parse(self, filelike):
for line in filelike:
try:
self._parse(line)
except Interupted:
print 'caught interupt'
self.save()
print 'exiting ...'
sys.exit(0)
def save(self):
# do what you need to save state
# like write the parse_lines to a file maybe
pass
parser = InteruptableParser()
parser.parse([1,2,3])
Can't test it though as I'm on linux at the moment.

The way I'd do it:
Puty the actual processing code in a class, and on that class I'd implement the Pickle protocol (http://docs.python.org/library/pickle.html ) (basically, write proper __getstate__ and __setstate__ functions)
This class would accept the filename, keep the open file, and the CSV reader instance as instance members. The __getstate__ method would save the current file position, and setstate would reopen the file, forward it to the proper position, and create a new reader.
I'd perform the actuall work in an __iter__ method, that would yeld to an external function after each line was processed.
This external function would run a "main loop" monitoring input for interrupts (sockets, keyboard, state of an specific file on the filesystem, etc...) - everything being quiet, it would just call for the next iteration of the processor. If an interrupt happens, it would pickle the processor state to an specific file on disk.
When startingm the program just has to check if a there is a saved execution, if so, use pickle to retrieve the executor object, and resume the main loop.
Here goes some (untested) code - the iea is simple enough:
from cPickle import load, dump
import csv
import os, sys
SAVEFILE = "running.pkl"
STOPNOWFILE = "stop.now"
class Processor(object):
def __init__(self, filename):
self.file = open(filename, "rt")
self.reader = csv.reader(self.file)
def __iter__(self):
for line in self.reader():
# do stuff
yield None
def __getstate__(self):
return (self.file.name, self.file.tell())
def __setstate__(self, state):
self.file = open(state[0],"rt")
self.file.seek(state[1])
self.reader = csv.reader(self.File)
def check_for_interrupts():
# Use your imagination here!
# One simple thing would e to check for the existence of an specific file
# on disk.
# But you go all the way up to instantiate a tcp server and listen to
# interruptions on the network
if os.path.exists(STOPNOWFILE):
return True
return False
def main():
if os.path.exists(SAVEFILE):
with open(SAVEFILE) as savefile:
processor = load(savefile)
os.unlink(savefile)
else:
#Assumes the name of the .csv file to be passed on the command line
processor = Processor(sys.argv[1])
for line in processor:
if check_for_interrupts():
with open(SAVEFILE, "wb") as savefile:
dump(processor)
break
if __name__ == "__main__":
main()

My Complete Code
I followed the advice of #jsbueno with a flag - but instead of another file, I kept it within the class as a variable:
I create a class - when I call it asks for ANY input and then begins another process doing my work. As its looped - if I were to press a key, the flag is set and only checked when the loop is called for my next parse. Thus I don't kill the current action.
Adding a process flag in the database for each object from the data I'm calling means I can start this any any time and resume where I left off.
class MultithreadParsing(object):
process = None
process_flag = True
def f(self):
print "\nMultithreadParsing has started\n"
while self.process_flag:
''' get my object from database '''
legacy = LegacyData.objects.filter(parsed=False)[0:1]
if legacy:
print "Processing: %s %s" % (legacy[0].name, legacy[0].disc_number)
for l in legacy:
''' ... Do what I want it to do ...'''
sleep(1)
else:
self.process_flag = False
print "Nothing to parse"
def __init__(self):
self.process = Process(target=self.f)
self.process.start()
print self.process
a = raw_input("Press any key to stop \n")
print "\nKILL FLAG HAS BEEN SENT\n"
if a:
print "\nKILL\n"
self.process_flag = False
Thanks for all you help guys (especially yours #jsbueno) - if it wasn't for you I wouldn't have got this class idea.