We Keep Coding

pytest how to use both getoption and parameterize

I have test which run subprocess on certain executable and test the stdout result.
So I use
#conftest.py
def pytest_addoption(parser):
parser.addoption("--executable", action="store")
#pytest.fixture(scope="session", autouse=True)
def pass_executable(request):
try:
return request.config.getoption("--executable")
except AttributeError:
pass
So that I can use command line arg to set the pass the executable. I wish to use this as a global variable across all my tests. However, I have trouble with the tests which requires #pytest.mark.parametrize decorator. So my solution is to create a test_update_path(pass_executable) to update a global variable PATH, which works.
# test.py
PATH = 'defaultpath/app'
def test_update_path(pass_executable):
global PATH
PATH = pass_executable
print("Gloabl path is update to: ")
print(PATH)
def test_1():
# This will work
print("Now next")
print(PATH)
cmd = [PATH]
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'
#pytest.mark.parametrize("args", [1, 2, 3])
def test_2(path, args):
print("Now next")
print(PATH)
cmd = paramparser(PATH, args)
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'
if __name__ == '__main__':
pytest.main()
pytest --executable=newpath/app -s will work fine, but it is an ugly hack. More importantly, it ran a test which was not doing any actual testing. It is also problematic as the argument is not an optional. Without setting --executable. The path will be an NoneType rather than the original default path .
Any suggestion please?
Appreciated.

You don't need global vars, just use the request fixture as test argument to get access to the command line arg, like you already have in pass_executable. This is how I would change both tests:
def test_1(request):
cmd = [request.config.getoption("--executable")]
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'
#pytest.mark.parametrize("arg", [1, 2, 3])
def test_2(request, arg):
cmd = paramparser(request.config.getoption("--executable"), arg)
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'
If you don't like the code duplication in both tests, extract it into a fixture and use it as a test argument, just like the built-in request:
#pytest.fixture
def executable(request):
return request.config.getoption("--executable")
def test_1(executable):
cmd = [executable]
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'
#pytest.mark.parametrize("arg", [1, 2, 3])
def test_2(executable, arg):
cmd = paramparser(executable, arg)
stdout, stderr = run_subprocess(cmd)
assert stdout == 'some expected result'

Python method not executed

Total noob here. I'm trying to create a python object and execute methods in an instance therein and it seems the code block I want to execute just won't run. The code block in question is run_job which when called just seems to do nothing. What am I doing wrong?
import datetime
import uuid
import paramiko
class scan_job(object):
def __init__(self, protocol, target, user_name, password, command):
self.guid = uuid.uuid1()
self.start_time = datetime.datetime.now()
self.target = target
self.command = command
self.user_name = user_name
self.password = password
self.protocol = protocol
self.result = ""
def run_job(self):
if self.protocol == 'SSH':
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
print "creating connection"
ssh.connect(self.target, self.user_name, self.password)
print "connected"
stdin, stdout, stderr = ssh.exec_command(self.command)
for line in stdout:
print '... ' + line.strip('\n')
self.result += line.strip('\n')
yield ssh
finally:
print "closing connection"
ssh.close()
print "closed"
else:
print "Unknown protocol"
def show_command(self):
print self.command
test = scan_job('SSH', '192.168.14.10', 'myuser', 'mypassword', 'uname -n')
test.show_command()
test.run_job()

Your method contains a yield statement, which makes it a generator. Generators are evaluated lazily. Consider:
>>> def gen():
... yield 10
... yield 3
... yield 42
...
>>> result = gen()
>>> next(result)
10
>>> next(result)
3
>>> next(result)
42
>>>
This is likely not what you intended to do.

Yield is a keyword that is used like return, except the function will
return a generator.
To read more about generators:
1) Understanding Generators in Python
2) What does the "yield" keyword do in Python?
3) Understanding the Yield Keyword in Python
All you need to do is, change:
yield ssh
To:
return ssh
So that, run_job will execute like a normal function, until it reaches its end, exception or return statement. However, if you want to run it without changing the yield statement. Here is how you can do it:
x = test.run_job()
x.next()

How to retrieve and validate subprocess.popen arguments when mocking a test?

In following example:
import subprocess
import mock
class MyArgs():
cmd = ''
cmd_args = ''
cmd_path = ''
def __init__(self):
pass
def set_args(self, c, a, p):
self.cmd = c
self.cmd_args = a
self.cmd_path = p
def get_command(self):
return ([self.cmd, self.cmd_args, self.cmd_path])
class Example():
args = MyArgs()
def __init__(self):
pass
def run_ls_command(self):
print 'run_ls_command command:' + str(self.get_command())
p = subprocess.Popen(self.get_command(), stdout=subprocess.PIPE)
out, err = p.communicate()
print out #to verify the mock is working, should output 'output' if the mock is called
return err
def set_args(self, c, a, p):
#this would be more complicated logic in
#future and likely not just one method, this is a MWE
self.args.set_args(c,a,p)
def get_command(self):
return self.args.get_command()
#mock.patch.object(subprocess, 'Popen', autospec=True)
def test_subprocess_popen(mock_popen):
mock_popen.return_value.returncode = 0
mock_popen.return_value.communicate.return_value = ("output", "Error")
e = Example()
e.set_args('ls', '-al', '/bin/foobar')
e.run_ls_command()
#todo: validate arguments called by the popen command for the test
test_subprocess_popen()
The longer term goal is being able to validate more complicated subprocess.Popen commands, which will be constructed by more manipulations on the Example object (though the concept will be the same as this example).
What I would like to do is somehow analyze the arguments sent to the p = subprocess.Popen(self.get_command(), stdout=subprocess.PIPE) command.
However I am not sure how to get those arguments - I know my mock is being called because my output matches expected for the mock.

How to make python scripts pipe-able both in bash and within python

Summary: I'd like to write python scripts that act like bash scripts on the command line, but then I'd also like to pipe them together easily in python. Where I'm having trouble is the glue to make the latter happen.
So imagine I wrote two scripts, script1.py and script2.py and I can pipe them together like so:
echo input_string | ./script1.py -a -b | ./script2.py -c -d
How do I get this behavior from within another python file?
Here's the way I know, but I don't like:
arg_string_1 = convert_to_args(param_1, param_2)
arg_string_2 = convert_to_args(param_3, param_4)
output_string = subprocess.check_output("echo " + input_string + " | ./script1.py " + arg_string_1 + " | ./script2.py " + arg_string_2)
If I didn't want to take advantage of multithreading, I could do something like this (?):
input1 = StringIO(input_string)
output1 = StringIO()
script1.main(param_1, param_2, input1, output1)
input2 = StringIO(output1.get_value())
output2 = StringIO()
script2.main(param_3, param_4, input2, output2)
Here's the approach I was trying, but I got stuck at writing the glue. I'd appreciate either learning how to finish my approach below, or suggestions for a better design/approach!
My approach: I wrote script1.py and script2.py to look like:
#!/usr/bin/python3
... # import sys and define "parse_args"
def main(param_1, param_2, input, output):
for line in input:
...
print(stuff, file=output)
if __name__ == "__main__":
parameter_1, parameter_2 = parse_args(sys.argv)
main(parameter_1, parameter_2, sys.stdin, sys.stdout)
Then I wanted to write something like this, but don't know how to finish:
pipe_out, pipe_in = ????
output = StringIO()
thread_1 = Thread(target=script1.main, args=(param_1, param_2, StreamIO(input_string), pipe_out))
thread_2 = Thread(target=script2.main, args=(param_3, param_4, pipe_in, output)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
output_str = output.get_value()

For the "pipe in", uses sys.stdin with the readlines() method. (Using method read() would read one character at a time.)
For passing information from one thread to another, you can use Queue. You must define one way to signal the end of data. In my example, since all data passed between threads are str, I simply use a None object to signal the end of data (since it cannot appear in the transmitted data).
One could also use more threads, or use different functions in threads.
I did not include the sys.argvin my example to keep it simple. Modifying it to get parameters (parameter1, ...) should be easy.
import sys
from threading import Thread
from Queue import Queue
import fileinput
def stdin_to_queue( output_queue ):
for inp_line in sys.stdin.readlines(): # input one line at at time
output_queue.put( inp_line, True, None ) # blocking, no timeout
output_queue.put( None, True, None ) # signal the end of data
def main1(input_queue, output_queue, arg1, arg2):
do_loop = True
while do_loop:
inp_data = input_queue.get(True)
if inp_data is None:
do_loop = False
output_queue.put( None, True, None ) # signal end of data
else:
out_data = arg1 + inp_data.strip('\r\n').upper() + arg2 # or whatever transformation...
output_queue.put( out_data, True, None )
def queue_to_stdout(input_queue):
do_loop = True
while do_loop:
inp_data = input_queue.get(True)
if inp_data is None:
do_loop = False
else:
sys.stdout.write( inp_data )
def main():
q12 = Queue()
q23 = Queue()
q34 = Queue()
t1 = Thread(target=stdin_to_queue, args=(q12,) )
t2 = Thread(target=main1, args=(q12,q23,'(',')') )
t3 = Thread(target=main1, args=(q23,q34,'[',']') )
t4 = Thread(target=queue_to_stdout, args=(q34,))
t1.start()
t2.start()
t3.start()
t4.start()
main()
Finally, I tested this program (python2) with a text file.
head sometextfile.txt | python script.py

Redirect the return value to stdout depending on whether the script is being run from the command line:
#!/usr/bin/python3
import sys
# Example function
def main(input):
# Do something with input producing stuff
...
return multipipe(stuff)
if __name__ == '__main__':
def multipipe(data):
print(data)
input = parse_args(sys.argv)
main(input)
else:
def multipipe(data):
return data
Each other script will have the same two definitions of multipipe. Now, use multipipe for output.
If you call all the scripts together from the command line $ ./scrip1.py | ./scrip2.py, each will have __name__ == '__main__' and so multipipe will print it all to stdout to be read as an argument by the next script (and return None, so each function returns None, but you're not looking at the return values anyway in this case).
If you call them within some other python script, each function will return whatever you passed to multipipe.
Effectively, you can use your existing functions, just replace print(stuff, file=output) with return multipipe(stuff). Nice and simple.
To use it with multithreading or multiprocessing, set the functions up so that each function returns a single thing, and plug them into a simple function that adds data to a multithreading queue. For an example of such a queueing system, see the sample at the bottom of the Queue docs. With that example, just make sure that each step in the pipeline puts None (or other sentinel value of your choice - I like ... for that since it's extremely rare that you'd pass the Ellipsis object for any reason other than as a marker for its singleton-ness) in the queue to the next one to signify done-ness.

There is a very simple solution using the standard Popen class.
Here's an example:
#this is the master python program
import subprocess
import sys
import os
#note the use of stdin and stdout arguments here
process1 = subprocess.Popen(['./script1.py'], stdin=sys.stdin, stdout=subprocess.PIPE)
process2 = subprocess.Popen(['./script2.py'], stdin=process1.stdout)
process1.wait()
process2.wait()
the two scripts are:
#!/usr/bin/env python
#script1.py
import sys
for line in sys.stdin:
print(line.strip().upper())
Here's the second one
#!/usr/bin/env python
#script2.py
import sys
for line in sys.stdin:
print("<{}>".format(line.strip()))

write() not called when subclassing code.InteractiveInterpreter

When subclassing code.InteractiveInterpreter I can't seem to get the write() method to run as I would expect per the documentation.
import code
class PythonInterpreter(code.InteractiveInterpreter):
def __init__(self, localVars):
self.runResult = ''
print 'init called'
code.InteractiveInterpreter.__init__(self, localVars)
def write(self, data):
print 'write called'
self.runResult = data
test = 'Hello'
interpreter = PythonInterpreter({'test':test})
interpreter.runcode('print test')
print 'Result:' + interpreter.runResult
Expected output:
init called
write called
Result: Hello
Actual output:
init called
Hello <- shouldn't print
Result:
Any thoughts?

The write method is not used by the code passed to runcode at all. You would have to redirect stdout for this to work, e.g. something like:
import code
class PythonInterpreter(code.InteractiveInterpreter):
def __init__(self, localVars):
self.runResult = ''
print 'init called'
code.InteractiveInterpreter.__init__(self, localVars)
def write(self, data):
# since sys.stdout is probably redirected,
# we can't use print
sys.__stdout__.write('write called\n')
self.runResult = data
def runcode(cd):
# redirecting stdout to our method write before calling code cd
sys.stdout = self
code.InteractiveInterpreter.runcode(self,cd)
# redirecting back to normal stdout
sys.stdout = sys.__stdout__