We Keep Coding

Display full help text in python click

I am having the following problem and I am fearful there isn't a straghtforward way to solve it so I am asking here. I am using Click to implement a CLI and I have created several grouped commands under the main command. This is the code:
#click.group()
def main():
pass
#main.command()
def getq():
'''Parameters: --questionnaire_id, --question_id, --session_id, --option_id'''
click.echo('Question Answers')
When I type the main command alone in my terminal it lists all the subcommands with the help text next to each one. However, the text is not displayed fully for the case of getq. Instead, it displays only "Parameters: --questionnaire_id, --question_id,... ."
Is there a way to display it all?
Thank You

The easiest way to do this is to use the command's short_help argument:
#click.group()
def main():
pass
#main.command(short_help='Parameters: --questionnaire_id, --question_id, --session_id, --option_id')
def getq():
click.echo('Question Answers')
If you insist to use the docstring for this and want to override the automatic shortening of it, then you could use a custom Group class overriding the format_commands method to directly use cmd.help instead of the get_short_help_str method:
import click
from gettext import gettext as _
class FullHelpGroup(click.Group):
def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
"""Extra format methods for multi methods that adds all the commands
after the options.
"""
commands = []
for subcommand in self.list_commands(ctx):
cmd = self.get_command(ctx, subcommand)
# What is this, the tool lied about a command. Ignore it
if cmd is None:
continue
if cmd.hidden:
continue
commands.append((subcommand, cmd))
# allow for 3 times the default spacing
if len(commands):
limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
rows = []
for subcommand, cmd in commands:
help = cmd.help if cmd.help is not None else ""
rows.append((subcommand, help))
if rows:
with formatter.section(_("Commands")):
formatter.write_dl(rows)
#click.group(cls=FullHelpGroup)
def main():
pass
#main.command()
def getq():
'''Parameters: --questionnaire_id, --question_id, --session_id, --option_id'''
click.echo('Question Answers')
if __name__ == "__main__":
main()
You most probably want to override the max_content_width (at most 80 columns by default) also. You could do this by overriding the context settings:
import shutil
#click.group(cls=FullHelpGroup,
context_settings={'max_content_width': shutil.get_terminal_size().columns - 10})
def main():
pass

How to write a Python script which tests the program and start from command line this way: python unit_test.py program_to_test.py input_data.txt [duplicate]

I would like to set
sys.argv
so I can unit test passing in different combinations. The following doesn't work:
#!/usr/bin/env python
import argparse, sys
def test_parse_args():
global sys.argv
sys.argv = ["prog", "-f", "/home/fenton/project/setup.py"]
setup = get_setup_file()
assert setup == "/home/fenton/project/setup.py"
def get_setup_file():
parser = argparse.ArgumentParser()
parser.add_argument('-f')
args = parser.parse_args()
return args.file
if __name__ == '__main__':
test_parse_args()
Then running the file:
pscripts % ./test.py
File "./test.py", line 4
global sys.argv
^
SyntaxError: invalid syntax
pscripts %

Changing sys.argv at runtime is a pretty fragile way of testing. You should use mock's patch functionality, which can be used as a context manager to substitute one object (or attribute, method, function, etc.) with another, within a given block of code.
The following example uses patch() to effectively "replace" sys.argv with the specified return value (testargs).
try:
# python 3.4+ should use builtin unittest.mock not mock package
from unittest.mock import patch
except ImportError:
from mock import patch
def test_parse_args():
testargs = ["prog", "-f", "/home/fenton/project/setup.py"]
with patch.object(sys, 'argv', testargs):
setup = get_setup_file()
assert setup == "/home/fenton/project/setup.py"

test_argparse.py, the official argparse unittest file, uses several means of setting/using argv:
parser.parse_args(args)
where args is a list of 'words', e.g. ['--foo','test'] or --foo test'.split().
old_sys_argv = sys.argv
sys.argv = [old_sys_argv[0]] + args
try:
return parser.parse_args()
finally:
sys.argv = old_sys_argv
This pushes the args onto sys.argv.
I just came across a case (using mutually_exclusive_groups) where ['--foo','test'] produces different behavior than '--foo test'.split(). It's a subtle point involving the id of strings like test.

global only exposes global variables within your module, and sys.argv is in sys, not your module. Rather than using global sys.argv, use import sys.
You can avoid having to change sys.argv at all, though, quite simply: just let get_setup_file optionally take a list of arguments (defaulting to None) and pass that to parse_args. When get_setup_file is called with no arguments, that argument will be None, and parse_args will fall back to sys.argv. When it is called with a list, it will be used as the program arguments.

I like to use unittest.mock.patch(). The difference to patch.object() is that you don't need a direct reference to the object you want to patch but use a string.
from unittest.mock import patch
with patch("sys.argv", ["file.py", "-h"]):
print(sys.argv)

It doesn't work because you're not actually calling get_setup_file. Your code should read:
import argparse
def test_parse_args():
sys.argv = ["prog", "-f", "/home/fenton/project/setup.py"]
setup = get_setup_file() # << You need the parentheses
assert setup == "/home/fenton/project/setup.py"

I achieved this by creating an execution manager that would set the args of my choice and remove them upon exit:
import sys
class add_resume_flag(object):
def __enter__(self):
sys.argv.append('--resume')
def __exit__(self, typ, value, traceback):
sys.argv = [arg for arg in sys.argv if arg != '--resume']
class MyTestClass(unittest.TestCase):
def test_something(self):
with add_resume_flag():
...

Very good question.
The trick to setting up unit tests is all about making them repeatable. This means that you have to eliminate the variables, so that the tests are repeatable. For example, if you are testing a function that must perform correctly given the current date, then force it to work for specific dates, where the date chosen does not matter, but the chosen dates match in type and range to the real ones.
Here sys.argv will be an list of length at least one. So create a "fakemain" that gets called with a list. Then test for the various likely list lengths, and contents. You can then call your fake main from the real one passing sys.argv, knowing that fakemain works, or alter the "if name..." part to do perform the normal function under non-unit testing conditions.

You'll normally have command arguments. You need to test them. Here is how to unit test them.
Assume program may be run like: % myprogram -f setup.py
We create a list to mimic this behaviour. See line (4)
Then our method that parses args, takes an array as an argument that is defaulted to None. See line (7)
Then on line (11) we pass this into parse_args, which uses the array if it isn't None. If it is None then it defaults to using sys.argv.
1: #!/usr/bin/env python
2: import argparse
3: def test_parse_args():
4: my_argv = ["-f", "setup.py"]
5: setup = get_setup_file(my_argv)
6: assert setup == "setup.py"
7: def get_setup_file(argv=None):
8: parser = argparse.ArgumentParser()
9: parser.add_argument('-f')
10: # if argv is 'None' then it will default to looking at 'sys.argv'
11: args = parser.parse_args(argv)
12: return args.f
13: if __name__ == '__main__':
14: test_parse_args()

You can attach a wrapper around your function, which prepares sys.argv before calling and restores it when leaving:
def run_with_sysargv(func, sys_argv):
""" prepare the call with given sys_argv and cleanup afterwards. """
def patched_func(*args, **kwargs):
old_sys_argv = list(sys.argv)
sys.argv = list(sys_argv)
try:
return func(*args, **kwargs)
except Exception, err:
sys.argv = old_sys_argv
raise err
return patched_func
Then you can simply do
def test_parse_args():
_get_setup_file = run_with_sysargv(get_setup_file,
["prog", "-f", "/home/fenton/project/setup.py"])
setup = _get_setup_file()
assert setup == "/home/fenton/project/setup.py"
Because the errors are passed correctly, it should not interfere with external instances using the testing code, like pytest.

Use embedded argparse in Python3

I want to make a Python module that can be used both by command line and other modules.
Like that :
python3 Capacity.py arg1 arg2 arg3
or
>>> capacity.execByString("arg1 arg2 arg3")
I made a class to (with some researches) get the result of argparse within the code :
class ArgumentParserError(Exception): pass
class Parseur(ArgumentParser):
def error(self, msg):
raise ArgumentParserError(msg)
def analyze(self, args):
if type(args) is not list:
args = args.split() # To work with a String
try:
result = self.parse_args(args)
return True, result
# Returns True and the namespace if OK
except ArgumentParserError as err:
return False, err.args[0]
# Returns False and the error message if not OK
I use it like this :
class Capacity():
def __init__(self):
self.parser = Parseur()
# Config the parser
def execByArguments(*args):
# Do the job
def execByString(command):
isOK, result = self.parser.analyze(command)
if isOk:
# Launch execByArguments with the rights args in result
else:
# Print error message
print(result)
def execFromCommandLine():
args = self.parser.parse_args()
# Launch execByArguments with the rights args
if __name__ == "__main__":
execFromCommandLine()
But there's 2 main problems and surely some I have'nt yet discovered :
the args are not parsed correctly (doubles quotes for example) as the split function has the "spaces" separator
using the -h flag close the program anyway
I'm convinced that making this another Parseur class is useless/not good and there's a workaround.
Launching the module via subprocess is not a good idea neither : I want to get the returned object in that case.
Can you help me to find a cool way to do what i want please ?
Thanks already.
PS : Write code on the online formular is such a pain ^^.

You are not far away. I would do it somehow like this:
class Capacity():
def __init__(self, argv):
# take over and store arguments (or process further parsing)
self.parser = Parseur()
isOk, result = self.parser.analyze(argv)
def argInputValidation(argv):
#checking the command line arguments given by user
#and returning valid argv, otherwise exit program
#with an error message.
return argv
if __name__ == "__main__":
obj = Capacity(argInputValidation(sys.argv[1:]))

Python - unittest trying to call imported custom argparser

I have a unittest that wants to call an imported module to do both parse_os based on the unittest's command-line option but it seems unittest does not recognize the option, any ideas:
./python testParser.py --mac
option --mac not recognized
Usage: testParser.py [options] [test] [...]
Options:
-h, --help Show this message
-v, --verbose Verbose output
-q, --quiet Minimal output
-f, --failfast Stop on first failure
-c, --catch Catch control-C and display results
-b, --buffer Buffer stdout and stderr during test runs
Examples:
testParser.py - run default set of tests
testParser.py MyTestSuite - run suite 'MyTestSuite'
testParser.py MyTestCase.testSomething - run MyTestCase.testSomething
testParser.py MyTestCase - run all 'test*' test methods
in MyTestCase
I want to run my unittest program like this: python testParser.py --mac
EDITTED: Works now by changing 'unittest.main()' to:
runner = unittest.TextTestRunner(stream=stderr_file)
itersuite = unittest.TestLoader().loadTestsFromTestCase(TT28046_ForensicSearchSmokeTest)
runner.run(itersuite)
Unittest program:
import logging
import unittest
from myargparse import *
class MyTest(unittest.TestCase):
def test_parse_os(self):
## Parse the args:
self.install = install_sw(parse_os(arg=""))
print 'Which os? %s' % self.install
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
# get the default logger
logger = logging.getLogger()
# add a file handler
logger.addHandler(logging.FileHandler('stdout.txt', mode='w'))
# set up a stream for all stderr output
stderr_file = open('stderr.txt', 'w')
# attach that stream to the testRunner
unittest.main(testRunner=unittest.TextTestRunner(stream=stderr_file))
My imported module:
import argparse
import os
import sys
def parse_os(arg):
my_os = ''
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mac",
action="store_true")
parser.add_argument("-w", "--win",
action="store_true")
args = parser.parse_args()
if args.mac:
print 'Mac'
my_os = "Mac"
if args.win:
print 'Windows'
my_os = "Windows"
return my_os
def install_sw(my_os):
installed_os = None
if my_os == 'Mac':
print 'Installing Mac...'
installed_os = 'Mac'
if my_os == 'Windows':
print 'Installing Windows...'
installed_os = 'Windows'
return installed_os

The sys.argv variable is a simple list so you can modify/replace it at your wish.
I'd consider using a context manager in this case, on the lines of:
class SysArgv(object):
def __init__(self, argv):
self._old_argv = sys.argv
sys.argv = argv
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
sys.argv = self._old_argv
return False
And used as:
In [4]: with SysArgv(['a', 'b', 'c']):
...: print(sys.argv)
...:
['a', 'b', 'c']
In your case simple wrap the test code like:
with SysArgv(['the_module_name.py', '--mac']):
# test code goes here
and the argparse module will see the arguments you wants.
As for passing the arguments to the unittest module when running the tests, it's possible passing the argv argument to unittest.main. From the documentation:
The `argv` argument can be a list of options passed to the program, with the first element being the program name. If not specified or `None`, the values of `sys.argv` are used.
However in this case you should modify the sys.argv variable before calling unittest.main:
if __name__ == '__main__':
options = ['name_of_module.py'] + sys.argv[-1:] # last argument as option for the test
with SysArgv(sys.argv[:-1]): # or modify how you want
unittest.main(argv=options)

Have you tried using just '-m' instead of '--mac'?
You may also try:
import optparse
parser = optparse.OptionParser()
parser.add_option("-m", "--mac",
dest="mac",
action="store_true",
help="Run as Mac")
parser.add_option("-w", "--win",
dest="win",
action="store_true",
help="Run as Win")
(options, args) = parser.parse_args()

Thank you all for your suggestions but I decided to go with this to limit the changes to my program.
Instead of calling 'unittest.main()', I just changed to call the following:
runner = unittest.TextTestRunner(stream=stderr_file)
itersuite = unittest.TestLoader().loadTestsFromTestCase(MyTest)
runner.run(itersuite)

Based on all the answers here, I originally did this simple hack, and it worked:
# Change sys.argv before unittest tries to parse our args
sys.argv = [sys.argv[0]] # Replace with only the first arg
unittest.main()
The I realized I could still use all the unitest command line args, and in my case I was just passing a bunch of paths, so anything starting with a "-" could just be passed on and there is no need to hack anything since unittest.main() has an argv argument
# Pass on options, and more importantly, don't pass on ALL args
options = [sys.argv[0]] + [a for a in sys.argv if a.startswith("-")]
unittest.main(argv=options)

Just filter out all argparse and not arparse parameters and send then ones not being argparse arguments to sys.argv which is the one unittest uses:
args, notknownargs = parser.parse_known_args()
sys.argv[1:] = notknownargs

How can I get optparse's OptionParser to ignore invalid options?

In python's OptionParser, how can I instruct it to ignore undefined options supplied to method parse_args?
e.g.
I've only defined option --foo for my OptionParser instance, but I call parse_args with list: [ '--foo', '--bar' ]
I don't care if it filters them out of the original list. I just want undefined options ignored.
The reason I'm doing this is because I'm using SCons' AddOption interface to add custom build options. However, some of those options guide the declaration of the targets. Thus I need to parse them out of sys.argv at different points in the script without having access to all the options. In the end, the top level Scons OptionParser will catch all the undefined options in the command line.

Here's one way to have unknown arguments added to the result args of OptionParser.parse_args, with a simple subclass.
from optparse import (OptionParser,BadOptionError,AmbiguousOptionError)
class PassThroughOptionParser(OptionParser):
"""
An unknown option pass-through implementation of OptionParser.
When unknown arguments are encountered, bundle with largs and try again,
until rargs is depleted.
sys.exit(status) will still be called if a known argument is passed
incorrectly (e.g. missing arguments or bad argument types, etc.)
"""
def _process_args(self, largs, rargs, values):
while rargs:
try:
OptionParser._process_args(self,largs,rargs,values)
except (BadOptionError,AmbiguousOptionError), e:
largs.append(e.opt_str)
And here's a snippet to show that it works:
# Show that the pass-through option parser works.
if __name__ == "__main__": #pragma: no cover
parser = PassThroughOptionParser()
parser.add_option('-k', '--known-arg',dest='known_arg',nargs=1, type='int')
(options,args) = parser.parse_args(['--shazbot','--known-arg=1'])
assert args[0] == '--shazbot'
assert options.known_arg == 1
(options,args) = parser.parse_args(['--k','4','--batman-and-robin'])
assert args[0] == '--batman-and-robin'
assert options.known_arg == 4

By default there is no way to modify the behavior of the call to error() that is raised when an undefined option is passed. From the documentation at the bottom of the section on how optparse handles errors:
If optparse‘s default error-handling behaviour does not suit your needs, you’ll need to
subclass OptionParser and override its exit() and/or error() methods.
The simplest example of this would be:
class MyOptionParser(OptionParser):
def error(self, msg):
pass
This would simply make all calls to error() do nothing. Of course this isn't ideal, but I believe that this illustrates what you'd need to do. Keep in mind the docstring from error() and you should be good to go as you proceed:
Print a usage message incorporating 'msg' to stderr and
exit.
If you override this in a subclass, it should not return -- it
should either exit or raise an exception.

Python 2.7 (which didn't exist when this question was asked) now provides the argparse module. You may be able to use ArgumentParser.parse_known_args() to accomplish the goal of this question.

This is pass_through.py example from Optik distribution.
#!/usr/bin/env python
# "Pass-through" option parsing -- an OptionParser that ignores
# unknown options and lets them pile up in the leftover argument
# list. Useful for programs that pass unknown options through
# to a sub-program.
from optparse import OptionParser, BadOptionError
class PassThroughOptionParser(OptionParser):
def _process_long_opt(self, rargs, values):
try:
OptionParser._process_long_opt(self, rargs, values)
except BadOptionError, err:
self.largs.append(err.opt_str)
def _process_short_opts(self, rargs, values):
try:
OptionParser._process_short_opts(self, rargs, values)
except BadOptionError, err:
self.largs.append(err.opt_str)
def main():
parser = PassThroughOptionParser()
parser.add_option("-a", help="some option")
parser.add_option("-b", help="some other option")
parser.add_option("--other", action='store_true',
help="long option that takes no arg")
parser.add_option("--value",
help="long option that takes an arg")
(options, args) = parser.parse_args()
print "options:", options
print "args:", args
main()

Per synack's request in a different answer's comments, I'm posting my hack of a solution which sanitizes the inputs before passing them to the parent OptionParser:
import optparse
import re
import copy
import SCons
class NoErrOptionParser(optparse.OptionParser):
def __init__(self,*args,**kwargs):
self.valid_args_cre_list = []
optparse.OptionParser.__init__(self, *args, **kwargs)
def error(self,msg):
pass
def add_option(self,*args,**kwargs):
self.valid_args_cre_list.append(re.compile('^'+args[0]+'='))
optparse.OptionParser.add_option(self, *args, **kwargs)
def parse_args(self,*args,**kwargs):
# filter out invalid options
args_to_parse = args[0]
new_args_to_parse = []
for a in args_to_parse:
for cre in self.valid_args_cre_list:
if cre.match(a):
new_args_to_parse.append(a)
# nuke old values and insert the new
while len(args_to_parse) > 0:
args_to_parse.pop()
for a in new_args_to_parse:
args_to_parse.append(a)
return optparse.OptionParser.parse_args(self,*args,**kwargs)
def AddOption_and_get_NoErrOptionParser( *args, **kwargs):
apply( SCons.Script.AddOption, args, kwargs)
no_err_optparser = NoErrOptionParser(optparse.SUPPRESS_USAGE)
apply(no_err_optparser.add_option, args, kwargs)
return no_err_optpars