We Keep Coding

assign variables to argparse not through command line

I am trying to write tests (using pytest) for a script, but I don't know how to create/pass arguments to main(), especially when it doesn't recieve arguments. and even if I change it to def main(args = None): then it'll initiate it on the first line.
tests.py
def test_main(capfd):
main()
out, err = capfd.readouterr()
assert out == "my expected output"
script.py
def init_parser():
parser = argparse.ArgumentParser(description="The script searches one or \
more named input files for lines \
containing a match to a regular \
expression pattern.")
parser.add_argument('regex', help='the regular expression.')
parser.add_argument('infile', nargs='*', type=argparse.FileType('r'), default=[sys.stdin],
help='the name of the file(s) to search.')
group = parser.add_mutually_exclusive_group()
group.add_argument('-u', '--underscore', action='store_true', help='prints \
"^" under the matching text.')
group.add_argument('-c', '--color', action='store_true', help='highlights \
matching text.')
group.add_argument('-m', '--machine', action='store_true', help='generates \
machine readable output.')
return parser
def main():
args = init_parser().parse_args()
for file in args.infile:
for i, line in enumerate(iter(file.readline, '')):
for substring in re.finditer(args.regex, line):
if args.underscore:
underscore_output(file.name, i + 1, line[:-1],
substring.start(), substring.end())
elif args.color:
color_output(file.name, i + 1, line[:-1],
substring.group())
elif args.machine:
machine_output(file.name, i + 1, substring.start(),
substring.group())
else:
print_line(file.name, i + 1, line[:-1])```

In the test code, you would patch out sys.argv before calling main:
def test_main(monkeypatch):
monkeypatch.setattr("sys.argv", ["script.py", "test_regex", "test_infile"])
main()
# put your assertions here

parse_args can take an explicit None argument to tell it to parse sys.argv[1:]. Write your code so that it can be easily tested:
def main(args=None):
args = init_parser().parse_args(args)
...
In production use, you'll call main() to let it parse sys.argv. For tests, you'll pass a specific set of arguments.
def test_main(capfd):
main(['[a-z]*', 'foo.txt', '-u']) # For example
out, err = capfd.readouterr()
assert out == "my expected output"

Selected options activate right subparsers

I have to invoke my script in this way.
script.py multiple --ways aa bb -as abab -bs bebe
In this case -as abab reffers to "aa" option from --ways parameter and -bs bebe to "bb" option.
And all chosed options should affect what "fulfill" method will be used.
If we chose 'aa' and 'bb' there should only be options '-as' and '-bs' not '-cs'.
import sys
from argparse import ArgumentParser
def fulfill_aa_parser(aa_parser):
aa_parser.add_argument('--ass', '-as', type=str, required=True, choices=['abababa'])
def fulfill_bb_parser(aa_parser):
aa_parser.add_argument('--bass', '-bs', type=str, required=True, choices=['bebebe'])
def fulfill_cc_parser(aa_parser):
aa_parser.add_argument('--cass', '-cs', type=str, required=True, choices=['cycycyc'])
def fulfill_multiple_parser(multiple_parser):
multiple_parser.add_argument('--ways', '-w', type=str, choices=['aa','bb', 'cc'], nargs='+', required=True)
def main(argv):
parser = ArgumentParser(description='TEST CASE')
subparsers = parser.add_subparsers(dest='type')
multiple_parser = subparsers.add_parser(
'multiple'
)
aabbparsers = multiple_parser.add_subparsers()
aa_parser = aabbparsers.add_parser('aa')
bb_parser = aabbparsers.add_parser('bb')
cc_parser = aabbparsers.add_parser('cc')
fulfill_multiple_parser(multiple_parser)
fulfill_aa_parser(aa_parser)
fulfill_bb_parser(bb_parser)
fulfill_cc_parser(cc_parser)
args = parser.parse_args(argv)
if args.type is None:
parser.print_help()
return
if __name__ == '__main__':
main(sys.argv[1:])
Parsing this in this way:
fulfill_aa_parser(multiple_parser)
fulfill_bb_parser(multiple_parser)
fulfill_cc_parser(multiple_parser)
will lead to parser always asking for '-as', '-bs' ,'-cs' and options in '--ways' will not affect this
EDIT : \
This is it looks when there is some thought put to it.
Just simply pass parser to this function
def fulfill_apple_argparser(parser):
parser.add_argument("--apple_argument")
def fulfill_banana_argparser(parser):
parser.add_argument("--banana_argument")
def fulfill_peach_argparser(parser):
parser.add_argument("--peach_argument")
def many_fruits_parse(parser, progs=None, list_of_fruits=('apple', 'banana', 'peach')):
progs = progs or []
if len(list_of_fruits) == 0 or parser in progs:
return
fulfill = {'apple': fulfill_apple_argparser, 'banana': fulfill_banana_argparser,
'peach': fulfill_peach_argparser}
subparsers = parser.add_subparsers(title='subparser', dest=parser.prog)
progs.append(parser)
for fruit in list_of_fruits:
secondary = [x for x in list_of_fruits if x != fruit]
fruit_parser = subparsers.add_parser(fruit, help=fruit)
fulfill[fruit](fruit_parser)
many_fruits_parse(fruit_parser, progs, secondary)

add_subparsers creates a special kind of positional argument, one that uses the add_parser command to create choices. Once a valid choice is provided, parsing is passed to that parser.
With
script.py multiple --ways aa bb -as abab -bs bebe
parser passes the task to multiple_parser. The --ways optional then gets 2 values
Namespace(ways=['aa','bb'])
Neither of those strings is used as a value for aabbparsers, and multiple_parser doesn't know what to do with '-as` or '-bs', and (I expect) will raise an error.
With:
script.py multiple aa -as abab
parsing is passed from parser to multiple_parser to aa_parser, which in turn handles '-as abab', producing (I think)
Namespace(as='abab')
Nesting as you do with multiple and aa is the only way to use multiple subparsers. You can't have two subparses 'in-parallel' (e.g. 'aa' and 'bb').
Especially when testing it's a good idea to provide a dest to the add_subparsers command. It gives information on which subparsers is being invoked.

python argparse doesn't take string option start with "--"

I simplifying my problem here. I need to do this:
python test.py --arg1 '--no-route53'
I added the parser like this:
parser.add_argument("--arg1", nargs="?", type=str, help="option")
args = parser.parse_args()
I wanted to get the '--no-route53' as a whole string and use it later in my script. But I keep getting this error:
test.py: error: unrecognized arguments: --no-route53
How can I work around it?
UPDATE1: if i give extra space after '--no-route53', like this and it worked:
python test.py --arg1 '--no-route53 '

I had the exact same issue a while ago. I ended up pre-parsing the sys.argvlist:
def bug_workaround(argv):
# arg needs to be prepended by space in order to not be interpreted as
# an option
add_space = False
args = []
for arg in argv[1:]: # Skip argv[0] as that should not be passed to parse_args
if add_space:
arg = " " + arg
add_space = True if arg == "--args" else False
args.append(arg)
return args
parser = argparse.ArgumentParser(description='My progrm.')
.
.
.
parser.add_argument('--args', type=str,
help='Extra args (in "quotes")')
args = parser.parse_args(bug_workaround(sys.argv))
# Prune added whitespace (for bug workaround)
if args.args:
args.args = args.args[1:]

Take multiple patterns from cli. argparse Python3

I have a python version of grep, that I am building for an assignment. I want my python module to take multiple patterns from the command line, just like grep. However no matter what I do, I keep getting conflicts with my 'debug' argument.
This is what it currently looks like from command line (with -h):
pgreper.py [-h] [--debug] pattern
At the moment I can only search with one pattern:
cat input.txt | ./pgreper.py "were"
I would like to be able to search the input.txt file like so, with multiple patterns:
cat input.txt | ./pgreper.py "were" "you"
However when I try and do this, I get the following error:
pgreper.py: error: unrecognized argument: you
I know that it is related to the fact I generate a pattern to search by reading sys.argv[1]. How would I go about editing my script, to allow it to take multiple patterns from sys.argv, without affecting the optional arguments I have implemented?
Many thanks :)
ps Please ignore my comments, thanks.
#!/usr/bin/python3
import sys
import re
import time
import datetime
import inspect
import argparse
parser = argparse.ArgumentParser(description='Python Grep.')
parser.add_argument('--debug', default='debug', action='store_true', help='Print debug messages')
parser.add_argument('pattern', type=str, help='Pattern for pgrepping')
args = parser.parse_args()
class CodeTrace(object):
def __init__(self, line, pattern):
self.line = line
self.pattern = pattern
# #staticmethod
def trace(self, line, pattern):
# Creating Timestamp
ts = time.time()
# Formatting Timestamp
ts = datetime.datetime.fromtimestamp(ts).strftime('[%Y-%m-%d %H:%M:%S:%f]')
stack = inspect.stack()
# Retrieve calling class information
the_class = stack[1][0].f_locals["self"].__class__
# Retrieve calling method information
the_method = stack[1][0].f_code.co_name
the_variables = stack[1][0].f_code.co_varnames
# Formats the contents of the debug trace into a readable format,
# Any parameters passed to the method and the return value, are included in the debug trace
debug_trace = ("{} {}.{}.{} {} {} ".format(ts, str(the_class), the_method, the_variables, pattern, line))
# Send out the debug trace as a standard error output
sys.stderr.write(debug_trace + "\n")
class Grepper(object):
def __init__(self, pattern):
self.pattern = pattern
# #CodeTrace.trace()
def matchline(self, pattern):
regex = re.compile(self.pattern)
for line in sys.stdin:
if regex.search(line):
sys.stdout.write(line)
if args.debug != 'debug':
(CodeTrace(line, pattern).trace(line, pattern))
def main():
pattern = str(sys.argv[1])
print(sys.argv)
Grepper(pattern).matchline(pattern)
if __name__ == "__main__":
main()

You can tell argparse to expect 1 or more arguments, using the nargs keyword argument:
parser.add_argument('patterns', type=str, nargs='+', help='Pattern(s) for pgrepping')
Here + means 1 or more. You can then combine these patterns:
pattern = '|'.join(['(?:{})'.format(p) for p in args.patterns])
and pass that to your grepper. The patterns are combined with | after first being placed in a non-capturing group ((?:...)) to make sure each pattern is treated as distinct.
I'd place all argument parsing in the main() function here:
def main():
parser = argparse.ArgumentParser(description='Python Grep.')
parser.add_argument('--debug', action='store_true', help='Print debug messages')
parser.add_argument('pattern', type=str, nargs='+', help='Pattern(s) for pgrepping')
args = parser.parse_args()
pattern = '|'.join(['(?:{})'.format(p) for p in args.pattern])
Grepper(pattern, args.debug).matchline()
I also removed the default for the --debug option; using store_true means it'll default to False; you can then simply test for args.debug being true or not.
You don't need to pass in pattern twice to Grepper(); you can simply use self.pattern in the matchline method, throughout. Instead, I'd pass in args.debug to Grepper() as well (no need for it to be a global).
Quick demo of what the argument parsing look like, including the help message:
>>> import argparse
>>> parser = argparse.ArgumentParser(description='Python Grep.')
>>> parser.add_argument('--debug', action='store_true', help='Print debug messages')
_StoreTrueAction(option_strings=['--debug'], dest='debug', nargs=0, const=True, default=False, type=None, choices=None, help='Print debug messages', metavar=None)
>>> parser.add_argument('pattern', type=str, nargs='+', help='Pattern(s) for pgrepping')
_StoreAction(option_strings=[], dest='pattern', nargs='+', const=None, default=None, type=<type 'str'>, choices=None, help='Pattern(s) for pgrepping', metavar=None)
>>> parser.print_help()
usage: [-h] [--debug] pattern [pattern ...]
Python Grep.
positional arguments:
pattern Pattern(s) for pgrepping
optional arguments:
-h, --help show this help message and exit
--debug Print debug messages
>>> parser.parse_args(['where'])
Namespace(debug=False, pattern=['where'])
>>> parser.parse_args(['were'])
Namespace(debug=False, pattern=['were'])
>>> parser.parse_args(['were', 'you'])
Namespace(debug=False, pattern=['were', 'you'])
>>> parser.parse_args(['--debug', 'were', 'you'])
Namespace(debug=True, pattern=['were', 'you'])
The pattern then looks like this:
>>> args = parser.parse_args(['were', 'you'])
>>> args.pattern
['were', 'you']
>>> pattern = '|'.join(['(?:{})'.format(p) for p in args.pattern])
>>> pattern
'(?:were)|(?:you)'
If instead you wanted all patterns to match, you'll need to alter Grepper() to take multiple patterns and test all those patterns. Use the all() function to make that efficient (only test as many patterns as is required):
def main():
parser = argparse.ArgumentParser(description='Python Grep.')
parser.add_argument('--debug', action='store_true', help='Print debug messages')
parser.add_argument('pattern', type=str, nargs='+', help='Pattern(s) for pgrepping')
args = parser.parse_args()
Grepper(args.pattern, args.debug).matchline()
and the Grepper class becomes:
class Grepper(object):
def __init__(self, patterns, debug=False):
self.patterns = [re.compile(p) for p in patterns]
self.debug = debug
def matchline(self, debug):
for line in sys.stdin:
if all(p.search(line) for p in self.patterns):
sys.stdout.write(line)
if self.debug:
CodeTrace(line, self.patterns).trace(line)
with appropriate adjustments for the CodeTrace class.

Your argparse argument configuration were not correct. This is really now how you configure the argument. Check the Python documentation for argparse as there is a very good example in there.
the format should always be yourscript.py -aARGUMENTVAL -bARGUMENTVAL ...etc. the -a and -b styles are important.
Your code is edited to have a better application of argparse module below. See if this works better (without action argument for debug):
import sys
import re
import time
import datetime
import inspect
import argparse
parser = argparse.ArgumentParser(description='Python Grep.')
parser.add_argument('-p', '--pattern', type=str, help='Pattern for pgrepping')
parser.add_argument('-d','--debug', type=str, default="false", help='Print debug messages')
args = vars(parser.parse_args());
class CodeTrace(object):
def __init__(self, line, pattern):
self.line = line
self.pattern = pattern
# #staticmethod
def trace(self, line, pattern):
# Creating Timestamp
ts = time.time()
# Formatting Timestamp
ts = datetime.datetime.fromtimestamp(ts).strftime('[%Y-%m-%d %H:%M:%S:%f]')
stack = inspect.stack()
# Retrieve calling class information
the_class = stack[1][0].f_locals["self"].__class__
# Retrieve calling method information
the_method = stack[1][0].f_code.co_name
the_variables = stack[1][0].f_code.co_varnames
# Formats the contents of the debug trace into a readable format,
# Any parameters passed to the method and the return value, are included in the debug trace
debug_trace = ("{} {}.{}.{} {} {} ".format(ts, str(the_class), the_method, the_variables, pattern, line))
# Send out the debug trace as a standard error output
sys.stderr.write(debug_trace + "\n")
class Grepper(object):
def __init__(self, pattern):
self.pattern = pattern
# #CodeTrace.trace()
def matchline(self, pattern):
regex = re.compile(self.pattern)
for line in sys.stdin:
if regex.search(line):
sys.stdout.write(line)
if args.debug != 'debug':
(CodeTrace(line, pattern).trace(line, pattern))
def main():
pattern = str(args['pattern'])
print(sys.argv)
Grepper(pattern).matchline(pattern)
if __name__ == "__main__":
main()
You can supply comma delimited string to separate patters `-p"were,you". Use python's powerful string functions for that
pattern = ((args['pattern']).replace(" ", "")).split(",");
the above will give you a list of patterns to look for?

Python argparse: nargs + or * depending on prior argument

I'm writing a server querying tool, and I have a little bit of code to parse arguments at the very top:
# Parse arguments
p = argparse.ArgumentParser()
g = p.add_mutually_exclusive_group(required=True)
g.add_argument('--odam', dest='query_type', action='store_const',
const='odam', help="Odamex Master query.")
g.add_argument('--odas', dest='query_type', action='store_const',
const='odas', help="Odamex Server query.")
p.add_argument('address', nargs='*')
args = p.parse_args()
# Default master server arguments.
if args.query_type == 'odam' and not args.address:
args.address = [
'master1.odamex.net:15000',
'master2.odamex.net:15000',
]
# If we don't have any addresses by now, we can't go on.
if not args.address:
print "If you are making a server query, you must pass an address."
sys.exit(1)
Is there a nicer way to do this, preferably all within the parser? That last error looks a little out of place, and it would be nice if I could make nargs for address depend on if --odam or ---odas is passed. I could create a subparser, but that would make help look a little odd since it would leave off the addresses part of the command.

You can do this with an custom argparse.Action:
import argparse
import sys
class AddressAction(argparse.Action):
def __call__(self, parser, args, values, option = None):
args.address=values
if args.query_type=='odam' and not args.address:
args.address=[
'master1.odamex.net:15000',
'master2.odamex.net:15000',
]
if not args.address:
parser.error("If you are making a server query, you must pass an address.")
p = argparse.ArgumentParser()
g = p.add_mutually_exclusive_group(required=True)
g.add_argument('--odam', dest='query_type', action='store_const',
const='odam', help="Odamex Master query.")
g.add_argument('--odas', dest='query_type', action='store_const',
const='odas', help="Odamex Server query.")
p.add_argument('address', nargs='*', action=AddressAction)
args = p.parse_args()
yields
% test.py --odas
If you are making a server query, you must pass an address.
% test.py --odam
Namespace(address=['master1.odamex.net:15000', 'master2.odamex.net:15000'], query_type='odam')
% test.py --odam 1 2 3
Namespace(address=['1', '2', '3'], query_type='odam')