I am attempting to use argparse to convert an argument into a timedelta object. My program reads in strings supplied by the user and converts them to various datetime objects for later usage. I cannot get the filter_length argument to process correctly though. My code:
import datetime
import time
import argparse
def mkdate(datestring):
return datetime.datetime.strptime(datestring, '%Y-%m-%d').date()
def mktime(timestring):
return datetime.datetime.strptime(timestring, '%I:%M%p').time()
def mkdelta(deltatuple):
return datetime.timedelta(deltatuple)
parser = argparse.ArgumentParser()
parser.add_argument('start_date', type=mkdate, nargs=1)
parser.add_argument('start_time', type=mktime, nargs=1, )
parser.add_argument('filter_length', type=mkdelta, nargs=1, default=datetime.timedelta(1))#default filter length is 1 day.
I run the program, passing 1 as the timedelta value (I only want it to be one day):
> python program.py 2012-09-16 11:00am 1
But I get the following error:
>>> program.py: error: argument filter_length: invalid mkdelta value: '1'
I don't understand why the value is invalid. If I call the mkdelta function on its own, like this:
mkdelta(1)
print mkdelta(1)
It returns:
datetime.timedelta(1)
1 day, 0:00:00
This is exactly the value that I'm looking for. Can someone help me figure out how to do this conversion properly using argparse?
Notice the quotes around '1' in your error message? You pass a string to mkdelta, whereas in your test code, you pass an integer.
Your function doesn't handle a string argument, which is what argparse is handing it; call int() on it:
def mkdelta(deltatuple):
return datetime.timedelta(int(deltatuple))
If you need to support more than days, you'll have to find a way to parse the argument passed in into timedelta arguments.
You could, for example, support d, h, m or s postfixes to denote days, hours, minutes or seconds:
_units = dict(d=60*60*24, h=60*60, m=60, s=1)
def mkdelta(deltavalue):
seconds = 0
defaultunit = unit = _units['d'] # default to days
value = ''
for ch in list(str(deltavalue).strip()):
if ch.isdigit():
value += ch
continue
if ch in _units:
unit = _units[ch]
if value:
seconds += unit * int(value)
value = ''
unit = defaultunit
continue
if ch in ' \t':
# skip whitespace
continue
raise ValueError('Invalid time delta: %s' % deltavalue)
if value:
seconds = unit * int(value)
return datetime.timedelta(seconds=seconds)
Now your mkdelta method accepts more complete deltas, and even integers still:
>>> mkdelta('1d')
datetime.timedelta(1)
>>> mkdelta('10s')
datetime.timedelta(0, 10)
>>> mkdelta('5d 10h 3m 10s')
datetime.timedelta(5, 36190)
>>> mkdelta(5)
datetime.timedelta(5)
>>> mkdelta('1')
datetime.timedelta(1)
The default unit is days.
You could use a custom action to collect all the remaining args and parse them into a timedelta.
This will allow you to write CLI commands such as
% test.py 2012-09-16 11:00am 2 3 4 5
datetime.timedelta(2, 3, 5004) # args.filter_length
You could also provide optional arguments for --days, --seconds, etc, so you can write CLI commands such as
% test.py 2012-09-16 11:00am --weeks 6 --days 0
datetime.timedelta(42) # args.filter_length
% test.py 2012-09-16 11:00am --weeks 6.5 --days 0
datetime.timedelta(45, 43200)
import datetime as dt
import argparse
def mkdate(datestring):
return dt.datetime.strptime(datestring, '%Y-%m-%d').date()
def mktime(timestring):
return dt.datetime.strptime(timestring, '%I:%M%p').time()
class TimeDeltaAction(argparse.Action):
def __call__(self, parser, args, values, option_string = None):
# print '{n} {v} {o}'.format(n = args, v = values, o = option_string)
setattr(args, self.dest, dt.timedelta(*map(float, values)))
parser = argparse.ArgumentParser()
parser.add_argument('start_date', type = mkdate)
parser.add_argument('start_time', type = mktime)
parser.add_argument('--days', type = float, default = 1)
parser.add_argument('--seconds', type = float, default = 0)
parser.add_argument('--microseconds', type = float, default = 0)
parser.add_argument('--milliseconds', type = float, default = 0)
parser.add_argument('--minutes', type = float, default = 0)
parser.add_argument('--hours', type = float, default = 0)
parser.add_argument('--weeks', type = float, default = 0)
parser.add_argument('filter_length', nargs = '*', action = TimeDeltaAction)
args = parser.parse_args()
if not args.filter_length:
args.filter_length = dt.timedelta(
args.days, args.seconds, args.microseconds, args.milliseconds,
args.minutes, args.hours, args.weeks)
print(repr(args.filter_length))
This gist seems to solve your problem: https://gist.github.com/jnothman/4057689
Just in case somebody lands here looking to add a pandas.Timedelta to an argument parser (as I just did), it turns out the following works just fine:
parser = argparse.ArgumentParser()
parser.add_argument('--timeout', type=pd.Timedelta)
Then:
>>> parser.parse_args(['--timeout', '24 hours'])
Namespace(timeout=Timedelta('1 days 00:00:00'))
Related
I'm looking for a conversion of just am/pm string into time so I could do comparison between 2 different time of the day. I tried using time.strptime or something similar but it seems they all require date as well as time.
My code below:
current_hour = 12
current_minute = 37
current_section = "PM"
due_hour = 9
due_minute = 0
due_section = "AM"
import datetime
ct_time = str(datetime.time(current_hour, current_minute))+current_section
print(ct_time)
due_time = str(datetime.time(due_hour, due_minute))+due_section
print(due_time)
ct_time_str = time.strptime(ct_time, '%H:%M:%S') # how to format this to time?
due_time_str= time.strptime(due_time,'%H:%M:%S') # how to format this to time?
if (ct_time_str>due_time_str):
print("still have time to turn in assignment")
else:
print("too late")
Getting the below error, not sure how to convert to 'time' from str.
Traceback (most recent call last):
File "main.py", line 15, in <module>
ct_time_str = time.strptime(ct_time, '%H:%M:%S')
NameError: name 'time' is not defined
datetime can be confusing because both the module and class are called datetime.
Change your import to from datetime import datetime, time. Also imports should go at the very top, but it's not strictly necessary.
When assigning ct_time and due_time, you use str(datetime.time(args)), it should just be str(time(args)).
strptime is from datetime, not time, so change time.strptime(args) to datetime.strptime(args)
Also like DeepSpace & martineau said, you need to add '%p' to the format string to account for the AM/PM part.
Final code:
from datetime import datetime, time
current_hour = 12
current_minute = 37
current_section = "PM"
due_hour = 9
due_minute = 0
due_section = "AM"
ct_time = str(time(current_hour, current_minute))+current_section
print(ct_time)
due_time = str(time(due_hour, due_minute))+due_section
print(due_time)
ct_time_str = datetime.strptime(ct_time, '%H:%M:%S%p')
due_time_str= datetime.strptime(due_time,'%H:%M:%S%p')
if (ct_time_str < due_time_str):
print("still have time to turn in assignment")
else:
print("too late")
edit:
changed if (ct_time_str < due_time_str): to if (ct_time_str > due_time_str):
How can I implement an optional parameter to an AWS Glue Job?
I have created a job that currently have a string parameter (an ISO 8601 date string) as an input that is used in the ETL job. I would like to make this parameter optional, so that the job use a default value if it is not provided (e.g. using datetime.now and datetime.isoformatin my case). I have tried using getResolvedOptions:
import sys
from awsglue.utils import getResolvedOptions
args = getResolvedOptions(sys.argv, ['ISO_8601_STRING'])
However, when I am not passing an --ISO_8601_STRING job parameter I see the following error:
awsglue.utils.GlueArgumentError: argument --ISO_8601_STRING is required
matsev and Yuriy solutions is fine if you have only one field which is optional.
I wrote a wrapper function for python that is more generic and handle different corner cases (mandatory fields and/or optional fields with values).
import sys
from awsglue.utils import getResolvedOptions
def get_glue_args(mandatory_fields, default_optional_args):
"""
This is a wrapper of the glue function getResolvedOptions to take care of the following case :
* Handling optional arguments and/or mandatory arguments
* Optional arguments with default value
NOTE:
* DO NOT USE '-' while defining args as the getResolvedOptions with replace them with '_'
* All fields would be return as a string type with getResolvedOptions
Arguments:
mandatory_fields {list} -- list of mandatory fields for the job
default_optional_args {dict} -- dict for optional fields with their default value
Returns:
dict -- given args with default value of optional args not filled
"""
# The glue args are available in sys.argv with an extra '--'
given_optional_fields_key = list(set([i[2:] for i in sys.argv]).intersection([i for i in default_optional_args]))
args = getResolvedOptions(sys.argv,
mandatory_fields+given_optional_fields_key)
# Overwrite default value if optional args are provided
default_optional_args.update(args)
return default_optional_args
Usage :
# Defining mandatory/optional args
mandatory_fields = ['my_mandatory_field_1','my_mandatory_field_2']
default_optional_args = {'optional_field_1':'myvalue1', 'optional_field_2':'myvalue2'}
# Retrieve args
args = get_glue_args(mandatory_fields, default_optional_args)
# Access element as dict with args[‘key’]
Porting Yuriy's answer to Python solved my problem:
if ('--{}'.format('ISO_8601_STRING') in sys.argv):
args = getResolvedOptions(sys.argv, ['ISO_8601_STRING'])
else:
args = {'ISO_8601_STRING': datetime.datetime.now().isoformat()}
There is a workaround to have optional parameters. The idea is to examine arguments before resolving them (Scala):
val argName = 'ISO_8601_STRING'
var argValue = null
if (sysArgs.contains(s"--$argName"))
argValue = GlueArgParser.getResolvedOptions(sysArgs, Array(argName))(argName)
I don't see a way to have optional parameters, but you can specify default parameters on the job itself, and then if you don't pass that parameter when you run the job, your job will receive the default value (note that the default value can't be blank).
Wrapping matsev's answer in a function:
def get_glue_env_var(key, default="none"):
if f'--{key}' in sys.argv:
return getResolvedOptions(sys.argv, [key])[key]
else:
return default
It's possible to create a Step Function that starts the same Glue job with different parameters. The state machine starts with a Choice state and uses different number of inputs depending on which is present.
stepFunctions:
stateMachines:
taskMachine:
role:
Fn::GetAtt: [ TaskExecutor, Arn ]
name: ${self:service}-${opt:stage}
definition:
StartAt: DefaultOrNot
States:
DefaultOrNot:
Type: Choice
Choices:
- Variable: "$.optional_input"
IsPresent: false
Next: DefaultTask
- Variable: "$. optional_input"
IsPresent: true
Next: OptionalTask
OptionalTask:
Type: Task
Resource: "arn:aws:states:::glue:startJobRun.task0"
Parameters:
JobName: ${self:service}-${opt:stage}
Arguments:
'--log_group.$': "$.specs.log_group"
'--log_stream.$': "$.specs.log_stream"
'--optional_input.$': "$. optional_input"
Catch:
- ErrorEquals: [ 'States.TaskFailed' ]
ResultPath: "$.errorInfo"
Next: TaskFailed
Next: ExitExecution
DefaultTask:
Type: Task
Resource: "arn:aws:states:::glue:startJobRun.sync"
Parameters:
JobName: ${self:service}-${opt:stage}
Arguments:
'--log_group.$': "$.specs.log_group"
'--log_stream.$': "$.specs.log_stream"
Catch:
- ErrorEquals: [ 'States.TaskFailed' ]
ResultPath: "$.errorInfo"
Next: TaskFailed
Next: ExitExecution
TaskFailed:
Type: Fail
Error: "Failure"
ExitExecution:
Type: Pass
End: True
If you're using the interface, you must provide your parameter names starting with "--" like "--TABLE_NAME", rather than "TABLE_NAME", then you can use them like the following (python) code:
args = getResolvedOptions(sys.argv, ['JOB_NAME', 'TABLE_NAME'])
table_name = args['TABLE_NAME']
I've been trying a few days now to get this code to work on MacOS but with no success. Can you please have a look at and see what I'm missing?
Running python 3.6 and I've uploaded the whole code. Thanks a lot
#!/usr/bin/env python3
from __future__ import print_function
try:
import pymarketcap
import decimal
from operator import itemgetter
import argparse
except Exception as e:
print('Make sure to install {0} (pip3 install {0}).'.format(str(e).split("'")[1]))
exit()
# arguments and setup
parser = argparse.ArgumentParser()
parser.add_argument('-m','--minimum_vol',type=float,help='Minimum Percent volume per exchange to have to count',default=1)
parser.add_argument('-p','--pairs',nargs='*',default=[],help='Pairs the coins can be arbitraged with - default=all')
parser.add_argument('-c','--coins_shown',type=int,default=10,help='Number of coins to show')
parser.add_argument('-e','--exchanges',nargs='*',help='Acceptable Exchanges - default=all',default=[])
parser.add_argument('-s','--simple',help='Toggle off errors and settings',default=False,action="store_true")
args = parser.parse_args()
cmc = pymarketcap.Pymarketcap()
info = []
count = 1
lowercase_exchanges = [x.lower() for x in args.exchanges]
all_exchanges = not bool(args.exchanges)
all_trading_pairs = not bool(args.pairs)
coin_format = '{: <25} {: >6}% {: >10} {: >15} {: <10} {: <10} {: <15} {: <5}'
if not args.simple:
print('CURRENT SETTINGS\n* MINIMUM_PERCENT_VOL:{}\n* TRADING_PAIRS:{}\n* COINS_SHOWN:{}\n* EXCHANGES:{}\n* ALL_TRADING_PAIRS:{}\n* ALL_EXCHANGES:{}\n'.format(args.minimum_vol,args.pairs,args.coins_shown,lowercase_exchanges,all_trading_pairs,all_exchanges))
# retrieve coin data
for coin in cmc.ticker():
try:
markets = cmc.markets(coin["id"])
except Exception as e:
markets = cmc.markets(coin["symbol"])
best_price = 0
best_exchange = ''
best_pair = ''
worst_price = 999999
worst_exchange = ''
worst_pair = ''
has_markets = False
for market in markets:
trades_into = market["pair"].replace (coin["symbol"]," ").replace("-"," ")
if market['percent_volume'] >= args.minimum_vol and market['updated'] and (trades_into in args.pairs or all_trading_pairs) and (market['exchange'].lower() in lowercase_exchanges or all_exchanges):
has_markets = True
if market['price_usd'] >= best_price:
best_price = market['price_usd']
best_exchange = market['exchange']
best_pair = trades_into
if market['price_usd'] <= worst_price:
worst_price = market['price_usd']
worst_exchange = market['exchange']
worst_pair = trades_into
if has_markets:
info.append([coin['name'],round((best_price/worst_price-1)*100,2),worst_price,worst_exchange,worst_pair,best_price,best_exchange,best_pair])
elif not args.simple:
print(coin['name'],'had no markets that fit the criteria.')
print('[{}/100]'.format(count),end='\r')
count += 1
# show data
info = sorted(info,key=itemgetter(1))[::-1]
print(coin_format.format("COIN","CHANGE","BUY PRICE","BUY AT","BUY WITH","SELL PRICE","SELL AT","SELL WITH"))
for coin in info[0:args.coins_shown]:
print(coin_format.format(*coin))#
also link - https://gist.github.com/anonymous/ec0643e0b27f33cf628fdafce29a698c
The error that I keep getting is this :
Traceback (most recent call last):
File "./run", line 45, in <module>
trades_into = market["pair"].replace(coin["symbol"],"").replace("-","")
TypeError: replace() argument 1 must be str, not int
Really appreciate your help with this.
your error is telling you all you need to know coin["symbol"] is an int, but replace accepts only Strings. You can fix this by doing str(coin["symbol"])
It says the argument you are passing is not in correct format, requires string and you have passed int
write this
trades_into = market["pair"].replace(str(coin["symbol"])," ").replace("-"," ")
instead of
trades_into = market["pair"].replace (coin["symbol"]," ").replace("-"," ")
Normally you can concatenate options like '-abbb', which will expand to '-a -b -b -b'. Counts would be 1 for a, abd 3 for b.
However when mixing prefix_chars I see something different ...
import argparse
parser = argparse.ArgumentParser( prefix_chars='-+' )
parser.add_argument( '-x', action='count', dest='counter1' )
parser.add_argument( '+x', action='count', dest='counter2' )
args = parser.parse_args( '-xxx +xxx -xxx'.split() )
print( 'counter1 = ' + str(args.counter1) )
print( 'counter2 = ' + str(args.counter2) )
Running this results in:
counter1 = 8
counter2 = 1
Apparently '+xxx' doesn't expand to '+x +x +x', but to '+x -x -x'.
Changing the prefix_chars to '+-' results in:
counter1 = 2
counter2 = 7
Now '-xxx' expands to '-x +x +x'.
Is this defined behaviour, or am I missing something?
This was patched in late 2010, in early 2.7
http://bugs.python.org/issue9352
================
I'm not aware of bug/issues or code changes that would affect this, but I could dig into it.
For a start, strings of single prefix options are handled rather deeply in the parsing. In the current argparse.py the relevant code is:
class ArgumentParser
def _parse_known_args
# function to convert arg_strings into an optional action
def consume_optional(start_index):
match_argument = self._match_argument
action_tuples = []
while True:
...
chars = self.prefix_chars # e.g. the `-+` parameter
if arg_count == 0 and option_string[1] not in chars:
action_tuples.append((action, [], option_string))
char = option_string[0]
option_string = char + explicit_arg[0]
new_explicit_arg = explicit_arg[1:] or None
optionals_map = self._option_string_actions
if option_string in optionals_map:
action = optionals_map[option_string]
explicit_arg = new_explicit_arg
else:
msg = _('ignored explicit argument %r')
raise ArgumentError(action, msg % explicit_arg)
It's the pair of lines:
char = option_string[0]
option_string = char + explicit_arg[0]
that preserves the initial -/+ when handling the repeated characters (in the unparsed explicit_arg string.
I can imagine the case where the code split +xyz into +x,-y,-z, and was corrected to use +x,+y,+z. But it will require some digging into bug/issues and/or the Python repository to find out if and when that change was made.
What does your problem argparse.py have at this point?
I am a Python re-newbie. I would like advice on handling program parameters which are in a file in json format. Currently, I am doing something like what is shown below, however, it seems too wordy, and the idea of typing the same literal string multiple times (sometimes with dashes and sometimes with underscores) seems juvenile - error prone - stinky... :-) (I do have many more parameters!)
#!/usr/bin/env python
import sys
import os
import json ## for control file parsing
# control parameters
mpi_nodes = 1
cluster_size = None
initial_cutoff = None
# ...
#process the arguments
if len(sys.argv) != 2:
raise Exception(
"""Usage:
run_foo <controls.json>
Where:
<control.json> is a dictionary of run parameters
"""
)
# We expect a .json file with our parameters
controlsFileName = sys.argv[1]
err = ""
err += "" #validateFileArgument(controlsFileName, exists=True)
# read in the control parameters from the .json file
try:
controls = json.load(open(controlsFileName, "r"))
except:
err += "Could not process the file '" + controlsFileName + "'!\n"
# check each control parameter. The first one is optional
if "mpi-nodes" in controls:
mpi_nodes = controls["mpi-nodes"]
else:
mpi_nodes = controls["mpi-nodes"] = 1
if "cluster-size" in controls:
cluster_size = controls["cluster-size"]
else:
err += "Missing control definition for \"cluster-size\".\n"
if "initial-cutoff" in controls:
initial_cutoff = controls["initial-cutoff"]
else:
err += "Missing control definition for \"initial-cutoff\".\n"
# ...
# Quit if any of these things were not true
if len(err) > 0:
print err
exit()
#...
This works, but it seems like there must be a better way. I am stuck with the requirements to use a json file and to use the hyphenated parameter names. Any ideas?
I was looking for something with more static binding. Perhaps this is as good as it gets.
Usually, we do things like this.
def get_parameters( some_file_name ):
source= json.loads( some_file_name )
return dict(
mpi_nodes= source.get('mpi-nodes',1),
cluster_size= source['cluster-size'],
initial_cutoff = source['initial-cutoff'],
)
controlsFileName= sys.argv[1]
try:
params = get_parameters( controlsFileName )
except IOError:
print "Could not process the file '{0}'!".format( controlsFileName )
sys.exit( 1 )
except KeyError, e:
print "Missing control definition for '{0}'.".format( e.message )
sys.exit( 2 )
A the end params['mpi_nodes'] has the value of mpi_nodes
If you want a simple variable, you do this. mpi_nodes = params['mpi_nodes']
If you want a namedtuple, change get_parameters like this
def get_parameters( some_file_name ):
Parameters= namedtuple( 'Parameters', 'mpi_nodes, cluster_size, initial_cutoff' )
return Parameters( source.get('mpi-nodes',1),
source['cluster-size'],
source['initial-cutoff'],
)
I don't know if you'd find that better or not.
the argparse library is nice, it can handle most of the argument parsing and validation for you as well as printing pretty help screens
[1] http://docs.python.org/dev/library/argparse.html
I will knock up a quick demo showing how you'd want to use it this arvo.
Assuming you have many more parameters to process, something like this could work:
def underscore(s):
return s.replace('-','_')
# parameters with default values
for name, default in (("mpi-nodes", 1),):
globals()[underscore(name)] = controls.get(name, default)
# mandatory parameters
for name in ("cluster-size", "initial-cutoff"):
try:
globals()[underscore(name)] = controls[name]
except KeyError:
err += "Missing control definition for %r" % name
Instead of manipulating globals, you can also make this more explicit:
def underscore(s):
return s.replace('-','_')
settings = {}
# parameters with default values
for name, default in (("mpi-nodes", 1),):
settings[underscore(name)] = controls.get(name, default)
# mandatory parameters
for name in ("cluster-size", "initial-cutoff"):
try:
settings[underscore(name)] = controls[name]
except KeyError:
err += "Missing control definition for %r" % name
# print out err if necessary
mpi_nodes = settings['mpi_nodes']
cluster_size = settings['cluster_size']
initial_cutoff = settings['initial_cutoff']
I learned something from all of these responses - thanks! I would like to get feedback on my approach which incorporates something from each suggestion. In addition to the conditions imposed by the client, I want something:
1) that is fairly obvious to use and to debug
2) that is easy to maintain and modify
I decided to incorporate str.replace, namedtuple, and globals(), creating a ControlParameters namedtuple in the globals() namespace.
#!/usr/bin/env python
import sys
import os
import collections
import json
def get_parameters(parameters_file_name ):
"""
Access all of the control parameters from the json filename given. A
variable of type namedtuple named "ControlParameters" is injected
into the global namespace. Parameter validation is not performed. Both
the names and the defaults, if any, are defined herein. Parameters not
found in the json file will get values of None.
Parameter usage example: ControlParameters.cluster_size
"""
parameterValues = json.load(open(parameters_file_name, "r"))
Parameters = collections.namedtuple( 'Parameters',
"""
mpi_nodes
cluster_size
initial_cutoff
truncation_length
"""
)
parameters = Parameters(
parameterValues.get(Parameters._fields[0].replace('_', '-'), 1),
parameterValues.get(Parameters._fields[1].replace('_', '-')),
parameterValues.get(Parameters._fields[2].replace('_', '-')),
parameterValues.get(Parameters._fields[3].replace('_', '-'))
)
globals()["ControlParameters"] = parameters
#process the program argument(s)
err = ""
if len(sys.argv) != 2:
raise Exception(
"""Usage:
foo <control.json>
Where:
<control.json> is a dictionary of run parameters
"""
)
# We expect a .json file with our parameters
parameters_file_name = sys.argv[1]
err += "" #validateFileArgument(parameters_file_name, exists=True)
if err == "":
get_parameters(parameters_file_name)
cp_dict = ControlParameters._asdict()
for name in ControlParameters._fields:
if cp_dict[name] == None:
err += "Missing control parameter '%s'\r\n" % name
print err
print "Done"