I am a Python re-newbie. I would like advice on handling program parameters which are in a file in json format. Currently, I am doing something like what is shown below, however, it seems too wordy, and the idea of typing the same literal string multiple times (sometimes with dashes and sometimes with underscores) seems juvenile - error prone - stinky... :-) (I do have many more parameters!)
#!/usr/bin/env python
import sys
import os
import json ## for control file parsing
# control parameters
mpi_nodes = 1
cluster_size = None
initial_cutoff = None
# ...
#process the arguments
if len(sys.argv) != 2:
raise Exception(
"""Usage:
run_foo <controls.json>
Where:
<control.json> is a dictionary of run parameters
"""
)
# We expect a .json file with our parameters
controlsFileName = sys.argv[1]
err = ""
err += "" #validateFileArgument(controlsFileName, exists=True)
# read in the control parameters from the .json file
try:
controls = json.load(open(controlsFileName, "r"))
except:
err += "Could not process the file '" + controlsFileName + "'!\n"
# check each control parameter. The first one is optional
if "mpi-nodes" in controls:
mpi_nodes = controls["mpi-nodes"]
else:
mpi_nodes = controls["mpi-nodes"] = 1
if "cluster-size" in controls:
cluster_size = controls["cluster-size"]
else:
err += "Missing control definition for \"cluster-size\".\n"
if "initial-cutoff" in controls:
initial_cutoff = controls["initial-cutoff"]
else:
err += "Missing control definition for \"initial-cutoff\".\n"
# ...
# Quit if any of these things were not true
if len(err) > 0:
print err
exit()
#...
This works, but it seems like there must be a better way. I am stuck with the requirements to use a json file and to use the hyphenated parameter names. Any ideas?
I was looking for something with more static binding. Perhaps this is as good as it gets.
Usually, we do things like this.
def get_parameters( some_file_name ):
source= json.loads( some_file_name )
return dict(
mpi_nodes= source.get('mpi-nodes',1),
cluster_size= source['cluster-size'],
initial_cutoff = source['initial-cutoff'],
)
controlsFileName= sys.argv[1]
try:
params = get_parameters( controlsFileName )
except IOError:
print "Could not process the file '{0}'!".format( controlsFileName )
sys.exit( 1 )
except KeyError, e:
print "Missing control definition for '{0}'.".format( e.message )
sys.exit( 2 )
A the end params['mpi_nodes'] has the value of mpi_nodes
If you want a simple variable, you do this. mpi_nodes = params['mpi_nodes']
If you want a namedtuple, change get_parameters like this
def get_parameters( some_file_name ):
Parameters= namedtuple( 'Parameters', 'mpi_nodes, cluster_size, initial_cutoff' )
return Parameters( source.get('mpi-nodes',1),
source['cluster-size'],
source['initial-cutoff'],
)
I don't know if you'd find that better or not.
the argparse library is nice, it can handle most of the argument parsing and validation for you as well as printing pretty help screens
[1] http://docs.python.org/dev/library/argparse.html
I will knock up a quick demo showing how you'd want to use it this arvo.
Assuming you have many more parameters to process, something like this could work:
def underscore(s):
return s.replace('-','_')
# parameters with default values
for name, default in (("mpi-nodes", 1),):
globals()[underscore(name)] = controls.get(name, default)
# mandatory parameters
for name in ("cluster-size", "initial-cutoff"):
try:
globals()[underscore(name)] = controls[name]
except KeyError:
err += "Missing control definition for %r" % name
Instead of manipulating globals, you can also make this more explicit:
def underscore(s):
return s.replace('-','_')
settings = {}
# parameters with default values
for name, default in (("mpi-nodes", 1),):
settings[underscore(name)] = controls.get(name, default)
# mandatory parameters
for name in ("cluster-size", "initial-cutoff"):
try:
settings[underscore(name)] = controls[name]
except KeyError:
err += "Missing control definition for %r" % name
# print out err if necessary
mpi_nodes = settings['mpi_nodes']
cluster_size = settings['cluster_size']
initial_cutoff = settings['initial_cutoff']
I learned something from all of these responses - thanks! I would like to get feedback on my approach which incorporates something from each suggestion. In addition to the conditions imposed by the client, I want something:
1) that is fairly obvious to use and to debug
2) that is easy to maintain and modify
I decided to incorporate str.replace, namedtuple, and globals(), creating a ControlParameters namedtuple in the globals() namespace.
#!/usr/bin/env python
import sys
import os
import collections
import json
def get_parameters(parameters_file_name ):
"""
Access all of the control parameters from the json filename given. A
variable of type namedtuple named "ControlParameters" is injected
into the global namespace. Parameter validation is not performed. Both
the names and the defaults, if any, are defined herein. Parameters not
found in the json file will get values of None.
Parameter usage example: ControlParameters.cluster_size
"""
parameterValues = json.load(open(parameters_file_name, "r"))
Parameters = collections.namedtuple( 'Parameters',
"""
mpi_nodes
cluster_size
initial_cutoff
truncation_length
"""
)
parameters = Parameters(
parameterValues.get(Parameters._fields[0].replace('_', '-'), 1),
parameterValues.get(Parameters._fields[1].replace('_', '-')),
parameterValues.get(Parameters._fields[2].replace('_', '-')),
parameterValues.get(Parameters._fields[3].replace('_', '-'))
)
globals()["ControlParameters"] = parameters
#process the program argument(s)
err = ""
if len(sys.argv) != 2:
raise Exception(
"""Usage:
foo <control.json>
Where:
<control.json> is a dictionary of run parameters
"""
)
# We expect a .json file with our parameters
parameters_file_name = sys.argv[1]
err += "" #validateFileArgument(parameters_file_name, exists=True)
if err == "":
get_parameters(parameters_file_name)
cp_dict = ControlParameters._asdict()
for name in ControlParameters._fields:
if cp_dict[name] == None:
err += "Missing control parameter '%s'\r\n" % name
print err
print "Done"
Related
I am trying to modify on the fly some of the parameters used by the exit function of a context manager. I am trying to bind the parameter to a variable known in the with block
from contextlib import contextmanager
import tempfile, shutil
#contextmanager
def tempdir(suffix = '', prefix = '', dir = None, ignore_errors = False,
remove = True):
"""
Context manager to generate a temporary directory with write permissions.
"""
d = tempfile.mkdtemp(suffix, prefix, dir)
try:
yield d
finally:
print "finalizing tempdir %s, remove= %s" %(d,remove)
if remove:
shutil.rmtree(d, ignore_errors)
willremove = True
with tempdir(remove = willremove) as t:
#attempt to modify parameter
willremove = False
print "willremove:%s" %willremove
pass
I would expect that changing the value of willremove would change the remove variable in the finally: part of the contextmanager function, but it doesn't help
This cannot be done because the parameters in python are passed 'by assignment', as pointed by Ned Batchelder in the following talk:
https://www.youtube.com/watch?v=_AEJHKGk9ns
I have a script that takes some data in an excel file and sorts it and adds some colors to some key things....
I'm using an external .ini file because it needs to change sometimes based on the users needs for the day
the ini file basically looks like this
[section]
#Color 1
color01 = ('00FCC84E')
cat1 = ('Item1','Item2')
#Color 2
color02 = ('00F4426E')
cat2 = ('Thingy Size 5/16')
My Script portion that uses Config Parser does this
import configparser
from configparser import ConfigParser
from ast import literal_eval
config = ConfigParser()
config.read("MyFile.ini")
config.sections()
def variables(section):
dict1 = {}
options = config.options(section)
for option in options:
try:
dict1[option] = config.get(section, option)
if dict1[option] == -1:
DebugPrint("skip: %s" % option)
except:
print("exception on %s!" % option)
dict1[option] = None
return dict1
color01V = literal_eval(config['ConfigFile']['color01'])
color02V = literal_eval(config['ConfigFile']['color02'])
cat01V = literal_eval(config['ConfigFile']['cat1'])
cat02V = literal_eval(config['ConfigFile']['cat2'])
print(cat01V)
print(cat02V)
this returns
('Item1','Item2')
Thingy Size 5/16
Why does the 2nd string return without the () and how do I fix it.
I actually NEED the () to appear when I use the variable later
Have you tried putting value of "cat1" in quotations?
[section]
color01 = ('00FCC84E')
cat1 = "('Item1','Item2')"
...
cat2 = "('Thingy Size 5/16')"
I created EXE file with Python (PySide) + PyInstaller. Once I try to use
print QtGui.QApplication.applicationVersion()
I don't see valid version in x.x.x.x format of the application.
Are there any built-in functions in PySide instead of this, or maybe should I use other library for it?
PS. I don't believe that Python doesn't have any methods to extract information about EXE :)
Have a look here.
There's everything you're looking for !
Hope this helps.
Found one way to do it:
import win32api
def get_version_info():
try:
filename = APP_FILENAME
return get_file_properties(filename)['FileVersion']
except BaseException, err:
log_error(err.message)
return '0.0.0.0'
def get_file_properties(filename):
"""
Read all properties of the given file return them as a dictionary.
"""
propNames = ('Comments', 'InternalName', 'ProductName',
'CompanyName', 'LegalCopyright', 'ProductVersion',
'FileDescription', 'LegalTrademarks', 'PrivateBuild',
'FileVersion', 'OriginalFilename', 'SpecialBuild')
props = {'FixedFileInfo': None, 'StringFileInfo': None, 'FileVersion': '0.0.0.0'}
try:
# backslash as parm returns dictionary of numeric info corresponding to VS_FIXEDFILEINFO struc
print filename
fixedInfo = win32api.GetFileVersionInfo(filename, '\\')
props['FixedFileInfo'] = fixedInfo
props['FileVersion'] = "%d.%d.%d.%d" % (fixedInfo['FileVersionMS'] / 65536,
fixedInfo['FileVersionMS'] % 65536, fixedInfo['FileVersionLS'] / 65536,
fixedInfo['FileVersionLS'] % 65536)
# \VarFileInfo\Translation returns list of available (language, codepage)
# pairs that can be used to retreive string info. We are using only the first pair.
lang, codepage = win32api.GetFileVersionInfo(filename, '\\VarFileInfo\\Translation')[0]
# any other must be of the form \StringfileInfo\%04X%04X\parm_name, middle
# two are language/codepage pair returned from above
strInfo = {}
for propName in propNames:
strInfoPath = u'\\StringFileInfo\\%04X%04X\\%s' % (lang, codepage, propName)
## print str_info
strInfo[propName] = win32api.GetFileVersionInfo(filename, strInfoPath)
props['StringFileInfo'] = strInfo
except BaseException, err:
log_error(err.message)
return props
I have the following code which attempts to get the DUT VID from the invoked command line:
parser = argparse.ArgumentParser(description='A Test',
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
group.add_argument("--vid",
type=int,
help="vid of DUT")
options = parser.parse_args()
Consider the command line "python test.py --vid 0xabcd"
I notice that argparse is raising an exception on this as it fails to complete the call int('0xabcd') because it is base 16. How do I get argparse to correctly handle this?
argparse is looking to create a callable type conversion from the 'type' value:
def _get_value(self, action, arg_string):
type_func = self._registry_get('type', action.type, action.type)
if not _callable(type_func):
msg = _('%r is not callable')
raise ArgumentError(action, msg % type_func)
# convert the value to the appropriate type
try:
result = type_func(arg_string)
# ArgumentTypeErrors indicate errors
except ArgumentTypeError:
name = getattr(action.type, '__name__', repr(action.type))
msg = str(_sys.exc_info()[1])
raise ArgumentError(action, msg)
# TypeErrors or ValueErrors also indicate errors
except (TypeError, ValueError):
name = getattr(action.type, '__name__', repr(action.type))
msg = _('invalid %s value: %r')
raise ArgumentError(action, msg % (name, arg_string))
# return the converted value
return result
By default int() is set to base 10. In order to accommodate base 16 and base 10 parameters we can enable auto base detection:
def auto_int(x):
return int(x, 0)
...
group.add_argument('--vid',
type=auto_int,
help='vid of DUT')
Note the type is update to be 'auto_int'.
Not enough reputation to comment on the other answer.
If you don't want to define the auto_int function, I found that this works very cleanly with a lambda.
group.add_argument('--vid',
type=lambda x: int(x,0),
help='vid of DUT')
Is there a similar mechanism in Python, to the effect set -x has on bash?
Here's some example output from bash in this mode:
+ for src in cpfs.c log.c popcnt.c ssse3_popcount.c blkcache.c context.c types.c device.c
++ my_mktemp blkcache.c.o
+++ mktemp -t blkcache.c.o.2160.XXX
++ p=/tmp/blkcache.c.o.2160.IKA
++ test 0 -eq 0
++ echo /tmp/blkcache.c.o.2160.IKA
+ obj=/tmp/blkcache.c.o.2160.IKA
I'm aware of the Python trace module, however its output seems to be extremely verbose, and not high level like that of bash.
Perhaps use sys.settrace:
Use traceit() to turn on tracing, use traceit(False) to turn off tracing.
import sys
import linecache
def _traceit(frame, event, arg):
'''
http://www.dalkescientific.com/writings/diary/archive/2005/04/20/tracing_python_code.html
'''
if event == "line":
lineno = frame.f_lineno
filename = frame.f_globals["__file__"]
if (filename.endswith(".pyc") or
filename.endswith(".pyo")):
filename = filename[:-1]
name = frame.f_globals["__name__"]
line = linecache.getline(filename, lineno)
print "%s # %s:%s" % (line.rstrip(), name, lineno,)
return _traceit
def _passit(frame, event, arg):
return _passit
def traceit(on=True):
if on: sys.settrace(_traceit)
else: sys.settrace(_passit)
def mktemp(src):
pass
def my_mktemp(src):
mktemp(src)
p=src
traceit()
for src in ('cpfs.c','log.c',):
my_mktemp(src)
traceit(False)
yields
mktemp(src) # __main__:33
pass # __main__:30
p=src # __main__:34
mktemp(src) # __main__:33
pass # __main__:30
p=src # __main__:34
if on: sys.settrace(_traceit) # __main__:26
else: sys.settrace(_passit) # __main__:27
To trace specific calls, you can wrap each interesting function with your own logger. This does lead to arguments expanded to their values rather than just argument names in the output.
Functions have to be passed in as strings to prevent issues where modules redirect to other modules, like os.path / posixpath. I don't think you can extract the right module name to patch from just the function object.
Wrapping code:
import importlib
def wrapper(ffull, f):
def logger(*args, **kwargs):
print "TRACE: %s (%s, %s)" % (ffull, args, kwargs)
return f(*args, **kwargs)
return logger
def log_execution(ffull):
parts = ffull.split('.')
mname = '.'.join(parts[:-1])
fname = parts[-1]
m = importlib.import_module(mname)
f = getattr(m, fname)
setattr(m, fname, wrapper(ffull, f))
Usage:
for f in ['os.path.join', 'os.listdir', 'sys.exit']:
log_execution(f)
p = os.path.join('/usr', 'bin')
os.listdir(p)
sys.exit(0)
....
% ./a.py
TRACE: os.path.join (('/usr', 'bin'), {})
TRACE: os.listdir (('/usr/bin',), {})
TRACE: sys.exit ((0,), {})
You should try to instrument the trace module to get an higher detail level.
What do you need exactly?