We Keep Coding

How do I pass arguments from command line to the python script to read and update values of certain keys present in a json file

I have certain data in a json file (say, example.json),
example.json
data = {
'name' : 'Williams',
'working': False,
'college': ['NYU','SU','OU'],
'NYU' : {
'student' : True,
'professor': False,
'years' : {
'fresher' : '1',
'sophomore': '2',
'final' : '3'
}
}
}
I wish to write a code wherein I can give the arguments on Command line, i.e. suppose if a script is saved in a file 'script.py', then,
In the terminal: If I enter *$ python3* script.py --get name --get NYU.student Then it outputs name=Williams
NYU.student=True
If I enter *$ python3* script.py' --set name=Tom --set NYU.student=False
Then, it updates name and NYU.student keys in the dictionay to Tom and False and outputs NYU.student=Tom and NYU.student=False on the command line.
I have tried the following code for the python script (i.e. script.py)
script.py
import json
import pprint
import argparse
if __name__== "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--get", help="first command")
parser.add_argument("--set", help="second command")
args=parser.parse_args()
with open('example.json','r') as read_file:
data=json.load(read_file)
if args.set == None:
key = ' '.join(args.get[:])
path = key.split('.')
now = data
for k in path:
if k in now:
now = now[k]
else:
print('Error: Invalid Key')
print(now)
elif args.get == Null:
key, value = ' '.join(args.set[:]).split('=')
path = key.split('.')
now = data
for k in path[:-1]:
if k in now:
now = now[k]
else:
print('Error: Invalid Key')
now[path[-1]] = value
with open('example.json','w') as write_file: #To write the updated data back to the same file
json.dump(data,write_file,indent=2)
However, my script is not working as I expect it to? Kindly, help me with the script

Your code has the following issues:
When joining the argument values in line number 23 and 35, you use a space. This leads to the "Error key" value. Removing the space will solve the issue.
key = ''.join(arg[:])
You defined the arguments to only pass one value. Not multiple. Therefore even if you pass multiple --get or --set values, the script only gets one value. Adding action="append" to line number 9 and 10 will solve the issue.
parser.add_argument("--get", help="first command", action="append")
parser.add_argument("--set", help="second command", action="append")
Full code:
import json
import pprint
import argparse
if __name__== "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--get", help="first command", action="append")
parser.add_argument("--set", help="second command", action="append")
args=parser.parse_args()
try:
with open('example.json','r') as read_file:
data=json.load(read_file)
except IOError:
print("ERROR: File not found")
exit()
if args.set == None:
for arg in args.get:
key = ''.join(arg[:])
path = key.split('.')
now = data
for k in path:
if k in now:
now = now[k]
else:
print('Error: Invalid Key')
print(f"{arg} = {now}")
elif args.get == None:
for arg in args.set:
key, value = ''.join(arg[:]).split('=')
path = key.split('.')
now = data
for k in path[:-1]:
if k in now:
now = now[k]
else:
print('Error: Invalid Key')
print(f"{arg}")
now[path[-1]] = value
with open('example.json','w') as write_file: #To write the updated data back to the same file
json.dump(data,write_file,indent=2)

here is the get part of the question, I hope that you can continue the set part of your assignment. good luck
python test.py --get name NYU.student
import json
import pprint
import argparse
def match(data: dict, filter: str):
current = data
for f in filter.split("."):
if f not in current:
return False
current = current[f]
return current == True
if __name__== "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--get", nargs="*", help="first command")
args = parser.parse_args()
with open('example.json','r') as f:
data = json.loads(f.read())
if args.get is not None and len(args.get) == 2:
attr_name = args.get[0]
if match(data, args.get[1]):
print("{}={}".format(attr_name, data[attr_name]))

In order to pass arguments using command line make use of sys module in python3. The sys module reads the command line arguments as a list of strings. The first element in the list is always the name of the file and subsequent elements are arg1, arg2 ..... so on.
Hope the following example helps to understand the usage of sys module.
Example Command :
python filename.py 1 thisisargument2 4
The corresponding code
import sys
# Note that all the command line args will be treated as strings
# Thus type casting will be needed for proper usage
print(sys.argv[0])
print(sys.argv[1])
print(sys.argv[2])
print(sys.argv[3])
Corresponding Output
filename.py
1
thisisargument2
4
Also please make a thorough google search before posting a question on stackoverflow.

Best practice for keeping a config file in python

I have a python script in which I have a config file which looks like this:
PROD = 'production'
DEV = 'dev'
ENVIRONMENT = None
and I have a function which gets the wanted environment from a command argument and sets it like:
if sys.argv[1] in [config.PROD, config.DEV]:
config.ENVIRONMENT = sys.argv[1]
I understood it's not good practice when I started importing the config file in multiple files and ENV kept resetting back to None.
So, my question is: what is the best practice is this case

I'm not sure exactly what the best practice is but I like using JSON files. I use the following class as a layer of abstraction for interfacing with the config (properties) file. You can create one JSONPropertiesFile and pass it around your application.
import json
from collections import OrderedDict
import os
from stat import * # ST_SIZE etc
from datetime import datetime
from copy import deepcopy
class JSONPropertiesFileError(Exception):
pass
class JSONPropertiesFile(object):
def __init__(self, file_path, default={}):
self.file_path = file_path
self._default_properties = default
self._validate_file_path(file_path)
def _validate_file_path(self, file_path):
if not file_path.endswith(".json"):
raise JSONPropertiesFileError(f"Must be a JSON file: {file_path}")
if not os.path.exists(file_path):
self.set(self._default_properties)
def set(self, properties):
new_properties = deepcopy(self._default_properties)
new_properties.update(properties)
with open(self.file_path, 'w') as file:
json.dump(new_properties, file, indent=4)
def get(self):
properties = deepcopy(self._default_properties)
with open(self.file_path) as file:
properties.update(json.load(file, object_pairs_hook=OrderedDict))
return properties
def get_file_info(self):
st = os.stat(self.file_path)
res = {
'size':st[ST_SIZE],
'size_str':str(round(st[ST_SIZE]/1000,2)) + ' KB',
'last_mod': datetime.fromtimestamp(st[ST_MTIME]).strftime("%Y-%m-%d")
}
return res
In your case you might use it like this:
file_path = "path/to/your/config/file"
default_properties = {
'PROD': 'production',
'DEV': 'dev',
'ENVIRONMENT': ""
}
config_file = JSONPropertiesFile(file_path, default_properties)
config = config_file.get()
print(config["PROD"])
config["PROD"] = "something else"
config_file.set(config) # save new config

Parsing arguments in python with any order

I am trying to parse command line arguments and I have never done it before. I have tried a few things. The code below is the last thing that I tried, but I received the error: "unrecognized arguments". I want to be able to put something like
copy-column-csv.py cname=Quiz source=source.csv target=target.csv out=out.csv
on the command line, but also have the ability to have it in any order. I am not sure how to go about this. Below is my code:
import argparse
import sys, re
import numpy as np
import smtplib
from random import randint
import csv
import math
import pandas as pd
parser = argparse.ArgumentParser()
parser.add_argument('-cname')
parser.add_argument('-source')
parser.add_argument('-target')
parser.add_argument('-out')
args = parser.parse_args()
#col = sys.argv[1]
#source = sys.argv[2]
#target = sys.argv[3]
#newtarg = sys.argv[4]
sourceFile = pd.read_csv(source)
targetFile = pd.read_csv(target)
del targetFile[cname]
targetFile[col] = pd.Series(sourceFile[col])
targetFile.to_csv(out, index = False)

Assuming well formed arguments, you could split sys.argv up into a dictionary:
args_dict = {}
for arg in sys.argv[1:]:
split = arg.split('=')
args_dict[split[0]] = split[1]
args_dict will look like this:
{'cname': 'Quiz',
'out': 'out.csv',
'source': 'source.csv',
'target': 'target.csv'}
And you can access the elements like so:
print args_dict['cname']
print args_dict['out']
print args_dict['source']
print args_dict['target']

Here is an example of how I'm doing it.
from optparse import OptionParser
from optparse import OptionGroup
parser = OptionParser(usage)
required = OptionGroup(parser, "Required Arguments")
required.add_option("--genome", dest="genome_file", help="File representing genome. FASTA nucleotide format. ")
required.add_option("--anno", dest="anno_file", help="File containing genome annotation information in GTF/GFF3 format. ")
required.add_option("--output", dest="prefix", help="Creates a folder named with the supplied prefix containing output files. ")
parser.add_option_group(required)
if len(args) != 0 or not options.genome_file or not options.anno_file or not options.prefix:
parser.error("Required arguments have not been supplied. \n\t\tUse -h to get more information.")
sys.exit()
You'll pass the arguments when you run it, for example, with --genome=myfile.txt, and in the code that value becomes options.genome_file.

Can I configure python to have matlab like print?

Can I configure python to have matlab like print, so that when I just have a function
returnObject()
that it simply prints that object without me having to type print around it? I assume this is not easy, but something like if an object does not get bound by some other var it should get printed, so that this would work.
a = 5 #prints nothing
b = getObject() #prints nothing
a #prints 5
b #prints getObject()
getObject() #prints the object

If you use an ipython notebook individual cells work like this. But you can only view one object per cell by typing the objects name. To see multiple objects you'd need to call print, or use lots of cells.

You could write a script to modify the original script based on a set of rules that define what to print, then run the modified script.
A basic script to do this would be:
f = open('main.py', 'r')
p = open('modified.py', 'w')
p.write('def main(): \n')
for line in f:
temp = line
if len(temp) == 1:
temp = 'print(' + line + ')'
p.write('\t' + temp)
p.close()
from modified import main
main()
The script main.py would then look like this:
x = 236
x
output:
236

Idea is as follows: parse AST of Python code, replace every expression with call to print and content of expression as argument and then run the modified version. I'm not sure whether it works with every code, but you might try. Save it as matlab.py and run your code as python3 -m matlab file.py.
#!/usr/bin/env python3
import ast
import os
import sys
class PrintAdder(ast.NodeTransformer):
def add_print(self, node):
print_func = ast.Name("print", ast.Load())
print_call = ast.Call(print_func, [node.value], [])
print_statement = ast.Expr(print_call)
return print_statement
def visit_Expr(self, node):
if isinstance(node.value, ast.Call) and node.value.func.id == 'print':
return node
return self.add_print(node)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-')
args = parser.parse_args()
with args.infile as infile:
code = infile.read()
file_name = args.infile.name
tree = ast.parse(code, file_name, 'exec')
tree = PrintAdder().visit(tree)
tree = ast.fix_missing_locations(tree)
bytecode = compile(tree, file_name, 'exec')
exec(bytecode)
if __name__ == '__main__':
main()

python global variable scope confusion. What are these variables not accessible?

I see there are a few of these questions on here, but I haven't found one that quit matches what I'm after.
I have a common file, lets call it tools.py. In this file I have a host of path definitions to use and an init_paths function to set some key paths based on command line arguments:
def init_paths(args):
global tools_dir, tools_src, tools_bin
if args.tools_set:
tools_dir = os.path.realpath(os.path.join(args.tools_set,"tools"))
else:
tools_dir = os.path.join(BAR_PATH, "tools")
FOO_PATH = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
BAR_PATH = os.path.join(FOO_PATH, "foobar")
tools_dir = none
tools_src = none
tools_bin = none
etc...
I have a main file, lets call it main.py where I want to use these.
if __name__ == "__main__":
args = parseArgs()
from tools import init_paths
init_paths(args)
doStuffFunction(args.one, args.two, args.three)
I have left out the meat and potatoes to be sure, but I believe this should be enough to illustrate my global scope problem. when I run this: python main.py --tools-set=/path/to/tools, I am expecting the call to init_paths, to set up some key paths I wish to use later in the doStuffFunction().
def doStuffFunction():
searchPath = os.path.join(tools_dir, "folder")
this fails: AttributeError: 'NoneType' object has no attribute endswith
pretty sure this is because it is not getting set. but why?
EDIT
main.py
#!/usr/bin/env python
import sys
import os
import argparse
import glob
from tools import *
def parseArgs():
parser = argparse.ArgumentParser(description="parse my args")
parser.add_argument("--toolchain-root", type=str,default=None,help='specify toolchain directory')
args = parser.parse_args()
return args
def doStuffFunction():
output = 'output'
if not os.path.isdir(output):
os.makedirs(output)
gimmySugar(output)
def gimmySugar(output):
fileList = []
linkBook= {}
searchPath = os.path.join(tools_BIN_ROOT,'gcc-4.8.5')
for root, dirs, files in os.walk(searchPath):
for libFile in glob.glob(root+'/*.so*'):
fileList.append(libFile)
if os.path.islink(libFile):
linksWith = os.readlink(libFile)
linkBook[libFile] = linksWith
if __name__ == "__main__":
# script was called directly from the command line
args = parseArgs()
from tools import init_settings
init_settings(args)
doStuffFunction()
tools.py
import os
def init_settings(args):
global tools_DIR, tools_SRC_ROOT, tools_OBJ_ROOT, tools_BIN_ROOT
if args.toolchain_root:
tools_DIR = os.path.realpath(os.path.join(args.toolchain_root, "toolchain"))
else:
tools_DIR = os.path.join(USER_DIR, "")
tools_SRC_ROOT = os.path.join(tools_DIR, "src")
tools_OBJ_ROOT = os.path.join(tools_DIR, "obj")
tools_BIN_ROOT = os.path.join(tools_DIR, "bin")
ROOT_PATH = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
OUTPUT_PATH = os.path.join(ROOT_PATH, "outputs")
BUILD_PATH = os.path.join(OUTPUT_PATH, "build")
USER_DIR = "/usr/lib64/"
tools_DIR = None
tools_SRC_ROOT = None
tools_OBJ_ROOT = None
tools_BIN_ROOT = None

#This line will failed
searchPath = os.path.join(tools_BIN_ROOT,'gcc-4.8.5')
The scope of global variables is module level, tools_BIN_ROOT will not be shared without passing across modules.
The variable tools_BIN_ROOT is global variable ONLY in tools.py. Insteadly, main.py do not contain any global variable in tools.py.
For inspecting this, you can use print(globals()) in both files.
Do not use global variable if possible.
This is a simple workaround.
(I strongly recommend you to refactor your code by config or OOP)
tools.py
def get_tools_BIN_ROOT():
return tools_BIN_ROOT
os.py
from tools import get_tools_BIN_ROOT
searchPath = os.path.join(get_tools_BIN_ROOT(),'gcc-4.8.5')