I have a python program will will be called by another python program via a subprocess:
The python program is like this:
import argparse
import sys
from base64 import b64decode
import json
def set_key(args, **kwargs):
path = "/tmp/key"
print(kwargs)
try:
with open(path, 'wb') as open_file:
b = b64decode(kwargs)
open_file.write(b)
return Result(True)
except OSError as e:
return Result(False)
return Result(True)
commands_map = {
"set-key": set_key,
}
def main(sys_argv=None):
parser = argparse.ArgumentParser(
description="a util command"
)
parser.add_argument("-N", dest="check_testing", action="store_true",
help="check testing")
parser.add_argument("cmd", choices=sorted(list(commands_map)))
args, remaining = parser.parse_known_args(sys_argv)
# Not entirely sure there's much benefit to this check.
if args.check_testing:
return 1 if args.testing else 0
result, vals = commands_map[args.cmd](
remaining, testing=args.testing,
check_testing=args.check_testing)
print(json.dumps(vals, indent=2, sort_keys=True))
return 0 if result else 1
if __name__ == "__main__":
sys.exit(main())
And the caller program is:
import subprocess
input = b'{\n "keys": "l0Pu3TlknxWqTZwDG1yJcjUDGcBH7c8F19fkxeNmBl/2wXQoochlbxLTKhgkzeQNRDvFkQfMBcdlsbcxrrEQX+IydyiLkU5o8Gmhe2JGP56CNPLIefl9WPvLlPQBdvjEWO2UBaBjo2VW3Xsd1Ng+xFSUbP/ls7dso+h5/Ty37Rw="\n}'
cmda = ['python3', 'test.py', 'set-key']
try:
s = subprocess.run(cmda, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, input=input)
except subprocess.CalledProcessError as e:
print(e.stderr)
When I try to execute the call program, it complains the subprocess program kwargs is a dict.
b'Traceback (most recent call last):\n File "test.py", line 46, in <module>\n sys.exit(main())\n File "test.py", line 40, in main\n check_testing=args.check_testing)\n File "test.py", line 12, in set_key\n b = b64decode(kwargs)\n File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/base64.py", line 80, in b64decode\n s = _bytes_from_decode_data(s)\n File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/base64.py", line 46, in _bytes_from_decode_data\n "string, not %r" % s.__class__.__name__) from None\nTypeError: argument should be a bytes-like object or ASCII string, not \'dict\'\n'
How can I properly pass the input to the subprocess program so that I can write to a file?
Also, the args and kwargs seems to be empty (I expect it should be taken from the input call from subprocess). Anyone know why they are empty?
Related
Current Implementation which needs optimization
import subprocess
childprocess = subprocess.Popen(
['python',
'/full_path_to_directory/called_script.py',
'arg1',
'arg2'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
returnVal = childprocess.communicate()[0]
print(retVal)
Is this a correct way to call another script(called_script.py) within the current working directory?
Is there a better way to call the other script? I used import script but it gives me below error
called_script.py
def func(arg1, arg2, arg3):
#doSomething
#sys.out.write(returnVal)
if __name__ == "__main__":
func(arg1, arg2, arg3)
Implementation 2 (throws exception and errored out)
caller_script.py
Both of them are under the same path (i.e. /home/bin)
import called_script
returnVal = called_script.func(arg1,arg2,arg3)
print(returnVal)
Output:
nullNone
Traceback (most recent call last):
File "/path_to_caller/caller_script.py", line 89, in <module>
l.simple_bind_s(binddn, pw)
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 206, in simple_bind_s
msgid = self.simple_bind(who,cred,serverctrls,clientctrls)
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 200, in simple_bind
return self._ldap_call(self._l.simple_bind,who,cred,EncodeControlTuples(serverctrls),EncodeControlTuples(clientctrls))
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 96, in _ldap_call
result = func(*args,**kwargs)
TypeError: argument 2 must be string or read-only buffer, not None
Another alternative I used and gave me an error is
Implementation 3(throws exception and errors out)
caller_script.py
import ldap
returnVal = subprocess.call(['python','called_script.py','arg1','arg2'])
print(returnVal)
l = ldap.initialize(cp.get('some_config_ref','some_url'))
try:
l.protocol_version = ldap.VERSION3
l.simple_bind_s(binddn, returnVal)
except ldap.INVALID_CREDENTIALS:
sys.stderr.write("Your username or password is incorrect.")
sys.exit(1)
except ldap.LDAPError, e:
if type(e.message) == dict and e.message.has_key('xyz'):
sys.stderr.write(e.message['xyz'])
else:
sys.stderr.write(e)
sys.exit(1)
Output:
returnVal0Traceback (most recent call last):
File "./path_to_script/caller_script.py", line 88, in <module>
l.simple_bind_s(binddn, pw)
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 206, in simple_bind_s
msgid = self.simple_bind(who,cred,serverctrls,clientctrls)
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 200, in simple_bind
return self._ldap_call(self._l.simple_bind,who,cred,EncodeControlTuples(serverctrls),EncodeControlTuples(clientctrls))
File "/usr/lib64/python2.6/site-packages/ldap/ldapobject.py", line 96, in _ldap_call
result = func(*args,**kwargs)
TypeError: argument 2 must be string or read-only buffer, not int
Here is an example where you are calling a function from another file, you pass one value, a list, which can have an arbitrary amount of numbers, and you get the sum. Make sure they are in the same directory or you will need the path. The function in your example "script.py" does not allow you to pass a value.
called_script.py
def add_many(list_add):
the_sum = sum(list_add)
return the_sum
caller_script.py
import called_script
a_list = [1, 2, 3, 4]
the_sum = called_script.add_many(a_list)
print(the_sum)
I have looked at this question to get started and it works just fine How can I recover the return value of a function passed to multiprocessing.Process?
But in my case I would like to write a small tool, that would connect to many computers and gather some statistics, each stat would be gathered within a Process to make it snappy. But as soon as I try to wrap up the multiprocessing command in a class for a machine then it fails.
Here is my code
import multiprocessing
import pprint
def run_task(command):
p = subprocess.Popen(command, stdout = subprocess.PIPE, universal_newlines = True, shell = False)
result = p.communicate()[0]
return result
MACHINE_NAME = "cptr_name"
A_STAT = "some_stats_A"
B_STAT = "some_stats_B"
class MachineStatsGatherer():
def __init__(self, machineName):
self.machineName = machineName
manager = multiprocessing.Manager()
self.localStats = manager.dict() # creating a shared ressource for the sub processes to use
self.localStats[MACHINE_NAME] = machineName
def gatherStats(self):
self.runInParallel(
self.GatherSomeStatsA,
self.GatherSomeStatsB,
)
self.printStats()
def printStats(self):
pprint.pprint(self.localStats)
def runInParallel(self, *fns):
processes = []
for fn in fns:
process = multiprocessing.Process(target=fn, args=(self.localStats))
processes.append(process)
process.start()
for process in processes:
process.join()
def GatherSomeStatsA(self, returnStats):
# do some remote command, simplified here for the sake of debugging
result = "Windows"
returnStats[A_STAT] = result.find("Windows") != -1
def GatherSomeStatsB(self, returnStats):
# do some remote command, simplified here for the sake of debugging
result = "Windows"
returnStats[B_STAT] = result.find("Windows") != -1
def main():
machine = MachineStatsGatherer("SOMEMACHINENAME")
machine.gatherStats()
return
if __name__ == '__main__':
main()
And here is the error message
Traceback (most recent call last):
File "C:\Users\mesirard\AppData\Local\Programs\Python\Python37\lib\multiprocessing\process.py", line 297, in _bootstrap
self.run()
File "C:\Users\mesirard\AppData\Local\Programs\Python\Python37\lib\multiprocessing\process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "d:\workdir\trunks6\Tools\VTKAppTester\Utils\NXMachineMonitorShared.py", line 45, in GatherSomeStatsA
returnStats[A_STAT] = result.find("Windows") != -1
TypeError: 'str' object does not support item assignment
Process Process-3:
Traceback (most recent call last):
File "C:\Users\mesirard\AppData\Local\Programs\Python\Python37\lib\multiprocessing\process.py", line 297, in _bootstrap
self.run()
File "C:\Users\mesirard\AppData\Local\Programs\Python\Python37\lib\multiprocessing\process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "d:\workdir\trunks6\Tools\VTKAppTester\Utils\NXMachineMonitorShared.py", line 50, in GatherSomeStatsB
returnStats[B_STAT] = result.find("Windows") != -1
TypeError: 'str' object does not support item assignment
The issue is coming from this line
process = multiprocessing.Process(target=fn, args=(self.localStats))
it should have a extra comma at the end of args like so
process = multiprocessing.Process(target=fn, args=(self.localStats,))
Here is my code:
import multiprocessing
import dill
class Some_class():
class_var = 'Foo'
def __init__(self, param):
self.name = param
def print_name(self):
print("we are in object "+self.name)
print(Some_class.class_var)
def run_dill_encoded(what):
fun, args = dill.loads(what)
return fun(*args)
def apply_async(pool, fun, args):
return pool.apply_async(run_dill_encoded, (dill.dumps((fun, args)),))
if __name__ == '__main__':
list_names = [Some_class('object_1'), Some_class('object_2')]
pool = multiprocessing.Pool(processes=4)
results = [apply_async(pool, Some_class.print_name, args=(x,)) for x in list_names]
output = [p.get() for p in results]
print(output)
It returns error:
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Python34\lib\multiprocessing\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "C:\...\temp_obj_output_standard.py", line 18, in run_dill_encoded
return fun(*args)
File "C:/...temp_obj_output_standard.py", line 14, in print_name
print(Some_class.class_var)
NameError: name 'Some_class' is not defined
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/...temp_obj_output_standard.py", line 31, in <module>
output = [p.get() for p in results]
File "C:/...temp_obj_output_standard.py", line 31, in <listcomp>
output = [p.get() for p in results]
File "C:\Python34\lib\multiprocessing\pool.py", line 599, in get
raise self._value
NameError: name 'Some_class' is not defined
Process finished with exit code 1
The code works fine without line print(Some_class.class_var). What is wrong with accessing class variables, both objects should have it and I don't think processes should conflict about it. Am I missing something?
Any suggestions on how to troubleshoot it? Do not worry about run_dill_encoded and
apply_async, I am using this solution until I compile multiprocess on Python 3.x.
P.S. This is already enough, but stackoverflow wants me to put more details, not really sure what to put.
Can someone help me I'm trying to translate the code from the link below to work in windows.
http://wilson.bronger.org/lens_calibration_tutorial/calibrate.py
import subprocess, os, os.path, sys, multiprocessing, math, re, contextlib, glob, codecs, struct, numpy
from scipy.optimize.minpack import leastsq
candidate_groups = [['distortion\\Standard--13.3--5.5.DNG'],['distortion\\Standard--17.5--5.7.DNG']]
def call_exiv2(candidate_group):
exiv2_process = subprocess.Popen(
["exiv2", "-PEkt", "-g", "Exif.Photo.LensModel", "-g", "Exif.Photo.FocalLength", "-g", "Exif.Photo.FNumber"]
+ candidate_group, stdout=subprocess.PIPE)
lines = exiv2_process.communicate()[0].splitlines()
assert exiv2_process.returncode in [0, 253]
result = {}
for line in lines:
filename, data = line.split("Exif.Photo.")
filename = filename.rstrip()
if not filename:
assert len(candidate_group) == 1
filename = candidate_group[0]
fieldname, field_value = data.split(None, 1)
exif_data = result.setdefault(filename, [None, float("nan"), float("nan")])
if fieldname == "LensModel":
exif_data[0] = field_value
elif fieldname == "FocalLength":
exif_data[1] = float(field_value.partition("mm")[0])
elif fieldname == "FNumber":
exif_data[2] = float(field_value.partition("F")[2])
for filename, exif_data in list(result.copy().items()):
result[filename] = tuple(exif_data)
return result
if __name__ == '__main__':
multiprocessing.freeze_support()
pool = multiprocessing.Pool()
for group_exif_data in pool.map(call_exiv2, candidate_groups):
file_exif_data.update(group_exif_data)
pool.close()
pool.join()
However, there seems to be issues with the multiprocessing sections. I initially used 2to3 to convert the code to python3 as I'm using 3.4. I also added the code below above the pool = multiprocessing.Pool() calls.
if __name___ == '__main__':
multiprocessing.freeze_support()
Which got rid of one error but I still have the following:
Traceback (most recent call last):
File "calibrate.py", line 166, in <module>
for group_exif_data in pool.map(call_exiv2, candidate_groups):
File "C:\Python34\lib\multiprocessing\pool.py", line 260, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Python34\lib\multiprocessing\pool.py", line 599, in get
raise self._value
TypeError: Type str doesn't support the buffer API
I initially thought it might have to do with Pickling, but couldn't figure it out. Any ideas?
I am getting the following error when running the script below,can anyhelp to identify what the problem is and how to overcome it
import subprocess
import sys
import os
def main ():
to = ''
ssh_command = ["ssh", "-p", "29418", "review-android.quicinc.com", "gerrit",
"query", "--format=JSON", "--current-patch-set",
"--commit-message", "--files", ]
with open('gerrit_output.txt', 'a') as fp:
with open('caf_gerrits.txt','r') as f :
for gerrit in f :
print gerrit
result = subprocess.check_output(ssh_command + [gerrit, ])
print result
fp.write(result)
if __name__ == '__main__':
main()
ERROR:-
545804
Traceback (most recent call last):
File "test.py", line 20, in <module>
File "test.py", line 15, in main
File "/usr/lib/python2.7/subprocess.py", line 537, in check_output
process = Popen(stdout=PIPE, *popenargs, **kwargs)
File "/usr/lib/python2.7/subprocess.py", line 679, in __init__
errread, errwrite)
File "/usr/lib/python2.7/subprocess.py", line 1249, in _execute_child
raise child_exception
TypeError: execv() arg 2 must contain only strings
The third element in ssh_command is an integer. It needs to be a string.
e.g:
ssh_command = ["ssh", "-p", 29418, ...
# ^ problem
And the solution is simple, just add some quotes:
ssh_command = ["ssh", "-p", "29418", ...
# ^Now it's a string.
First you need to add the quotes around 29418 as mgilson mentioned. Second let's break down what you're trying to run:
ssh_command = ["ssh", "-p", 29418, "review-android.company.com", "gerrit",
"query", "--format", "JSON", "--current-patch-set",
"--commit-message", "--files", ]
That equals
ssh -p 29418 review-android.company.com gerrit query --format JSON --current-patch-set --commit-message --files
(then I'm assuming you have filenames in your caf_gerrits.txt file which get appended at the end)
One thing that pops out at me is that you may want to say --format=JSON in which case your elements in the ssh_command array should be combined as [..., "--format=JSON", ...]. The other thing you may want to do is print result after your result= line to help with the debugging.
check all the values that you pass to the subprocess command. One of them isn't a string.
I had the same error
TypeError: execv() arg 2 must contain only strings for
p1 = Popen(['aws', 'route53', 'change-resource-record-sets', '--hosted-zone-id', zone, '--change-batch', r53json], stdout=PIPE)
But as you can see it wasn't the 3rd parameter. The only variable was zone and I found that it was a list. Passing the parameter ass zone.id fixed the issue.
TypeError: execv() arg 2 must contain only strings has all the clue you need.