I have a nodejs app, and am using child_process to send data to be run in a python script. However, "sys.argv" only returns "script1.py", the file name, rather than the object I want to send to python.
Here is my nodejs:
app.get("/", (request, response) => {
console.log('hi')
var dataToSend;
var nodeData = "jeff";
console.log('breh')
var obj = {obj1: image3v, obj2: image4v}
const python = spawn('python', ['script1.py'], obj);
console.log('welp')
python.stdout.on('data', function (data) {
dataToSend = data.toString();
});
python.stderr.on('data', data => {
console.error(`stderr: ${data}`);
console.log(data)
})
python.on('exit', (code) => {
console.log('exited')
console.log(dataToSend);
//run new stuffhere
});
})
My script1.py:
import sys
print('python begins')
print(sys.argv[0])
print(sys.argv[0][1])
print(sys.argv[0][2])
print(sys.argv)
print(sys.orig_argv)
However, all of these print statements in python print either 'script1.py', ['script1.py'], or one of the letters in script1.py via the [0][1], etc. If I try to do sys.argv[1], I get index out of bounds.
Looking at other references, my code should be working, but I think there's something I'm not seeing. I don't know what else to try to access the object.
Thanks.
To spawn an external command and pass arguments on the commandline, the arguments need to be serialized, for example, to a string. There are limits to the size of commandline arguments, so if they are too large, some other method must be used (e.g., passing via stdin/stdout, named pipes, etc.).
Here's a demonstration of passing arguments from node to python.
spawnSO.js:
const { spawn } = require('node:child_process');
const params = {"fred": 1, "wilma": "wife", "dino": ["barks", 2.718]};
const python = spawn('python', ['spawned.py', JSON.stringify(params)]);
python.stdout.on('data', (data) => {
console.log(`stdout: ${data}`);
});
python.stderr.on('data', (data) => {
console.error(`stderr: ${data}`);
});
python.on('close', (code) => {
console.log(`child process exited with code ${code}`);
});
spawned.py:
import sys
print(sys.orig_argv)
print("len(sys.argv): ", len(sys.argv))
for k, arg in enumerate(sys.argv):
print(f"argv num: {k}, argv: {arg}")
import json
from_json_string = json.loads(sys.argv[1])
print(f"type of converted JSON: {type(from_json_string)}, loaded value: {from_json_string}")
Here's the output:
$ node spawnSO.js
stdout: ['python', 'spawned.py', '{"fred":1,"wilma":"wife","dino":["barks",2.718]}']
len(sys.argv): 2
argv num: 0, argv: spawned.py
argv num: 1, argv: {"fred":1,"wilma":"wife","dino":["barks",2.718]}
type of converted JSON: <class 'dict'>, loaded value: {'fred': 1, 'wilma': 'wife', 'dino': ['barks', 2.718]}
child process exited with code 0
When consuming a Hug REST endpoint from .net JSON has embedded characters. A complete failing example posted below. Any help greatly appreciated.
Python
#hug.post('/test')
def test(response, body=None):
input = body.get('input')
print('INSIDE TEST ' + input)
if input:
dict = {"lastname":"Jordan"}
dict["firstname"] = input
return json.dumps(dict, sort_keys=True, default=str)
.NET (can only use .net 3.5)
private static object GetParsedData(string data)
{
var posturl = "http://localhost:8000/test";
try
{
using (var client = new WebClient())
{
// upload values is the POST verb
var values = new NameValueCollection()
{
{ "input", data },
};
var response = client.UploadValues(posturl, values);
var responseString = Encoding.UTF8.GetString(response);
var settings = new JsonSerializerSettings
{
NullValueHandling = NullValueHandling.Ignore,
MissingMemberHandling = MissingMemberHandling.Ignore
};
JObject rss = JObject.Parse(responseString);
Console.WriteLine((string)rss["lastname"]);
}
}
catch (WebException ex)
{
if (ex.Response is HttpWebResponse)
{
var code = ((HttpWebResponse)ex.Response).StatusCode;
var desc = ((HttpWebResponse)ex.Response).StatusDescription;
}
//_logger.Error(ex.Message);
}
return false;
}
responseString looks like this:
"\"{\\\"firstname\\\": \\\"Mike\\\", \\\"lastname\\\": \\\"Jordan\\\"}\""
JObject.Parse throws error:
Newtonsoft.Json.JsonReaderException:
'Error reading JObject from JsonReader. Current JsonReader item is not an object: String. Path '', line 1, position 53.
Workaround - If I do something horrible like this to responseString JObject parses correctly:
str = str.Replace("\\", "");
str = str.Substring(1, len - 2);
Whats going on?
The default hug output format is json; it is not necessary to call json.dumps on return values, hug will do this automatically.
I am trying to source the /etc/environment file using golang.
I parsed each line of the file and used the following piece of code:
var myExp = regexp.MustCompile(`(?P<first>.*)=(?P<second>.*)`)
to get key=value from the file. But some values have = in it and the above regex fails.
For example, one of the lines in the environment would look like:
CONFIG_BASE64=SDFSWESC1= and I want it to be separated by the first occurence of =. i.e, Key isCONFIG_BASE64 and Value is SDFSWESC1=
What's wrong with strings.SplitN()? Going to regex seems overkill for this.
package main
import (
"bufio"
"fmt"
"os"
"strings"
)
func main() {
file, err := os.Open("/etc/environment")
if err != nil {
panic(err)
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
// MAGIC
split := strings.SplitN(scanner.Text(), "=", 2)
// Line didn't have an = in it
if len(split) < 2 {
continue
}
// Skip comments, pretty naive though
if strings.HasPrefix(split[0], "#") {
continue
}
fmt.Printf("key %s value %s\n", split[0], split[1])
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading standard input:", err)
}
}
In Python, you can do this:
"File {file} had error {error}".format(file=myfile, error=err)
or this:
"File %(file)s had error %(error)s" % {"file": myfile, "error": err}
In Go, the simplest option is:
fmt.Sprintf("File %s had error %s", myfile, err)
which doesn't let you swap the order of the parameters in the format string, which you need to do for I18N. Go does have the template package, which would require something like:
package main
import (
"bytes"
"text/template"
"os"
)
func main() {
type Params struct {
File string
Error string
}
var msg bytes.Buffer
params := &Params{
File: "abc",
Error: "def",
}
tmpl, _ := template.New("errmsg").Parse("File {{.File}} has error {{.Error}}")
tmpl.Execute(&msg, params)
msg.WriteTo(os.Stdout)
}
which seems like a long way to go for an error message. Is there a more reasonable option that allows me to give string parameters independent of order?
With strings.Replacer
Using strings.Replacer, implementing a formatter of your desire is very easy and compact.
func main() {
file, err := "/data/test.txt", "file not found"
log("File {file} had error {error}", "{file}", file, "{error}", err)
}
func log(format string, args ...string) {
r := strings.NewReplacer(args...)
fmt.Println(r.Replace(format))
}
Output (try it on the Go Playground):
File /data/test.txt had error file not found
We can make it more pleasant to use by adding the brackets to the parameter names automatically in the log() function:
func main() {
file, err := "/data/test.txt", "file not found"
log2("File {file} had error {error}", "file", file, "error", err)
}
func log2(format string, args ...string) {
for i, v := range args {
if i%2 == 0 {
args[i] = "{" + v + "}"
}
}
r := strings.NewReplacer(args...)
fmt.Println(r.Replace(format))
}
Output (try it on the Go Playground):
File /data/test.txt had error file not found
Yes, you could say that this only accepts string parameter values. This is true. With a little more improvement, this won't be true:
func main() {
file, err := "/data/test.txt", 666
log3("File {file} had error {error}", "file", file, "error", err)
}
func log3(format string, args ...interface{}) {
args2 := make([]string, len(args))
for i, v := range args {
if i%2 == 0 {
args2[i] = fmt.Sprintf("{%v}", v)
} else {
args2[i] = fmt.Sprint(v)
}
}
r := strings.NewReplacer(args2...)
fmt.Println(r.Replace(format))
}
Output (try it on the Go Playground):
File /data/test.txt had error 666
A variant of this to accept params as a map[string]interface{} and return the result as a string:
type P map[string]interface{}
func main() {
file, err := "/data/test.txt", 666
s := log33("File {file} had error {error}", P{"file": file, "error": err})
fmt.Println(s)
}
func log33(format string, p P) string {
args, i := make([]string, len(p)*2), 0
for k, v := range p {
args[i] = "{" + k + "}"
args[i+1] = fmt.Sprint(v)
i += 2
}
return strings.NewReplacer(args...).Replace(format)
}
Try it on the Go Playground.
With text/template
Your template solution or proposal is also way too verbose. It can be written as compact as this (error checks omitted):
type P map[string]interface{}
func main() {
file, err := "/data/test.txt", 666
log4("File {{.file}} has error {{.error}}", P{"file": file, "error": err})
}
func log4(format string, p P) {
t := template.Must(template.New("").Parse(format))
t.Execute(os.Stdout, p)
}
Output (try it on the Go Playground):
File /data/test.txt has error 666
If you want to return the string (instead of printing it to the standard output), you may do it like this (try it on the Go Playground):
func log5(format string, p P) string {
b := &bytes.Buffer{}
template.Must(template.New("").Parse(format)).Execute(b, p)
return b.String()
}
Using explicit argument indices
This was already mentioned in another answer, but to complete it, know that the same explicit argument index may be used arbitrary number of times and thus resulting in the same parameter substituted in multiple times. Read more about this in this question: Replace all variables in Sprintf with same variable
I don't know of any easy way of naming the parameters, but you can easily change the order of the arguments, using explicit argument indexes:
From docs:
In Printf, Sprintf, and Fprintf, the default behavior is for each formatting verb to format successive arguments passed in the call. However, the notation [n] immediately before the verb indicates that the nth one-indexed argument is to be formatted instead. The same notation before a '*' for a width or precision selects the argument index holding the value. After processing a bracketed expression [n], subsequent verbs will use arguments n+1, n+2, etc. unless otherwise directed.
Then you can, ie:
fmt.Printf("File %[2]s had error %[1]s", err, myfile)
The parameter can also be a map, so the following function would work if you don't mind parsing every error format every time you use it:
package main
import (
"bytes"
"text/template"
"fmt"
)
func msg(fmt string, args map[string]interface{}) (str string) {
var msg bytes.Buffer
tmpl, err := template.New("errmsg").Parse(fmt)
if err != nil {
return fmt
}
tmpl.Execute(&msg, args)
return msg.String()
}
func main() {
fmt.Printf(msg("File {{.File}} has error {{.Error}}\n", map[string]interface{} {
"File": "abc",
"Error": "def",
}))
}
It's still a little wordier than I would have liked, but it's better than some other options, I suppose. You could turn map[string]interface{} into a local type and reduce it further to:
type P map[string]interface{}
fmt.Printf(msg("File {{.File}} has error {{.Error}}\n", P{
"File": "abc",
"Error": "def",
}))
Alas, there's no built-in function in Go for string interpolation with named parameters (yet). But you are not the only one suffering out there :) Some packages should exist, for example: https://github.com/imkira/go-interpol . Or, if feeling adventurous, you could write such a helper yourself, as the concept is actually quite simple.
Cheers,
Dennis
You can try the Go Formatter library that implements replacement fields surrounded by curly braces {} format strings similar to Python format.
Working code example Go Playground:
package main
import (
"fmt"
"gitlab.com/tymonx/go-formatter/formatter"
)
func main() {
formatted, err := formatter.Format("Named placeholders {file}:{line}:{function}():", formatter.Named{
"line": 3,
"function": "func1",
"file": "dir/file",
})
if err != nil {
panic(err)
}
fmt.Println(formatted)
}
Output:
Named placeholders dir/file:3:func1():
Instead of using template.New, where you have to provide a template name, you
can just instantiate a template pointer:
package main
import (
"strings"
"text/template"
)
func format(s string, v interface{}) string {
t, b := new(template.Template), new(strings.Builder)
template.Must(t.Parse(s)).Execute(b, v)
return b.String()
}
func main() {
params := struct{File, Error string}{"abc", "def"}
println(format("File {{.File}} has error {{.Error}}", params))
}
Use os.Expand to replace fields in a format string. Expand replaces ${var} or $var in the string using a func(string) string mapping function.
Here are a couple of ways to wrap os.Expand in convenient to use functions:
func expandMap(s string, m map[string]string) string {
return os.Expand(s, func(k string) string { return m[k] })
}
func expandArgs(s string, kvs ...string) string {
return os.Expand(s, func(k string) string {
for i := 1; i < len(kvs); i++ {
if kvs[i-1] == k {
return kvs[i]
}
}
return ""
})
}
Example use:
s = expandMap("File ${file} had error ${error}",
map[string]string{"file": "myfile.txt", "error": "Not found"})
s = expandArgs("File ${file} had error ${error}",
"file", "myfile.txt", "error", "Not found"))
Run the code on the playground.
You can get quite close to that sweet python formatting experience:
message := FormatString("File {file} had error {error}", Items{"file"=myfile, "error"=err})
Declare the following somewhere in your code:
type Items map[string]interface{}
func FormatString(template string, items Items) string {
for key, value := range items {
template = strings.ReplaceAll(template, fmt.Sprintf("{%v}", key), fmt.Sprintf("%v", value))
}
return template
}
💡 note that my implementation is very naive and inefficient for high-performance needs
sudo make me a package
Seeing the development experience potential with having a simple signature like this, I've got tempted and uploaded a go package called format.
package main
import (
"fmt"
"github.com/jossef/format"
)
func main() {
formattedString := format.String(`hello "{name}". is lizard? {isLizard}`, format.Items{"name": "Mr Dude", "isLizard": false})
fmt.Println(formattedString)
}
https://repl.it/#jossef/format
text/template is interesting. I Provide some example below
Usage
func TestFString(t *testing.T) {
// Example 1
fs := &FString{}
fs.MustCompile(`Name: {{.Name}} Msg: {{.Msg}}`, nil)
fs.MustRender(map[string]interface{}{
"Name": "Carson",
"Msg": 123,
})
assert.Equal(t, "Name: Carson Msg: 123", fs.Data)
fs.Clear()
// Example 2 (with FuncMap)
funcMap := template.FuncMap{
"largest": func(slice []float32) float32 {
if len(slice) == 0 {
panic(errors.New("empty slice"))
}
max := slice[0]
for _, val := range slice[1:] {
if val > max {
max = val
}
}
return max
},
"sayHello": func() string {
return "Hello"
},
}
fs.MustCompile("{{- if gt .Age 80 -}} Old {{else}} Young {{- end -}}"+ // "-" is for remove empty space
"{{ sayHello }} {{largest .Numbers}}", // Use the function which you created.
funcMap)
fs.MustRender(Context{
"Age": 90,
"Numbers": []float32{3, 9, 13.9, 2.1, 7},
})
assert.Equal(t, "Old Hello 13.9", fs.Data)
}
Lib
package utils
import (
"text/template"
)
type Context map[string]interface{}
type FString struct {
Data string
template *template.Template
}
func (fs *FString) MustCompile(expr string, funcMap template.FuncMap) {
fs.template = template.Must(template.New("f-string").
Funcs(funcMap).
Parse(expr))
}
func (fs *FString) Write(b []byte) (n int, err error) {
fs.Data += string(b)
return len(b), nil
}
func (fs *FString) Render(context map[string]interface{}) error {
if err := fs.template.Execute(fs, context); err != nil {
return err
}
return nil
}
func (fs *FString) MustRender(context Context) {
if err := fs.Render(context); err != nil {
panic(err)
}
}
func (fs *FString) Clear() string {
// return the data and clear it
out := fs.Data
fs.Data = ""
return out
}
important document
https://golang.org/pkg/text/template/#hdr-Actions
Here is a function I wrote which replaces fields with strings in a map, similar to what you can do with Python. It takes a string which should have fields that look like ${field} and replaces them with any such keys in the given map like map['field']='value':
func replaceMap(s string,m *map[string]string) string {
r := regexp.MustCompile("\\${[^}]*}")
for x,i := range *m {
s = strings.Replace(s,"${"+x+"}",i,-1)
}
// Remove missing parameters
s = r.ReplaceAllString(s,"")
return s
}
Playground example:
https://go.dev/play/p/S5rF5KLooWq
Python has two very useful library method (binascii.a2b_hex(keyStr) and binascii.hexlify(keyBytes)) which I have been struggling with in Swift. Is there anything readily available in Swift. If not, how would one implement it? Given all the bounds and other checks (like even-length key) are done.
Data from Swift 3 has no "built-in" method to print its contents as
a hex string, or to create a Data value from a hex string.
"Data to hex string" methods can be found e.g. at How to convert Data to hex string in swift or How can I print the content of a variable of type Data using Swift? or converting String to Data in swift 3.0. Here is an implementation from the first link:
extension Data {
func hexEncodedString() -> String {
return map { String(format: "%02hhx", $0) }.joined()
}
}
Here is a possible implementation of the reverse "hex string to Data"
conversion (taken from Hex String to Bytes (NSData) on Code Review, translated to Swift 3 and improved)
as a failable inititializer:
extension Data {
init?(fromHexEncodedString string: String) {
// Convert 0 ... 9, a ... f, A ...F to their decimal value,
// return nil for all other input characters
func decodeNibble(u: UInt8) -> UInt8? {
switch(u) {
case 0x30 ... 0x39:
return u - 0x30
case 0x41 ... 0x46:
return u - 0x41 + 10
case 0x61 ... 0x66:
return u - 0x61 + 10
default:
return nil
}
}
self.init(capacity: string.utf8.count/2)
var iter = string.utf8.makeIterator()
while let c1 = iter.next() {
guard
let val1 = decodeNibble(u: c1),
let c2 = iter.next(),
let val2 = decodeNibble(u: c2)
else { return nil }
self.append(val1 << 4 + val2)
}
}
}
Example:
// Hex string to Data:
if let data = Data(fromHexEncodedString: "0002468A13579BFF") {
let idata = Data(data.map { 255 - $0 })
// Data to hex string:
print(idata.hexEncodedString()) // fffdb975eca86400
} else {
print("invalid hex string")
}
Not really familiar with Python and the checks it performs when convert the numbers, but you can expand the function below:
func convert(_ str: String, fromRadix r1: Int, toRadix r2: Int) -> String? {
if let num = Int(str, radix: r1) {
return String(num, radix: r2)
} else {
return nil
}
}
convert("11111111", fromRadix: 2, toRadix: 16)
convert("ff", fromRadix: 16, toRadix: 2)
Swift 2
extension NSData {
class func dataFromHexString(hex: String) -> NSData? {
let regex = try! NSRegularExpression(pattern: "^[0-9a-zA-Z]*$", options: .CaseInsensitive)
let validate = regex.firstMatchInString(hex, options: NSMatchingOptions.init(rawValue: 0), range: NSRange(location: 0, length: hex.characters.count))
if validate == nil || hex.characters.count % 2 != 0 {
return nil
}
let data = NSMutableData()
for i in 0..<hex.characters.count/2 {
let hexStr = hex.substring(i * 2, length: 2)
var ch: UInt32 = 0
NSScanner(string: hexStr).scanHexInt(&ch)
data.appendBytes(&ch, length: 1)
}
return data
}
}
let a = 0xabcd1234
print(String(format: "%x", a)) // Hex to String
NSData.dataFromHexString("abcd1234") // String to hex