I have this piece of code. It should call my python script if it is true.
The problem is: the script doesn't get executed. I'm using Heroku and I already have nodejs and python in my buildpacks.
const express = require('express');
const bodyParser = require('body-parser');
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
let {PythonShell} = require('python-shell');
app.post('/api/apipath', (req, res) => {
if (currentPhase == phaseId){
PythonShell.run('./src/main.py', null, function (err){
if (err) throw err;
console.log('Finish');
});
res.end();
}else{
res.end();
}
}
Note: I already forced the condition to be true and it didn't work. The script was not called
I'm not familiar with python-shell but you can use exec from Node's child_process to run your script
const { exec } = require('child_process')
function runPythonScript(script, runFile = false) {
console.log(`Run? ${runFile}`)
if (runFile) {
exec(`python ${script}`, (error, stdout, stderr) => {
if (error) {
console.error(`exec error: ${error}`)
return
}
console.log(`stdout: ${stdout}`)
console.error(`stderr: ${stderr}`)
})
}
}
Running the code:
// this will run
runPythonScript('./src/main.py', 1 === 1)
// this will not run
runPythonScript('./src/main.py', 0)
// this will run since both currentPhase and phaseId are empty variables
var currentPhase, phaseId
runPythonScript('./src/main.py', currentPhase === phaseId)
CLI
add this code for if you want to define currentPhase and phaseId from the command line
function init() {
const [, , ...args] = process.argv
var currentPhase, phaseId
currentPhase = args[args.indexOf('--currentPhase') - 1]
phaseId = args[args.indexOf('--phaseId') - 1]
runPythonScript('./program.py', currentPhase === phaseId)
}
init()
from the CLI, you can run
# sets currentPhase to 99 and phaseId as 98 (does not run)
node lib/filename-to-runPythonScript.js 99 --currentPhase 98 --phaseId
# sets both currentPhase and phaseId to 123 (does run)
node lib/filename-to-runPythonScript.js 123 --currentPhase 123 --phaseId
AWS CloudWatch Events to trigger a Lambda function that polls for messages on a queue. Messages are then sent to a second Lambda function that writes the message to an Kinesis agent.
I found a solution to this article but written in node.js. But I am looking for Python script. Could you please help me to get a article where I can find a way to convert it by myself.
https://github.com/aws-samples/amazonmq-invoke-aws-lambda
const stomp = require('stompit')
const AWS = require('aws-sdk')
let lambda = new AWS.Lambda()
const QUEUE_NAME = process.env.QUEUE_NAME
const CONNECTION_OPTIONS = {
host: process.env.HOST,
port: 61614,
ssl: true,
connectHeaders: {
login: process.env.LOGIN,
passcode: process.env.PASSWORD
}
}
exports.handler = (event, context, callback) => {
var client
/* */
const onError = (error) => {
console.error(`[ERROR] ${error}`)
callback(error)
}
/* */
const onMessage = (error, message) => {
console.log('Message received')
message.readString('utf-8', (error, body) => {
if (error) {
onError(error)
return
}
let payload = {
message: body,
timestamp: Date.now()
}
console.log(payload)
let params = {
FunctionName: process.env.WORKER_FUNCTION,
Payload: JSON.stringify(payload)
}
lambda.invoke(params, (error, data) => {
if (error) {
console.error(`Could not invoke Lambda: ${error}`)
}
})
})
}
/* Main */
stomp.connect(CONNECTION_OPTIONS, (error, client) => {
if (error) {
onError(error)
return
}
var headers = {
destination: `/queue/${QUEUE_NAME}`,
ack: 'auto'
}
client.subscribe(headers, onMessage)
setTimeout(() => {
client.disconnect()
callback(null, { 'message': 'Finished' })
}, context.getRemainingTimeInMillis() - 1000)
})
}
You don't make it clear what you are trying to achieve, how far you got to, and what issue you are facing.
You can improve your question by following the the SO guidelines.
The library to interact with AWS services in python(3) is boto3.
I have an Express Node.js application, but I also have a machine learning algorithm to use in Python. Is there a way I can call Python functions from my Node.js application to make use of the power of machine learning libraries?
Easiest way I know of is to use "child_process" package which comes packaged with node.
Then you can do something like:
const spawn = require("child_process").spawn;
const pythonProcess = spawn('python',["path/to/script.py", arg1, arg2, ...]);
Then all you have to do is make sure that you import sys in your python script, and then you can access arg1 using sys.argv[1], arg2 using sys.argv[2], and so on.
To send data back to node just do the following in the python script:
print(dataToSendBack)
sys.stdout.flush()
And then node can listen for data using:
pythonProcess.stdout.on('data', (data) => {
// Do something with the data returned from python script
});
Since this allows multiple arguments to be passed to a script using spawn, you can restructure a python script so that one of the arguments decides which function to call, and the other argument gets passed to that function, etc.
Hope this was clear. Let me know if something needs clarification.
Example for people who are from Python background and want to integrate their machine learning model in the Node.js application:
It uses the child_process core module:
const express = require('express')
const app = express()
app.get('/', (req, res) => {
const { spawn } = require('child_process');
const pyProg = spawn('python', ['./../pypy.py']);
pyProg.stdout.on('data', function(data) {
console.log(data.toString());
res.write(data);
res.end('end');
});
})
app.listen(4000, () => console.log('Application listening on port 4000!'))
It doesn't require sys module in your Python script.
Below is a more modular way of performing the task using Promise:
const express = require('express')
const app = express()
let runPy = new Promise(function(success, nosuccess) {
const { spawn } = require('child_process');
const pyprog = spawn('python', ['./../pypy.py']);
pyprog.stdout.on('data', function(data) {
success(data);
});
pyprog.stderr.on('data', (data) => {
nosuccess(data);
});
});
app.get('/', (req, res) => {
res.write('welcome\n');
runPy.then(function(fromRunpy) {
console.log(fromRunpy.toString());
res.end(fromRunpy);
});
})
app.listen(4000, () => console.log('Application listening on port 4000!'))
The python-shell module by extrabacon is a simple way to run Python scripts from Node.js with basic, but efficient inter-process communication and better error handling.
Installation:
With npm:
npm install python-shell.
Or with yarn:
yarn add python-shell
Running a simple Python script:
const PythonShell = require('python-shell').PythonShell;
PythonShell.run('my_script.py', null, function (err) {
if (err) throw err;
console.log('finished');
});
Running a Python script with arguments and options:
const PythonShell = require('python-shell').PythonShell;
var options = {
mode: 'text',
pythonPath: 'path/to/python',
pythonOptions: ['-u'],
scriptPath: 'path/to/my/scripts',
args: ['value1', 'value2', 'value3']
};
PythonShell.run('my_script.py', options, function (err, results) {
if (err)
throw err;
// Results is an array consisting of messages collected during execution
console.log('results: %j', results);
});
For the full documentation and source code, check out https://github.com/extrabacon/python-shell
You can now use RPC libraries that support Python and Javascript such as zerorpc
From their front page:
Node.js Client
var zerorpc = require("zerorpc");
var client = new zerorpc.Client();
client.connect("tcp://127.0.0.1:4242");
client.invoke("hello", "RPC", function(error, res, more) {
console.log(res);
});
Python Server
import zerorpc
class HelloRPC(object):
def hello(self, name):
return "Hello, %s" % name
s = zerorpc.Server(HelloRPC())
s.bind("tcp://0.0.0.0:4242")
s.run()
Many of the examples are years out of date and involve complex setup. You can give JSPyBridge/pythonia a try (full disclosure: I'm the author). It's vanilla JS that lets you operate on foreign Python objects as if they existed in JS. In fact, it does interoperability so Python code can in return call JS through callbacks and passed functions.
numpy + matplotlib example, with the ES6 import system:
import { py, python } from 'pythonia'
const np = await python('numpy')
const plot = await python('matplotlib.pyplot')
// Fixing random state for reproducibility
await np.random.seed(19680801)
const [mu, sigma] = [100, 15]
// Inline expression evaluation for operator overloading
const x = await py`${mu} + ${sigma} * ${np.random.randn(10000)}`
// the histogram of the data
const [n, bins, patches] = await plot.hist$(x, 50, { density: true, facecolor: 'g', alpha: 0.75 })
console.log('Distribution', await n) // Always await for all Python access
await plot.show()
python.exit()
Through CommonJS (without top level await):
const { py, python } = require('pythonia')
async function main() {
const np = await python('numpy')
const plot = await python('matplotlib.pyplot')
...
// the rest of the code
}
main().then(() => python.exit()) // If you don't call this, the process won't quit by itself.
Most of previous answers call the success of the promise in the on("data"), it is not the proper way to do it because if you receive a lot of data you will only get the first part. Instead you have to do it on the end event.
const { spawn } = require('child_process');
const pythonDir = (__dirname + "/../pythonCode/"); // Path of python script folder
const python = pythonDir + "pythonEnv/bin/python"; // Path of the Python interpreter
/** remove warning that you don't care about */
function cleanWarning(error) {
return error.replace(/Detector is not able to detect the language reliably.\n/g,"");
}
function callPython(scriptName, args) {
return new Promise(function(success, reject) {
const script = pythonDir + scriptName;
const pyArgs = [script, JSON.stringify(args) ]
const pyprog = spawn(python, pyArgs );
let result = "";
let resultError = "";
pyprog.stdout.on('data', function(data) {
result += data.toString();
});
pyprog.stderr.on('data', (data) => {
resultError += cleanWarning(data.toString());
});
pyprog.stdout.on("end", function(){
if(resultError == "") {
success(JSON.parse(result));
}else{
console.error(`Python error, you can reproduce the error with: \n${python} ${script} ${pyArgs.join(" ")}`);
const error = new Error(resultError);
console.error(error);
reject(resultError);
}
})
});
}
module.exports.callPython = callPython;
Call:
const pythonCaller = require("../core/pythonCaller");
const result = await pythonCaller.callPython("preprocessorSentiment.py", {"thekeyYouwant": value});
python:
try:
argu = json.loads(sys.argv[1])
except:
raise Exception("error while loading argument")
I'm on node 10 and child process 1.0.2. The data from python is a byte array and has to be converted. Just another quick example of making a http request in python.
node
const process = spawn("python", ["services/request.py", "https://www.google.com"])
return new Promise((resolve, reject) =>{
process.stdout.on("data", data =>{
resolve(data.toString()); // <------------ by default converts to utf-8
})
process.stderr.on("data", reject)
})
request.py
import urllib.request
import sys
def karl_morrison_is_a_pedant():
response = urllib.request.urlopen(sys.argv[1])
html = response.read()
print(html)
sys.stdout.flush()
karl_morrison_is_a_pedant()
p.s. not a contrived example since node's http module doesn't load a few requests I need to make
You could take your python, transpile it, and then call it as if it were javascript. I have done this succesfully for screeps and even got it to run in the browser a la brython.
The Boa is good for your needs, see the example which extends Python tensorflow keras.Sequential class in JavaScript.
const fs = require('fs');
const boa = require('#pipcook/boa');
const { tuple, enumerate } = boa.builtins();
const tf = boa.import('tensorflow');
const tfds = boa.import('tensorflow_datasets');
const { keras } = tf;
const { layers } = keras;
const [
[ train_data, test_data ],
info
] = tfds.load('imdb_reviews/subwords8k', boa.kwargs({
split: tuple([ tfds.Split.TRAIN, tfds.Split.TEST ]),
with_info: true,
as_supervised: true
}));
const encoder = info.features['text'].encoder;
const padded_shapes = tuple([
[ null ], tuple([])
]);
const train_batches = train_data.shuffle(1000)
.padded_batch(10, boa.kwargs({ padded_shapes }));
const test_batches = test_data.shuffle(1000)
.padded_batch(10, boa.kwargs({ padded_shapes }));
const embedding_dim = 16;
const model = keras.Sequential([
layers.Embedding(encoder.vocab_size, embedding_dim),
layers.GlobalAveragePooling1D(),
layers.Dense(16, boa.kwargs({ activation: 'relu' })),
layers.Dense(1, boa.kwargs({ activation: 'sigmoid' }))
]);
model.summary();
model.compile(boa.kwargs({
optimizer: 'adam',
loss: 'binary_crossentropy',
metrics: [ 'accuracy' ]
}));
The complete example is at: https://github.com/alibaba/pipcook/blob/master/example/boa/tf2/word-embedding.js
I used Boa in another project Pipcook, which is to address the machine learning problems for JavaScript developers, we implemented ML/DL models upon the Python ecosystem(tensorflow,keras,pytorch) by the boa library.
/*eslint-env es6*/
/*global require*/
/*global console*/
var express = require('express');
var app = express();
// Creates a server which runs on port 3000 and
// can be accessed through localhost:3000
app.listen(3000, function() {
console.log('server running on port 3000');
} )
app.get('/name', function(req, res) {
console.log('Running');
// Use child_process.spawn method from
// child_process module and assign it
// to variable spawn
var spawn = require("child_process").spawn;
// Parameters passed in spawn -
// 1. type_of_script
// 2. list containing Path of the script
// and arguments for the script
// E.g : http://localhost:3000/name?firstname=Levente
var process = spawn('python',['apiTest.py',
req.query.firstname]);
// Takes stdout data from script which executed
// with arguments and send this data to res object
var output = '';
process.stdout.on('data', function(data) {
console.log("Sending Info")
res.end(data.toString('utf8'));
});
console.log(output);
});
This worked for me. Your python.exe must be added to you path variables for this code snippet. Also, make sure your python script is in your project folder.
const util = require('util');
const exec = util.promisify(require('child_process').exec);
function runPythonFile() {
const { stdout, stderr } = await exec('py ./path_to_python_file -s asdf -d pqrs');
if (stdout) { // do something }
if (stderr) { // do something }
}
For more information visit official Nodejs child process page: https://nodejs.org/api/child_process.html#child_processexeccommand-options-callback
you can check out my package on npm
https://www.npmjs.com/package/#guydev/native-python
it provides a very simple and powerful way to run python functions from node
import { runFunction } from '#guydev/native-python'
const example = async () => {
const input = [1,[1,2,3],{'foo':'bar'}]
const { error, data } = await runFunction('/path/to/file.py','hello_world', '/path/to/python', input)
// error will be null if no error occured.
if (error) {
console.log('Error: ', error)
}
else {
console.log('Success: ', data)
// prints data or null if function has no return value
}
}
python module
# module: file.py
def hello_world(a,b,c):
print( type(a), a)
# <class 'int'>, 1
print(type(b),b)
# <class 'list'>, [1,2,3]
print(type(c),c)
# <class 'dict'>, {'foo':'bar'}
I'm making an uploader for my project, and I try to avoid any flash-based solutions (I dont like flash much and target for mobile platforms support).
The whole process seems rather simple: I have a form, nice jQuery progress bar, I can make ajax requests with timeout from a script to update progressbar status...
If I do it according to Webpy cookbook, the only thing I don't get is how to recieve any info from server: how many bytes/chunks/whatever were already written?
You can create ajax uploader using FormData for the modern browsers.
$.fn.uploader = function( options ) {
var defaults = {},
opts = $.extend( defaults, options ),
that = $( this ),
url = that.data( "upload_url" ),
is_uploading = false;
function upload( files ) {
$.get( "/file/blank.html" );
if ( FormData === undefined ) {
alert( "Your browser does not support standard HTML5 Drag and Drop" );
return;
}
var xhr = new XMLHttpRequest(),
new_element = $( "<li><p>Loading</p><span></span></li>" )
.appendTo( that ),
xhr_upload = xhr.upload,
form = new FormData();
xhr_upload.addEventListener( "progress", function( e ) {
if( e.lengthComputable ) {
var p = Math.round( e.loaded * 100 / e.total );
new_element.children( "span" ).text( e.loaded == e.total ? "Processing..." : p + "%" );
}
}, false);
xhr_upload.addEventListener( "load", function( e ){}, false);
xhr_upload.addEventListener( "error", function( error ) { alert( "error: " + error ); }, false);
xhr.open( "POST", url, true );
xhr.setRequestHeader( "X-Requested-With", "XMLHttpRequest" );
xhr.onreadystatechange = function ( e ) {
if ( xhr.readyState == 4 ) {
is_uploading = false;
if( xhr.status == 200 ) {
var data = $.parseJSON( e.target.responseText );
if ( data.status == 0 ) {
new_element
.fadeOut(function (){ $( this ).remove(); })
.children( "span" ).text( "Upload error!" );
} else {
that.html( data.html );
}
} else {
new_element
.fadeOut(function (){ $( this ).remove(); })
.children( "span" ).text( "Upload error!" );
}
}
};
$.each( files, function() {
form.append( "files", this );
});
is_uploading = true;
xhr.send( form );
}
that.bind({
"dragover": function( e ) {
var dt = e.originalEvent.dataTransfer;
if( !dt || is_uploading ) { return; };
if( $.browser.webkit ) { dt.dropEffect = "copy"; };
$( this ).addClass( "active" );
return false;
},
"dragleave": function( e ) {
$( this ).removeClass( "active" );
},
"dragenter": function( e ){ return false; },
"drop": function( e ){
var dt = e.originalEvent.dataTransfer;
$( this ).removeClass( "active" );
if( !dt || !dt.files || is_uploading ) { return; };
upload( dt.files );
return false;
}
});
$( document ).bind({
'dragenter': function( e ) { return false; },
'dragleave': function( e ) { return false; },
'dragover': function( e ) {
var dt = e.originalEvent.dataTransfer;
if ( !dt ) { return; }
dt.dropEffect = "none";
return false;
}
});
};
On the server side I process it like this:
def POST(self):
i = web.webapi.rawinput()
try:
files = i.files
if not isinstance(files, list):
files = [files]
for f in files:
if f.filename:
filetype, encoding = mimetypes.guess_type(f.filename)
# do smth with f.file
except KeyError:
pass
if web.ctx.env.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest':
web.header("Content-Type", "application/json")
return json.dumps(dict(status=1, html=unicode(render_partial.files(uploaded_files))))
else:
raise web.seeother(web.ctx.env.get("HTTP_REFERER", "/"))
Otherwise you may look into nginx upload progress module or apache2 upload progress module, uWSGI has got this feature too.