I have an Express Node.js application, but I also have a machine learning algorithm to use in Python. Is there a way I can call Python functions from my Node.js application to make use of the power of machine learning libraries?
Easiest way I know of is to use "child_process" package which comes packaged with node.
Then you can do something like:
const spawn = require("child_process").spawn;
const pythonProcess = spawn('python',["path/to/script.py", arg1, arg2, ...]);
Then all you have to do is make sure that you import sys in your python script, and then you can access arg1 using sys.argv[1], arg2 using sys.argv[2], and so on.
To send data back to node just do the following in the python script:
print(dataToSendBack)
sys.stdout.flush()
And then node can listen for data using:
pythonProcess.stdout.on('data', (data) => {
// Do something with the data returned from python script
});
Since this allows multiple arguments to be passed to a script using spawn, you can restructure a python script so that one of the arguments decides which function to call, and the other argument gets passed to that function, etc.
Hope this was clear. Let me know if something needs clarification.
Example for people who are from Python background and want to integrate their machine learning model in the Node.js application:
It uses the child_process core module:
const express = require('express')
const app = express()
app.get('/', (req, res) => {
const { spawn } = require('child_process');
const pyProg = spawn('python', ['./../pypy.py']);
pyProg.stdout.on('data', function(data) {
console.log(data.toString());
res.write(data);
res.end('end');
});
})
app.listen(4000, () => console.log('Application listening on port 4000!'))
It doesn't require sys module in your Python script.
Below is a more modular way of performing the task using Promise:
const express = require('express')
const app = express()
let runPy = new Promise(function(success, nosuccess) {
const { spawn } = require('child_process');
const pyprog = spawn('python', ['./../pypy.py']);
pyprog.stdout.on('data', function(data) {
success(data);
});
pyprog.stderr.on('data', (data) => {
nosuccess(data);
});
});
app.get('/', (req, res) => {
res.write('welcome\n');
runPy.then(function(fromRunpy) {
console.log(fromRunpy.toString());
res.end(fromRunpy);
});
})
app.listen(4000, () => console.log('Application listening on port 4000!'))
The python-shell module by extrabacon is a simple way to run Python scripts from Node.js with basic, but efficient inter-process communication and better error handling.
Installation:
With npm:
npm install python-shell.
Or with yarn:
yarn add python-shell
Running a simple Python script:
const PythonShell = require('python-shell').PythonShell;
PythonShell.run('my_script.py', null, function (err) {
if (err) throw err;
console.log('finished');
});
Running a Python script with arguments and options:
const PythonShell = require('python-shell').PythonShell;
var options = {
mode: 'text',
pythonPath: 'path/to/python',
pythonOptions: ['-u'],
scriptPath: 'path/to/my/scripts',
args: ['value1', 'value2', 'value3']
};
PythonShell.run('my_script.py', options, function (err, results) {
if (err)
throw err;
// Results is an array consisting of messages collected during execution
console.log('results: %j', results);
});
For the full documentation and source code, check out https://github.com/extrabacon/python-shell
You can now use RPC libraries that support Python and Javascript such as zerorpc
From their front page:
Node.js Client
var zerorpc = require("zerorpc");
var client = new zerorpc.Client();
client.connect("tcp://127.0.0.1:4242");
client.invoke("hello", "RPC", function(error, res, more) {
console.log(res);
});
Python Server
import zerorpc
class HelloRPC(object):
def hello(self, name):
return "Hello, %s" % name
s = zerorpc.Server(HelloRPC())
s.bind("tcp://0.0.0.0:4242")
s.run()
Many of the examples are years out of date and involve complex setup. You can give JSPyBridge/pythonia a try (full disclosure: I'm the author). It's vanilla JS that lets you operate on foreign Python objects as if they existed in JS. In fact, it does interoperability so Python code can in return call JS through callbacks and passed functions.
numpy + matplotlib example, with the ES6 import system:
import { py, python } from 'pythonia'
const np = await python('numpy')
const plot = await python('matplotlib.pyplot')
// Fixing random state for reproducibility
await np.random.seed(19680801)
const [mu, sigma] = [100, 15]
// Inline expression evaluation for operator overloading
const x = await py`${mu} + ${sigma} * ${np.random.randn(10000)}`
// the histogram of the data
const [n, bins, patches] = await plot.hist$(x, 50, { density: true, facecolor: 'g', alpha: 0.75 })
console.log('Distribution', await n) // Always await for all Python access
await plot.show()
python.exit()
Through CommonJS (without top level await):
const { py, python } = require('pythonia')
async function main() {
const np = await python('numpy')
const plot = await python('matplotlib.pyplot')
...
// the rest of the code
}
main().then(() => python.exit()) // If you don't call this, the process won't quit by itself.
Most of previous answers call the success of the promise in the on("data"), it is not the proper way to do it because if you receive a lot of data you will only get the first part. Instead you have to do it on the end event.
const { spawn } = require('child_process');
const pythonDir = (__dirname + "/../pythonCode/"); // Path of python script folder
const python = pythonDir + "pythonEnv/bin/python"; // Path of the Python interpreter
/** remove warning that you don't care about */
function cleanWarning(error) {
return error.replace(/Detector is not able to detect the language reliably.\n/g,"");
}
function callPython(scriptName, args) {
return new Promise(function(success, reject) {
const script = pythonDir + scriptName;
const pyArgs = [script, JSON.stringify(args) ]
const pyprog = spawn(python, pyArgs );
let result = "";
let resultError = "";
pyprog.stdout.on('data', function(data) {
result += data.toString();
});
pyprog.stderr.on('data', (data) => {
resultError += cleanWarning(data.toString());
});
pyprog.stdout.on("end", function(){
if(resultError == "") {
success(JSON.parse(result));
}else{
console.error(`Python error, you can reproduce the error with: \n${python} ${script} ${pyArgs.join(" ")}`);
const error = new Error(resultError);
console.error(error);
reject(resultError);
}
})
});
}
module.exports.callPython = callPython;
Call:
const pythonCaller = require("../core/pythonCaller");
const result = await pythonCaller.callPython("preprocessorSentiment.py", {"thekeyYouwant": value});
python:
try:
argu = json.loads(sys.argv[1])
except:
raise Exception("error while loading argument")
I'm on node 10 and child process 1.0.2. The data from python is a byte array and has to be converted. Just another quick example of making a http request in python.
node
const process = spawn("python", ["services/request.py", "https://www.google.com"])
return new Promise((resolve, reject) =>{
process.stdout.on("data", data =>{
resolve(data.toString()); // <------------ by default converts to utf-8
})
process.stderr.on("data", reject)
})
request.py
import urllib.request
import sys
def karl_morrison_is_a_pedant():
response = urllib.request.urlopen(sys.argv[1])
html = response.read()
print(html)
sys.stdout.flush()
karl_morrison_is_a_pedant()
p.s. not a contrived example since node's http module doesn't load a few requests I need to make
You could take your python, transpile it, and then call it as if it were javascript. I have done this succesfully for screeps and even got it to run in the browser a la brython.
The Boa is good for your needs, see the example which extends Python tensorflow keras.Sequential class in JavaScript.
const fs = require('fs');
const boa = require('#pipcook/boa');
const { tuple, enumerate } = boa.builtins();
const tf = boa.import('tensorflow');
const tfds = boa.import('tensorflow_datasets');
const { keras } = tf;
const { layers } = keras;
const [
[ train_data, test_data ],
info
] = tfds.load('imdb_reviews/subwords8k', boa.kwargs({
split: tuple([ tfds.Split.TRAIN, tfds.Split.TEST ]),
with_info: true,
as_supervised: true
}));
const encoder = info.features['text'].encoder;
const padded_shapes = tuple([
[ null ], tuple([])
]);
const train_batches = train_data.shuffle(1000)
.padded_batch(10, boa.kwargs({ padded_shapes }));
const test_batches = test_data.shuffle(1000)
.padded_batch(10, boa.kwargs({ padded_shapes }));
const embedding_dim = 16;
const model = keras.Sequential([
layers.Embedding(encoder.vocab_size, embedding_dim),
layers.GlobalAveragePooling1D(),
layers.Dense(16, boa.kwargs({ activation: 'relu' })),
layers.Dense(1, boa.kwargs({ activation: 'sigmoid' }))
]);
model.summary();
model.compile(boa.kwargs({
optimizer: 'adam',
loss: 'binary_crossentropy',
metrics: [ 'accuracy' ]
}));
The complete example is at: https://github.com/alibaba/pipcook/blob/master/example/boa/tf2/word-embedding.js
I used Boa in another project Pipcook, which is to address the machine learning problems for JavaScript developers, we implemented ML/DL models upon the Python ecosystem(tensorflow,keras,pytorch) by the boa library.
/*eslint-env es6*/
/*global require*/
/*global console*/
var express = require('express');
var app = express();
// Creates a server which runs on port 3000 and
// can be accessed through localhost:3000
app.listen(3000, function() {
console.log('server running on port 3000');
} )
app.get('/name', function(req, res) {
console.log('Running');
// Use child_process.spawn method from
// child_process module and assign it
// to variable spawn
var spawn = require("child_process").spawn;
// Parameters passed in spawn -
// 1. type_of_script
// 2. list containing Path of the script
// and arguments for the script
// E.g : http://localhost:3000/name?firstname=Levente
var process = spawn('python',['apiTest.py',
req.query.firstname]);
// Takes stdout data from script which executed
// with arguments and send this data to res object
var output = '';
process.stdout.on('data', function(data) {
console.log("Sending Info")
res.end(data.toString('utf8'));
});
console.log(output);
});
This worked for me. Your python.exe must be added to you path variables for this code snippet. Also, make sure your python script is in your project folder.
const util = require('util');
const exec = util.promisify(require('child_process').exec);
function runPythonFile() {
const { stdout, stderr } = await exec('py ./path_to_python_file -s asdf -d pqrs');
if (stdout) { // do something }
if (stderr) { // do something }
}
For more information visit official Nodejs child process page: https://nodejs.org/api/child_process.html#child_processexeccommand-options-callback
you can check out my package on npm
https://www.npmjs.com/package/#guydev/native-python
it provides a very simple and powerful way to run python functions from node
import { runFunction } from '#guydev/native-python'
const example = async () => {
const input = [1,[1,2,3],{'foo':'bar'}]
const { error, data } = await runFunction('/path/to/file.py','hello_world', '/path/to/python', input)
// error will be null if no error occured.
if (error) {
console.log('Error: ', error)
}
else {
console.log('Success: ', data)
// prints data or null if function has no return value
}
}
python module
# module: file.py
def hello_world(a,b,c):
print( type(a), a)
# <class 'int'>, 1
print(type(b),b)
# <class 'list'>, [1,2,3]
print(type(c),c)
# <class 'dict'>, {'foo':'bar'}
Related
app.get('/StartMonitoring', (req, res) => {
var spawn = require('child_process').spawn;
var python_process = spawn('python', ["social_distancing_detector.py"]);
// parseInt(data)
python_process.stdout.on('data', function (data) {
var violations = parseInt(JSON.parse(data));
mysqlConnection.query(`INSERT INTO daily_violations (violations, date_created, created_at) VALUES (${violations}, '${dt.format("Y-m-d")}', '${dt.format("Y-m-d H:M:S")}')`, (err, rows) => {
if (!err) {
res.render('index', { resultz: violations});
} else {
console.log(err);
}
});
});
});
i want to get a specific print value of python and pass it to my node.js program. it seems to pass the first print value
Step-1:
Open Visual Studio, go to File Menu, select New Project (Ctrl+Shift+N). A new window will open to create a project. Search for Windows Service (C#) from the upper right corner search box. If you do not get any Windows Service, your visual studio need to install properly (install .NET Desktop Development).
Give a service name PythonService and select OK.
Step-2:
After creating service project, User will get 04 files by default- App.config, Program.cs, ProjectInstaller.cs, Service1.cs
App.config is to configure any app data or creating connection
Program.cs will initiate the service
ProjectInstaller.cs is the installer for service. Double click on it. Click right button and add service installer. serviceInstaller1 will be added on serviceProcessInstaller1. Select property of serviceInstaller1, give a ServieName, change StartType to Automatic (if needed).
Go to serviceProcessInstaller1 property, change Account to LocalSystem
Up to this step, I provided basic service creating. Now below described the main goal.
Step-3:
Service1.cs file will consist all the necessary code for python cmd run and execute python command. Then Django server will run. The code for core python command run is-
public partial class Service1 : ServiceBase
{
private System.Timers.Timer timer = new System.Timers.Timer();
int ScheduleTime = Convert.ToInt32(ConfigurationManager.AppSettings["ThreadTime"]);
string _workingDirectory = ConfigurationManager.AppSettings["WorkingDirectory"];
string _RunServerCommand = ConfigurationManager.AppSettings["RunServerCommand"];
int sent = 0;
public Service1()
{
InitializeComponent();
}
protected override void OnStart(string[] args)
{
try
{
CommonFeature.WriteToFile("timer instantiated");
timer.Elapsed += OnElapsedTime;
timer.Interval = 5000;
timer.Enabled = true;
timer.AutoReset = false;
timer.Start();
}
catch (Exception ex)
{
CommonFeature.WriteToFile(ex.Message);
}
}
private void OnElapsedTime(object sender, ElapsedEventArgs e)
{
Execute();
}
protected override void OnStop()
{
timer.Enabled = false;
timer.Stop();
ThreadStart stop = new ThreadStart(ClosePythonInstaller);
Thread stopThread = new Thread(stop);
stopThread.Start();
Thread.Sleep(ScheduleTime * 5000);
stopThread.Abort();
}
private static void ClosePythonInstaller()
{
Process[] processList = Process.GetProcesses();
foreach (Process clsProcess in processList)
{
if (clsProcess.ProcessName == "python")
{
if (!clsProcess.HasExited)
{
clsProcess.Kill();
clsProcess.WaitForExit();
}
}
}
}
private void Execute()
{
//CommonFeature.WriteToFile(string.Format("{0}", _counter1++));
try
{
if (Process.GetProcessesByName("python").Count() == 0)
{
CommonFeature.WriteToFile("Service started");
var workingDirectory = _workingDirectory;
var runServerCommand = _RunServerCommand;
Process process = new Process();
process.StartInfo.FileName = "cmd.exe";
process.StartInfo.CreateNoWindow = true;
process.StartInfo.RedirectStandardInput = true;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.UseShellExecute = false;
process.StartInfo.WorkingDirectory = workingDirectory;
process.Exited += new EventHandler(process_Exited);
process.Start();
CommonFeature.WriteToFile("process started");
process.StandardInput.WriteLine(runServerCommand);
process.StandardInput.Flush();
process.StandardInput.Close();
using (StreamReader reader = process.StandardOutput)
{
CommonFeature.WriteToFile("cmd command executed");
//CommonFeature.WriteToFile(string.Format("{0}", _counter2++));
string result = reader.ReadToEnd();
Console.Write(result);
}
CommonFeature.WriteToFile("script run in cmd complete");
}
}
catch (Exception)
{
throw;
}
}
protected void process_Exited(object sender, EventArgs e)
{
var p = (Process)sender;
p.WaitForExit();
sent += p.ExitCode;
}
}
This is the complete executable code for PythonService.
If there has any necessity to check the service is running or not then, user can push a message through the below class and get the final result executed or not.
public class CommonFeature
{
public static void WriteToFile(string Message)
{
string LogStatus = ConfigurationManager.AppSettings.Get("LogStatus");
if (LogStatus != null && LogStatus.ToUpper() == "ON")
{
string path = AppDomain.CurrentDomain.BaseDirectory + "\\Logs";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string filepath = AppDomain.CurrentDomain.BaseDirectory + "\\Logs\\ServiceLog_" + DateTime.Now.Date.ToShortDateString().Replace('/', '_') + ".txt";
if (!File.Exists(filepath))
{
// Create a file to write to.
using (StreamWriter sw = File.CreateText(filepath))
{
sw.WriteLine(Message);
}
}
else
{
using (StreamWriter sw = File.AppendText(filepath))
{
sw.WriteLine(Message);
}
}
}
}
}
I have this piece of code. It should call my python script if it is true.
The problem is: the script doesn't get executed. I'm using Heroku and I already have nodejs and python in my buildpacks.
const express = require('express');
const bodyParser = require('body-parser');
app.use(bodyParser.urlencoded({extended: true}));
app.use(bodyParser.json());
let {PythonShell} = require('python-shell');
app.post('/api/apipath', (req, res) => {
if (currentPhase == phaseId){
PythonShell.run('./src/main.py', null, function (err){
if (err) throw err;
console.log('Finish');
});
res.end();
}else{
res.end();
}
}
Note: I already forced the condition to be true and it didn't work. The script was not called
I'm not familiar with python-shell but you can use exec from Node's child_process to run your script
const { exec } = require('child_process')
function runPythonScript(script, runFile = false) {
console.log(`Run? ${runFile}`)
if (runFile) {
exec(`python ${script}`, (error, stdout, stderr) => {
if (error) {
console.error(`exec error: ${error}`)
return
}
console.log(`stdout: ${stdout}`)
console.error(`stderr: ${stderr}`)
})
}
}
Running the code:
// this will run
runPythonScript('./src/main.py', 1 === 1)
// this will not run
runPythonScript('./src/main.py', 0)
// this will run since both currentPhase and phaseId are empty variables
var currentPhase, phaseId
runPythonScript('./src/main.py', currentPhase === phaseId)
CLI
add this code for if you want to define currentPhase and phaseId from the command line
function init() {
const [, , ...args] = process.argv
var currentPhase, phaseId
currentPhase = args[args.indexOf('--currentPhase') - 1]
phaseId = args[args.indexOf('--phaseId') - 1]
runPythonScript('./program.py', currentPhase === phaseId)
}
init()
from the CLI, you can run
# sets currentPhase to 99 and phaseId as 98 (does not run)
node lib/filename-to-runPythonScript.js 99 --currentPhase 98 --phaseId
# sets both currentPhase and phaseId to 123 (does run)
node lib/filename-to-runPythonScript.js 123 --currentPhase 123 --phaseId
AWS CloudWatch Events to trigger a Lambda function that polls for messages on a queue. Messages are then sent to a second Lambda function that writes the message to an Kinesis agent.
I found a solution to this article but written in node.js. But I am looking for Python script. Could you please help me to get a article where I can find a way to convert it by myself.
https://github.com/aws-samples/amazonmq-invoke-aws-lambda
const stomp = require('stompit')
const AWS = require('aws-sdk')
let lambda = new AWS.Lambda()
const QUEUE_NAME = process.env.QUEUE_NAME
const CONNECTION_OPTIONS = {
host: process.env.HOST,
port: 61614,
ssl: true,
connectHeaders: {
login: process.env.LOGIN,
passcode: process.env.PASSWORD
}
}
exports.handler = (event, context, callback) => {
var client
/* */
const onError = (error) => {
console.error(`[ERROR] ${error}`)
callback(error)
}
/* */
const onMessage = (error, message) => {
console.log('Message received')
message.readString('utf-8', (error, body) => {
if (error) {
onError(error)
return
}
let payload = {
message: body,
timestamp: Date.now()
}
console.log(payload)
let params = {
FunctionName: process.env.WORKER_FUNCTION,
Payload: JSON.stringify(payload)
}
lambda.invoke(params, (error, data) => {
if (error) {
console.error(`Could not invoke Lambda: ${error}`)
}
})
})
}
/* Main */
stomp.connect(CONNECTION_OPTIONS, (error, client) => {
if (error) {
onError(error)
return
}
var headers = {
destination: `/queue/${QUEUE_NAME}`,
ack: 'auto'
}
client.subscribe(headers, onMessage)
setTimeout(() => {
client.disconnect()
callback(null, { 'message': 'Finished' })
}, context.getRemainingTimeInMillis() - 1000)
})
}
You don't make it clear what you are trying to achieve, how far you got to, and what issue you are facing.
You can improve your question by following the the SO guidelines.
The library to interact with AWS services in python(3) is boto3.
Whenever I am running the following code I am getting the error
(node) warning: possible EventEmitter memory leak detected. 11 message listeners added. Use emitter.setMaxListeners() to increase limit.
The code:
function compute() {
var PythonShell = require('python-shell');
var options = {
scriptPath: '/home/pi/projectFinal/nodejsControllingServo/sensorTest'
};
var pyshell = new PythonShell('sensor.py',options);
var eventsEmitter = require("events").EventEmitter;
pyshell.on('message', function (message) {
myValueList [counter] = message;
counter=counter+1;
});
pyshell.end(function (err) {
if (err) throw err;
for ( var i = 4; i < myValueList.length; i++) {
console.log ("My value : " + myValueList[i]);
});
sleep.usleep(5000);
process.nextTick(compute);
}
compute();
What could be the possible solution so that I can get rid of memory leak?
Thanks in advance