Render html table from csv using node.js, express and python - python
I have a simple csv file and want to render a html table from it using express and ejs.
csv:
,Unnamed: 0,sign_type,r1,g1,b1,r2,g2,b2,r3,g3,b3,r4,g4,b4,r5,g5,b5,r6,g6,b6,r7,g7,b7,r8,g8,b8,r9,g9,b9,r10,g10,b10,r11,g11,b11,r12,g12,b12,r13,g13,b13,r14,g14,b14,r15,g15,b15,r16,g16,b16
0,1,pedestrian,155,228,251,135,188,101,156,227,245,145,211,228,166,233,245,212,254,52,212,254,11,188,229,117,170,216,120,211,254,3,212,254,19,172,235,244,172,235,244,172,228,235,177,235,244,22,52,53
1,2,pedestrian,142,217,242,166,204,44,142,217,242,147,219,242,164,228,229,84,116,17,217,254,26,155,203,128,213,253,51,217,255,21,217,255,21,158,225,237,164,227,237,182,228,143,171,228,196,164,227,237
2,3,pedestrian,57,54,50,187,201,68,51,51,45,59,62,65,156,171,50,254,255,36,211,226,70,78,73,64,220,234,59,254,255,51,253,255,44,66,68,68,69,65,59,76,84,22,82,93,17,58,60,60
3,4,pedestrian,22,35,41,171,178,26,19,27,29,19,27,29,42,37,3,217,228,19,221,235,20,181,183,73,237,234,44,251,254,2,235,243,12,19,27,29,20,29,34,64,61,4,211,222,78,19,27,29
4,5,pedestrian,169,179,170,231,254,27,97,107,99,123,147,152,221,236,117,205,225,80,235,254,60,90,110,9,216,236,66,229,255,12,235,254,60,163,168,152,124,117,91,188,205,78,125,147,20,160,183,187
my .html file is as follows:
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8">
<title>Table</title>
</head>
<body>
<%= table %>
</body>
</html>
my app.js is:
const express = require('express');
const app = express();
const bodyParser = require('body-parser');
let {PythonShell} = require('python-shell');
app.set('view engine', 'ejs');
app.get('/', function(req, res){
var pyshell = new PythonShell('excel_to_html.py');
pyshell.on('message', function (message) {
res.render('list', {table: message}) // ###### console.log(message);
});
});
var port = 3000;
app.listen(port, function(){
console.log('Server running on port ' + port)
});
and the python file which converts the .csv to html is:
import pandas as pd
def convert():
csv_file = pd.read_csv('test.csv')
html = csv_file.to_html()
return html
if __name__ == '__main__':
var = convert()
print(var)
So summarizing:
1) I have a csv file and using a .py file and pandas to convert the csv into html.
2) I'm calling the python script inside the app.js file and getting the returned html.
3) I want to render that html table in my index.html using ejs.
If I replace the res.render('list', {table: message}) with the commented part console.log(message); I'm able to see the html table in the terminal log.
but when I try to render it using res.render('list', {table: message}) i get a not infinite looping error like this:
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
at ServerResponse.setHeader (_http_outgoing.js:485:11)
at ServerResponse.header (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\response.js:771:10)
at ServerResponse.send (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\response.js:170:12)
at done (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\response.js:1008:10)
at tryHandleCache (E:\Documents\CODE\WebDev\todolist\v1\node_modules\ejs\lib\ejs.js:260:5)
at View.exports.renderFile [as engine] (E:\Documents\CODE\WebDev\todolist\v1\node_modules\ejs\lib\ejs.js:459:10)
at View.render (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\view.js:135:8)
at tryRender (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\application.js:640:10)
at Function.render (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\application.js:592:3)
at ServerResponse.render (E:\Documents\CODE\WebDev\todolist\v1\node_modules\express\lib\response.js:1012:7)
And I get only this in the browser:
<table border="1" class="dataframe">
how should I render this table? Is it easily possible?
Thank you!
Basically Express's res.render ends the response; which means we can only use it once, and it seems like the listener pyshell.on('message'... is being triggered more than once.
You will need to prepare the table then render it. Example:
const express = require('express');
const app = express();
const { PythonShell } = require('python-shell');
app.set('view engine', 'ejs');
app.get('/', function(req, res){
prepareHtmlTable(function(err, html) {
if (err) return res.sendStatus(500);
res.render('list', { table: html });
})
});
function prepareHtmlTable(callback) {
const pyshell = new PythonShell('excel_to_html.py');
pyshell.on('message', function (message) {
// prepare html table here
let html = message;
pyshell.end(function (err,code,signal) {
if (err) return callback(err);
callback(null, html);
});
});
}
const port = 3000;
app.listen(port, function(){
console.log('Server running on port ' + port)
});
Related
Client-side calls server to run python
I'm quite a beginner and I've been looking for an answer for days now.. I am building a webapp using nodejs. Here is the app.js I node to start (my server-side) http = require("http"), path = require("path"), url = require("url"), fs = require("fs"); function sendError(errCode, errString, response) { response.writeHead(errCode, {"Content-Type": "text/plain"}); response.write(errString + "\n"); response.end(); return; } function sendFile(err, file, response) { if(err) return sendError(500, err, response); response.writeHead(200); response.write(file, "binary"); response.end(); } function getFile(exists, response, localpath) { if(!exists) return sendError(404, '404 Not Found', response); fs.readFile(localpath, "binary", function(err, file){ sendFile(err, file, response);}); } function getFilename(request, response) { var urlpath = url.parse(request.url).pathname; // following domain or IP and port var localpath = path.join(process.cwd(), urlpath); // if we are at root fs.exists(localpath, function(result) { getFile(result, response, localpath)}); } var server = http.createServer(getFilename); server.listen(1000); console.log("Server available..."); I can therefore access my app using localhost:1000/index.html My index.html (client-side) is a html page containing buttons i want to use to start a python script. <!DOCTYPE html> <html lang="fr"> <head> <meta charset="utf-8"> <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script> <title>Lights</title> <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" integrity="sha384-9aIt2nRpC12Uk9gS9baDl411NQApFmC26EwAOH8WgZl5MYYxFfc+NcPb1dKGj7Sk" crossorigin="anonymous"> </head> <body class="text-white bg-dark"> <center> <input type="submit" value="OFF" name="OFF" onclick="turnOff()"> <input type="submit" value="ON" name="ON" onclick="turnOn()"> </center> <div id="texttest"> </div> <div id="fil"> </div> <script src="funcs.js"></script> </body> </html> from there you can see i use a script to run the "onclick()" this script is calling a python run.. But obviously I can't do that 'cause I am trying to run python client side.. ERROR "require not definded" my funcs.js const { exec } = require('child_process'); function turnOff() { console.log("enter Off"); exec('python3 turnOff.py', (err, stdout, stderr) => { if (err) { document.getElementById("fil").innerHTML = "error..."+err; } else { document.getElementById("fil").innerHTML = "running python..."; } }); document.getElementById("texttest").innerHTML = "trying off sucess :<br> " ; } function turnOn() { console.log("enter On"); exec('python3 turnOn.py', (err, stdout, stderr) => { if (err) { document.getElementById("fil").innerHTML = "error..."+err; } else { document.getElementById("fil").innerHTML = "running python..."; } }); document.getElementById("texttest").innerHTML = "trying on sucess :<br> " ; } I should be using AJAX request, but i can't seem to find how to run python on server side using it.. because the ajax request is sent from html inside nodejs.. What can I do ? Thanks for your help
You're right about your errors require is defined on server side only. funcs.js is executed on client side, so you can't use require. And you'll need to make an AJAX call in order to reach your goal. // server.js const { exec } = require('child_process'); const http = require('http'); const server = http.createServer((req, res) => { /* When a request will call the http://localhost:5000/run url it will execute this code block */ if (req.url === '/run') { exec('python ...', () => { // ... res.write('Python script executed'); res.end(); }); } }); server.listen(5000); // funcs.js function turnOff() { console.log("enter Off"); // Turn off will send a request on the http://localhost:5000/run endpoint fetch("/run", { method: "POST" }) .then(() => { document.getElementById("fil").innerHTML = "running python..."; }) .catch(() => { document.getElementById("fil").innerHTML = "error..."+err; }) }
How to access a python script with python-shell in a sails.js App?
I want to access a python script and pass an image then wait for a json response . I should do this inside the "upload" function after getting the image. Been searching but was not able to find a python-shell with sails. Since sails is built on top of node.js then there most be a way. module.exports = { fileUpload: function(req, res) { res.writeHead(200, { 'content-type': 'text/html' }); res.end( '<form action="http://localhost:1337/postIts/upload" enctype="multipart/form-data" method="post">' + '<input type="text" name="title"><br>' + '<input type="file" name="uploadFile"><br>' + '<input type="submit" value="Upload">' + '</form>' ) }, upload: function(req, res) { var uploadFile = req.file('uploadFile') uploadFile.upload({ saveAs: 'image.png' }, function onUploadComplete(err, files) { if (err) return res.serverError(err); return res.json({ message: files.length + ' file(s) uploaded successfully!', files: files[0] }); }); } }; I want to pass this uploaded image to python script and wait for a json output which I will return to the client. I havent tried anything because I am not sure of the proper way of doing this. It should be something similar to this but i dont know the proper way. of doing it in my Controller. there should be an import python-shell somewhere, maybe before the module.export ? var PythonShell = require('python-shell'); var courseid=req.param('courseid'); sails.log.debug(coursed); var options = { args: [courseid] }; PythonShell.run('word2vec.py', options, function (err, results) { if (err) throw err; console.log('results: %s', results[0]); }); return res.ok();
So far I have tried installing pythonshell then tried an import from the controller and add the codes to the upload function which calls for the .py script but I get an error, I will upload a picture of the error. And here is the new code: import { PythonShell } from 'python-shell'; var PythonShell = require('python-shell'); module.exports = { upload: function(req, res) { var uploadFile = req.file('uploadFile') uploadFile.upload({ saveAs: 'image.png', }, function onUploadComplete(err, files) { let options = { mode: 'text', pythonOptions: ['-u'], // get print results in real-time scriptPath: sails.config.pythonPath, args: ['1233ID', files[0].fd] }; PythonShell.run('main2.py', options, function(err, results) { if (err) throw err; console.log('results: %s', results); }); if (err) return res.serverError(err); return res.json({ message: files.length + ' file(s) uploaded successfully!', files: files[0].fd }); }); } }; see the screenshot for the error
Invalid Type Error while using Msgpack with Flask and AngularJS
I have a simple Flask app like so: import msgpack from flask import Flask, render_template app = Flask(__name__) #app.route('/') def index(): return render_template('index.html') #app.route('/test') def test(): return msgpack.packb([1, 2, 3]) #app.route('/ping') def ping(): return "Pong" if __name__ == '__main__': app.run(debug=True, port=5000) In the Flask route /test, I'm using msgpack to pack an array which is returned as a response. When this response is printed to the console, it looks something like this: b'\x93\x01\x02\x03' I'm using AngularJS 1.7 in my front end which looks like: <!doctype html> <html ng-app="MyApp"> <body ng-cloak> <div class="container" ng-controller="MyController"> <h1>Hello, {{text}}</h1> <button class="btn btn-primary" ng-click="ping()">Ping</button> <button class="btn btn-primary" ng-click="getMsgpackData()">Get Msgpack Data</button> </div> <script src="jquery-3.3.1.slim.min.js"></script> <script src="/angular.js/1.7.2/angular.min.js"></script> <script src="https://rawgithub.com/kawanet/msgpack-lite/master/dist/msgpack.min.js"></script> <script> var myApp = angular.module("MyApp", []); myApp.controller("MyController", ["$scope", "$http", function($scope, $http) { $scope.text = "World"; $scope.ping = function() { $http.get("/ping").then(function success(response) { $scope.text = response.data; }, function error(response) { console.log(response); }); } $scope.getMsgpackData = function() { $http.get("/test").then(function success(response) { console.log(response); $scope.text = msgpack.decode(response.data); }, function error(response) { console.log(response); }); } }]); </script> </body> </html> When the getMsgpackData function is invoked in MyController, I get the following error in the browser's console: Error: Invalid type: 0x� at n.r [as decode] (msgpack.min.js:1) at n.fetch (msgpack.min.js:1) at n.s [as read] (msgpack.min.js:1) at Object.n [as decode] (msgpack.min.js:1) at success ((index):46) at angular.js:17396 at m.$digest (angular.js:18557) at m.$apply (angular.js:18945) at k (angular.js:12799) at V (angular.js:13056) "Possibly unhandled rejection: {}" It seems like the first HEX value \x93 is not being decoded. I'm using kawanet/msgpack-lite in the front end as I found that it can be used in a browser. Please help!
The error message tells you you sent the wrong type of object to the decode() method. The buffer decoding documentation shows that only buffers, arrays and Uint8Array objects are supported. Change the response type to arraybuffer (from the default text), then feed the response data to msgpack.decode() as a Uint8Array typed array: $http.get("/test", {responseType: "arraybuffer"}).then(function success(response) { var arr = new Uint8Array(response.data); console.log(arr); $scope.text = msgpack.decode(arr); }, function error(response) { console.log(response); }); While not necessary here, you do want to set a different content type on your Flask response. Currently the /test route is advertising the content as text/html, while you really should be labelling as application/msgpack: #app.route('/test') def test(): return msgpack.packb([1, 2, 3]), {'content-type': 'application/msgpack'}
Can't open websocket on mobile devices
I am attempting to set up a websocket with server running on my raspberry pi. The following code is slightly modified from an example I found here. I have built a whole webpage around this example allowing me to control gpio and send messages to a serial device plugged into the pi. Both that site and this example work perfectly from my laptop (windows 10 using Chrome or Firefox). However when I connect from my phone (Android 5.0.1 using Chrome for android). It appears to never open the socket. In the example code it just displays "messages go here. My first thought was the chrome on android didn't support websockets but I was able to connect and echo messages on this site http://www.websocket.org/echo.html. So it appears the functionality is there. What else would prevent the socket from opening ? pysocket.py import tornado.httpserver import tornado.websocket import tornado.ioloop import tornado.web class WSHandler(tornado.websocket.WebSocketHandler): def check_origin(self, origin): return True def open(self): print 'New connection was opened' self.write_message("Welcome to my websocket!") def on_message(self, message): print 'Incoming message:', message self.write_message("You said: " + message) def on_close(self): print 'Connection was closed...' application = tornado.web.Application([ (r'/ws', WSHandler), ]) if __name__ == "__main__": http_server = tornado.httpserver.HTTPServer(application) http_server.listen(8888) tornado.ioloop.IOLoop.instance().start() pysocket.php <!doctype html> <html> <head> <title>WebSockets with Python & Tornado</title> <meta charset="utf-8" /> <style type="text/css"> body { text-align: center; min-width: 500px; } </style> <script src="//ajax.googleapis.com/ajax/libs/jquery/1.8.3/jquery.min.js"></script> <script> $(function(){ var ws; var logger = function(msg){ var now = new Date(); var sec = now.getSeconds(); var min = now.getMinutes(); var hr = now.getHours(); $("#log").html($("#log").html() + "<br/>" + hr + ":" + min + ":" + sec + " ___ " + msg); //$("#log").animate({ scrollTop: $('#log')[0].scrollHeight}, 100); $('#log').scrollTop($('#log')[0].scrollHeight); } var sender = function() { var msg = $("#msg").val(); if (msg.length > 0) ws.send(msg); $("#msg").val(msg); } ws = new WebSocket("ws://raspberrypi-mike:8888/ws"); ws.onmessage = function(evt) { logger(evt.data); }; ws.onclose = function(evt) { $("#log").text("Connection was closed..."); $("#thebutton #msg").prop('disabled', true); }; ws.onopen = function(evt) { $("#log").text("Opening socket..."); }; $("#msg").keypress(function(event) { if (event.which == 13) { sender(); } }); $("#thebutton").click(function(){ sender(); }); }); </script> </head> <body> <h1>WebSockets with Python & Tornado</h1> <div id="log" style="overflow:scroll;width:500px; height:200px;background-color:#ffeeaa; margin:auto; text-align:left">Messages go here</div> <div style="margin:10px"> <input type="text" id="msg" style="background:#fff;width:200px"/> <input type="button" id="thebutton" value="Send" /> </div> www.LowPowerLab.com </body> </html>
ADDED SOME EXTRA CODE I would recommend using node.js for that: var express = require("express"); var app = express(); var http = require("http").Server(app); var path = require("path"); var io = require('socket.io')(http); var SerialPort = require('serialport'); var gpio = require('rpio'); http.listen(3000); var serialPort = new SerialPort.SerialPort("/dev/ttyAMA0", { baudrate: 115200, dataBits: 8, parity: "none", stopBits: 1, flowControl: false }); io.on('connection', function(socket){ console.log('Connected'); socket.on("WriteSerial:get",function(data){ var hex = new Buffer (data, "hex"); //be careful passing data writeSerial(serialPort, hex); io.emit("WriteSerial:response", "Data writen!"); }); socket.on("ReadGPIO:get",function(data){ var input = readPin(data.pin); io.emit("ReadGPIO:response", input); }); socket.on("WriteGPIO:get",function(data){ writePin(data.pin, data.time); io.emit("WriteGPIO:response", "Set!"); }); socket.on("unWriteGPIO:get",function(data){ unwritePin(data); io.emit("unWriteGPIO:response", "Set!"); }); } app.use(express.static(path.join(__dirname, '/'))); app.get("/home",function(req,res,next){ res.sendFile(path.join(__dirname + "/index.html")); }); function writeSerial (port, data) { port.write(data, function(err) { if (err) { return console.log('Error on write: ', err.message); } else { console.log('Data written: ' + data); } }); } function readPin(pin){ rpio.open(pin, rpio.INPUT); var read = rpio.read(pin) ? 'high' : 'low'; return read; } function writePin(pin, timeInMs){ rpio.open(pin, rpio.OUTPUT, rpio.LOW); rpio.write(pin, rpio.HIGH); if (timeInMs > 0) { setTimeout(function(){ rpio.write(pin, rpio.LOW); }, timeInMs); } //You can put 0 if You want it to be high until You shut it down } function unWritePin(pin){ if(readPin(pin) === 'high') { rpio.write(pin, rpio.LOW); } else { console.log("Pin already low!"); } } Be sure You have instaled the node.js with right version. If not do this in terminal: sudo apt-get remove nodered && sudo apt-get remove nodejs nodejs-legacy && curl -sL https://deb.nodesource.com/setup_4.x | sudo bash - && sudo apt-get install -y nodejs Make a folder 'server' in '/home/pi/', add server.js to it. Add code I provided to server.js. Open that folder with terminal: cd /home/pi/server/ Afther that install all modules used on server: sudo npm install express && sudo npm install http && sudo npm install path && sudo npm install socket.io && sudo npm install serialport --unsafe-perm && sudo npm install rpio --unsafe-perm Now all we have to do is create client side part. In folder '/home/pi/server' create index.html file and add folder called 'js'. In folder 'js' add socket.io.js for client side which You can find in folder '/home/pi/server/node_modules/socket.io/node_modules/socket.io-client/'. Include socket.io.js for client side into Your index.html like this: <script type="text/javascript" src="js/socket.io.js" /></script> Also add main.js file to 'js' folder where You will put Your javascript code in and include it to index.html: <script type="text/javascript" src="js/main.js" /></script> <script type="text/javascript" src="js/jquery.js" /></script> I will not make any graphics for but some main.js code is here: $(document).ready(function() { var socket = io.connect('http://your_ip_address_rpi:3000'); $( "#myButton" ).click(function(){ io.emit("WriteSerial:get", $("#myTextHolder").val()); //"FAAF531C" this is string of hex, should be added some filter to pass error when char is not part of HEX! }); $( "#myButton2" ).click(function(){ io.emit("WriteGPIO:get", {"pin" : $("#myPinHolder").val(), "time" : $("#myTimeHolder").val()}) } To run server on RPI startup add 'sudo node /home/pi/server/server.js &' to '/etc/rc.local' before 'exit 0' with 'sudo nano' editor. It will work really good on any device.
For the hostnames of the devices to work across network, a device has to advertise its own hostname or just responding to DNS queries for its own hostname. Whatever implementation the Raspberry Pi is using, your laptop is supporting it, but your phone isn't. So, to be able to connect, you need to change your hostname raspberrypi-mike to your Raspberry Pi's IP address, inside your JavaScript code.
How to call a python script with ajax in cherrypy app
I am trying to get the output from a python script and put it into a table in the html of my cherrypy app. Example app: import string, os import cherrypy file_path = os.getcwd() html = """<head> <meta content="text/html; charset=ISO-8859-1" http-equiv="content-type"> <title>CCMF</title> <link rel='shortcut icon' type='image/x-icon' href='img/favicon.ico' /> <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script> <script> function b1() { var request = $.ajax({ url: "b1.py", type: "POST", dataType: "text" }); request.done(function(msg) { $("#output").html(msg); }); request.fail(function(jqXHR, textStatus) { alert( "Request failed: " + textStatus ); }); } </script> </head> <button onclick="b1()">call b1.py</button> ... <td id = "output"; style="vertical-align: top; height: 90%; width: 100%;"> <--output goes here --> </td> ... </html> """ class ccmf(object): #cherrypy.expose def index(self): return html if __name__ == '__main__': cherrypy.server.socket_host = "127.0.0.1" cherrypy.server.socket_port = 8084 config = { "/img": { "tools.staticdir.on": True, "tools.staticdir.dir": os.path.join(file_path, "img"), } } cherrypy.tree.mount(ccmf(), "/", config=config) cherrypy.engine.start() cherrypy.engine.block() and here's the example python script b1.py: def b1(): op = "ajax b1 pushed" print op return op b1() The ajax get's called but returns the failure alert. I have tried GET, POST, "text", "html", b1.py is in the same directory, no joy. All currently running on my local box. Any hints greatly appreciated!
You are completely misunderstanding how modern, CherryPy's for instance, routing works. Unlike outdated approaches that were commonly employed with CGI and Apache's mod_* (mod_php, mod_python, etc.), where you directly point to the file containing the script with URL, modern routing is an application level activity. Your application receives all requests and dispatches them according to the established method. CherryPy in that sense has two major approaches: built-in object tree dispatcher and Routes adapter. For most simple and mid-level cases built-in dispatcher is fair enough. Basically it can look like this. app.py #!/usr/bin/env python # -*- coding: utf-8 -*- import os import cherrypy from cherrypy.lib.static import serve_file path = os.path.abspath(os.path.dirname(__file__)) config = { 'global' : { 'server.socket_host' : '127.0.0.1', 'server.socket_port' : 8080, 'server.thread_pool' : 8 } } class App: #cherrypy.expose def index(self): return serve_file(os.path.join(path, 'index.html')) #cherrypy.expose #cherrypy.tools.json_out() def getData(self): return { 'foo' : 'bar', 'baz' : 'another one' } if __name__ == '__main__': cherrypy.quickstart(App(), '/', config) index.html <!DOCTYPE html> <html> <head> <meta http-equiv='content-type' content='text/html; charset=utf-8'> <title>CCMF</title> <script type='text/javascript' src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js'></script> <script type='text/javascript'> $(document).ready(function() { $('button').on('click', function() { var request = $.ajax({'url': '/getData'}); request.done(function(response) { $('#foo').text(response.foo); $('#baz').text(response.baz); }); request.fail(function(jqXHR, textStatus) { alert('Request failed: ' + textStatus); }); }) }); </script> </head> <body> <button>make ajax call</button> <h1>Foo</h1> <div id='foo'></div> <h1>Baz</h1> <div id='baz'></div> </body> </html>