I'm trying to send a group of .mp4 files first from a web browser client, then to express, then to flask. This code might look simple, but it has taken literally days to get this far, because nobody on the internet has posted an example of how to do this with large files (.mp4 video files), and to be honest, there are a lot of inconsistencies in using these libraries.
I think it might be an encoding error from express to Flask, however. The actual data seems to be being transferred, but ffmpeg doesn't recognize the received .mp4 files as a valid video file.
From the client, I send an XHR request like this:
var formData = new FormData();
for(var x=0; x< datas.videoDatas.length; x++){
// data.videoDatas[x] is a blob of a .mp4 file. it can be played
formData.append(x+"_video.mp4",datas.videoDatas[x]);
}
var xhr = new XMLHttpRequest();
xhr.open("POST", 'http://localhost:3000/uploadVid', true);
xhr.onreadystatechange = function() {
if(this.readyState == XMLHttpRequest.DONE && this.status == 200) {
console.log(xhr.response);
}
}
xhr.onload=console.log;
xhr.send(formData);
The express server at port 3000 receives the post request using the "multer" library. Data is stored into memory as a buffer.
var express = require('express');
var multer=require("multer");
var request=require("request")
var app = express();
var storage = multer.memoryStorage();
const upload = multer({
storage: storage
}).any();
app.post('/uploadVid',function(req,res){
var originalResponse=res;
console.log("begin upload");
upload(req, res, function(err) {
console.log("begin upload to flask server");
var requestStruct={
url: 'http://localhost:3001/receiveUpload2?numFiles='+req.files.length,
method: 'POST',
form: {
}
}
for(var x=0; x < req.files.length; x++){
var testBuffer=req.files[x].buffer; // should be a buffer of the .mp4 file
//testBuffer = new Buffer(10);
//testBuffer=testBuffer.toString();
requestStruct.form[x+'_customBufferFile']= {
value: testBuffer,
options: {
filename: req.files[x].fieldname
}
}
}
request(requestStruct, function(err,res,body){
originalResponse.send(body)
});
});
});
http.createServer(app).listen(3000)
In the python flask server, the .mp4 files are received as a buffer and written to a .mp4 file. However, the .mp4 file is unplayable by ffmpeg "Unknown EBML doctype '(none)'0/0 0_video.mp4: End of file"
from flask import Flask
from flask import request;
app = Flask(__name__);
#app.route('/receiveUpload2', methods=["POST"])
def receiveUpload2():
print("uploaded received....");
numFiles=int(request.args.get("numFiles").encode('ascii', 'ignore'));
print("numFiles",int(numFiles));
for x in range(0,numFiles):
name=request.form.getlist(str(x)+'_customBufferFile[options][filename]');
name=name[0];
print("writing",name);
filebytes=request.form.getlist(str(x)+'_customBufferFile[value]');
filebytes=filebytes[0];
#print(filebytes);
newFileByteArray = bytearray(filebytes,'utf8')
with open("./uploads/"+name,"wb") as output:
output.write(newFileByteArray);
app.run(host="localhost", port=3001);
Related
I have marshalled PIL image into bytes via toBytes() on my Flask Server.
Then these bytes was then embedded in Json as a string and transferred via http to my Angular web application.
Is there a way to turn this binary into a viewable image on my web application? Or is there a better way transfering image from my Flask server to my Angular web application?
A part of my code python flask code:
imgByteArrTemp = io.BytesIO()
img.save(imgByteArrTemp, format="PNG")
# imgByteArrTemp = base64.b64encode(imgByteArrTemp.getvalue())
imgByteArrTemp = imgByteArrTemp.getvalue()
My GET method:
#app.route("/get-resulting-image", methods=["GET"])
def get_resulting_image():
global q
if not q.empty():
item = q.get()
return send_file(io.BytesIO(item),
mimetype = "image/png",
as_attachment=True,
download_name="%s.png")
return "no image to provide"
My typescript app.component.ts:
createImageFromBlob(image:Blob){
let reader = new FileReader();
reader.addEventListener("load", () => {
this.imageBinary = reader.result;
console.log(this.imageBinary);
},false);
if (image){
reader.readAsDataURL(image);
}
}
getResponse(){
...
this.restService.getCorrespondingImage().subscribe(
data1 => {
this.createImageFromBlob(data1);
this.isImageLoading = false;
},
error => {
this.isImageLoading = false;
console.log(error)
}
);
...
}
app.component.html
<img [src]="imageBinary" *ngIf="!isImageLoading">
rest.service.ts
import { Injectable } from '#angular/core';
import {HttpClient} from '#angular/common/http';
import {Observable, throwError} from 'rxjs';
import {catchError, retry} from 'rxjs/operators';
#Injectable({
providedIn: 'root'
})
export class RestService {
constructor(private http: HttpClient) { }
....
getCorrespondingImage():Observable<Blob>{
return this.http.get(
"http://192.168.1.2:9500/get-resulting-image",
{responseType:"blob"});
}
}
Hope this could help someone out there.
I have doc file stored in mongodb. I want to retrieve stored doc file on mongoDB using python. Any suggestion will really be helpful. I'm unable to find article to move forward. All I could find is uploading file through python and retrieving through python. But I want to retrieve file stored on mongoDB which is not created on python but retrieving through python. P.S.: I stored file on mongodb gridfs using express js.
Upload.JS:: JS used to upload file on mongodb gridfs .
const util = require("util");
const multer = require("multer");
const GridFsStorage = require("multer-gridfs-storage");
const mongoose = require('mongoose');
const Grid = require('gridfs-stream');
const mongoURI ='mongodb://localhost:27017/file_uploaded_new';
const promise = mongoose.connect(mongoURI, { useNewUrlParser: true });
const conn = mongoose.connection;
let gfs;
conn.once('open',() => {
gfs = Grid(conn, mongoose.mongo);
gfs.collection('uploads');
});
var storage = new GridFsStorage({
db: promise,
options: {useNewUrlParser: true, useUnifiedTopology: true },
file: (req, file) => {
return new Promise((resolve, reject) => {
if (file.mimetype === 'application/vnd.openxmlformats-
officedocument.wordprocessingml.document') {
resolve({
bucketName: 'words',
filename: `${file.originalname}`
})
} else {
reject(Error("File type has been rejected"));
}
});
}
});
var uploadFile = multer({storage: storage}).single("file");
var uploadFilesMiddleware = util.promisify(uploadFile);
module.exports = uploadFilesMiddleware;
I have successfully uploaded docx file in MongoDB. How can I retrieve the uploaded word file using python script?
How do I send a file to Node.js AND parameter data in POST. I am happy to use any framework. I have attempted it with formidable but I am happy to change.
In my attempt, the file sends, but req.body is empty
Python code to upload:
with open('fileName.txt', 'rb') as f:
payLoad = dict()
payLoad["data"] = "my_data"
r = requests.post('http://xx.xx.xx.xx:8080/sendFile',json = payLoad, files={'fileName.txt': f})
Server Side Node.js:
var express = require('express');
var formidable = require('formidable');
var app = express();
var bodyParser = require('body-parser');
app.use( bodyParser.json() );
app.use(bodyParser.urlencoded({ extended: false }));
app.post('/sendFile', function (req, res){
console.log(req.body )
// req.body is empty
I don't know how to correctly send the file using python, but to receive file with node.js you can use express-fileupload
var fileUpload = require('express-fileupload');
app.use(fileUpload());
app.post('/upload', function(req, res) {
if (!req.files)
return res.status(400).send('No files were uploaded.');
// The name of the input field (i.e. "sampleFile") is used to retrieve the uploaded file
let sampleFile = req.files.sampleFile;
// Use the mv() method to place the file somewhere on your server
sampleFile.mv('/somewhere/on/your/server/filename.jpg', function(err) {
if (err)
return res.status(500).send(err);
res.send('File uploaded!');
});
});
https://www.npmjs.com/package/express-fileupload
I am trying nodejs for the first time. I am using it with python shell. I am trying to transfer a file from one PC to another using Post request
app.js (Server PC)
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.post('/mytestapp', function(req, res) {
console.log(req)
var command = req.body.command;
var parameter = req.body.parameter;
console.log(command + "|" + parameter)
pyshell.send(command + "|" + parameter);
res.send("POST Handler for /create")
});
python file send file from (Client PC)
f = open(filePath, 'rb')
try:
response = requests.post(serverURL, data={'command':'savefile'}, files={os.path.basename(filePath): f})
I use fiddler and the request seems to contain the file on Client PC, but I can't seem to get the file on Server PC. How can I extract and save the file? Is it because I am missing headers? what should I use? thanks
I'm going to guess and say you're using Express based on the syntax in your question. Express doesn't ship with out of the box support for file uploading.
You can use the multer or busboy middleware packages to add multipart upload support.
Its actually pretty easy to do this, here is a sample with multer
const express = require('express')
const bodyParser = require('body-parser')
const multer = require('multer')
const server = express()
const port = process.env.PORT || 1337
// Create a multer upload directory called 'tmp' within your __dirname
const upload = multer({dest: 'tmp'})
server.use(bodyParser.json())
server.use(bodyParser.urlencoded({extended: true}))
// For this route, use the upload.array() middleware function to
// parse the multipart upload and add the files to a req.files array
server.port('/mytestapp', upload.array('files') (req, res) => {
// req.files will now contain an array of files uploaded
console.log(req.files)
})
server.listen(port, () => {
console.log(`Listening on ${port}`)
})
Originally, I was using Windows Phone 8 System.Net.Http library to upload an image from SD card to a Python server with the following code:
private async void UploadFile()
{
try
{
// Make sure there is a picture selected
if (photoStream != null)
{
// initialize the client
// need to make sure the server accepts network IP-based
// requests.
// ensure correct IP and correct port address
var fileUploadUrl = #"http://IPAddress/";
var client = new HttpClient();
// Reset the photoStream position
// If you don't reset the position, the content lenght
// sent will be 0
photoStream.Position = 0;
// This is the postdata
MultipartFormDataContent content = new MultipartFormDataContent();
content.Add(new StreamContent(photoStream), "fn", fileName);
// upload the file sending the form info and ensure a result.
// it will throw an exception if the service doesn't return
// a valid successful status code
await client.PostAsync(fileUploadUrl, content)
.ContinueWith((postTask) =>
{
postTask.Result.EnsureSuccessStatusCode();
});
}
// Disable the Upload button
btnUpload.IsEnabled = false;
// reset the image control
DisplayImage.Source = null;
}
catch
{
}
}
In the server side, I used the following python code to decode the request:
ctype, pdict = cgi.parse_header(self.headers['content-type'])
data = cgi.parse_multipart(self.rfile, pdict)
It worked until I switched from Windows Phone 8 to Windows Phone 8.1 Silverlight. Since the System.Net.Http library is no longer supported in Windows Phone 8.1 Silverlight, I used Windows.Web.Http instead with modification of the code:
private async void UploadFile()
{
try
{
// Make sure there is a picture selected
if (file != null)
{
// initialize the client
// need to make sure the server accepts network IP-based
// requests.
// ensure correct IP and correct port address
var fileUploadUrl = new Uri(#"http://IPAddress/", UriKind.Absolute);
var client = new HttpClient();
HttpMultipartFormDataContent content = new HttpMultipartFormDataContent();
HttpStreamContent streamContent = new HttpStreamContent(photoStream);
content.Add(streamContent, "fn", fileName);
await client.PostAsync(fileUploadUrl, content);
}
// Disable the Upload button
btnUpload.IsEnabled = false;
// reset the image control
DisplayImage.Source = null;
}
catch
{
}
}
The server side is unchanged. However, the resulting file uploaded now has extra bytes inserted periodically that look something like '\r\n10000\r\n' or 0D 0A 31 30 30 30 30 0D 0A. The file also appears to begin and end with the same extra byte pattern. On the server pdict maps the boundary to something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, but this seems to be the same behavior from when the file was uploaded via System.Net.Http.
I tried removing these bytes by hand on a smaller file which seemed to confirm that the file is otherwise correct.
Why are these extra bytes present? Is this a problem with the Python server mishandling the POST request?
Here's my solution when I encountered this issue:
Client:
public async Task UploadImage(byte[] image, string url)
{
Stream stream = new System.IO.MemoryStream(image);
HttpStreamContent streamContent = new HttpStreamContent(stream.AsInputStream());
Uri resourceAddress = null;
Uri.TryCreate(url.Trim(), UriKind.Absolute, out resourceAddress);
Windows.Web.Http.HttpRequestMessage request = new Windows.Web.Http.HttpRequestMessage(Windows.Web.Http.HttpMethod.Post, resourceAddress);
request.Content = streamContent;
var httpClient = new Windows.Web.Http.HttpClient();
var cts = new CancellationTokenSource();
Windows.Web.Http.HttpResponseMessage response = await httpClient.SendRequestAsync(request).AsTask(cts.Token);
}
Controller:
public async Task<HttpResponseMessage> Post()
{
Stream requestStream = await this.Request.Content.ReadAsStreamAsync();
byte[] byteArray = null;
using (MemoryStream ms = new MemoryStream())
{
await requestStream.CopyToAsync(ms);
byteArray = ms.ToArray();
}
.
.
.
return Request.CreateResponse(HttpStatusCode.OK);
}