I am attempting to send images from nodejs to python, use those images for a function, then return a different image back to nodejs. However, I'm having issues with this and getting errors:
Error: aborted
at connResetException (node:internal/errors:692:14)
at TLSSocket.socketCloseListener (node:_http_client:414:19)
at TLSSocket.emit (node:events:539:35)
at TLSSocket.emit (node:domain:475:12)
at node:net:709:12 {
code: 'ECONNRESET'
}
I know it is an issue sending an image itself, because I tried sending text and that worked. To solve it, I used pure trial and error without any luck. Here is the nodejs Code(uses express) currently:
const options = {
method: 'POST',
url: 'pathToPythonstufff',
formData: {
image1: {
value: image1v2,
options: {
filename: 'falsified2.jpg',
contentType: 'image/jpeg'
}
},
image2: {
value: image2v2,
options: {
filename: 'jeff.jpg',
contentType: 'image/jpeg'
}
}
}
};
const pleaseF = request(options, (err, res, body) => {
console.log('sent')
if (err) {
console.log(err);
console.log('err here')
}
console.log(body)
console.log(res.body)
console.log(res.image)
console.log(res.img)
//console.log(res)
/*let img_str = body.img;
// Decode the image
let img_buffer = new Buffer.from(img_str, 'base64');
fs.writeFileSync('difference3.png', img_buffer);*/
});
pleaseF.on('error', (e) => {
console.log(`problem with request: ${e.message}`);
});
And the python code(commented out is the goal, but I commented it out to test and there is no change in nodejs error with or without it commented):
import cv2
import numpy as np
from flask import Flask, request, jsonify, after_this_request
import base64
app = Flask(__name__)
print('hello')
#app.route('/difference', methods=['POST'])
def difference():
# print('hello2')
# # Retrieve image data from the request
# image1 = request.files.get('image1').read()
# image2 = request.files.get('image2').read()
# print('hello again')
# # Decode image data into OpenCV format
# image1 = cv2.imdecode(np.frombuffer(image1, np.uint8), cv2.IMREAD_COLOR)
# image2 = cv2.imdecode(np.frombuffer(image2, np.uint8), cv2.IMREAD_COLOR)
# # Resize images to a common size
# image1 = cv2.resize(image1, (500, 500))
# image2 = cv2.resize(image2, (500, 500))
# # Convert images to grayscale
# image1_gray = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
# image2_gray = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
# # Extract differences between the images
# diff = cv2.absdiff(image1_gray, image2_gray)
# thresh = cv2.threshold(diff, 25, 255, cv2.THRESH_BINARY)[1]
# # Find contours in the thresholded image
# contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# # Draw green boxes around the contours
# for cnt in contours:
# x, y, w, h = cv2.boundingRect(cnt)
# cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
# # Encode the image in PNG format
# _, buffer = cv2.imencode('.png', image1)
# # Create a response object with the image data
# # response = jsonify(img_data=buffer.tobytes())
# # response.headers.set('Content-Type', 'image/png')
# # response2 = jsonify(img_data=buffer)
# # response.headers.set('Content-Type', 'image/png')
# # imageNew = cv2.imdecode(buffer, cv2.IMREAD_COLOR)
# # response = jsonify(img_data=imageNew)
# # response.headers.set('Content-Type', 'image/png')
# # _, buffer = cv2.imencode('.png', image1)
# # Create a response object with the image data
# #response = jsonify(img_data=buffer.tobytes())
# # response.headers.set('Content-Type', 'image/png')
# # return response
# img_str = base64.b64encode(buffer.tobytes()).decode()
# response = jsonify(img_data=img_str)
# response.headers.set('Content-Type', 'image/png')
# return jsonify({'image': response})
#_, welp = cv2.imencode('.jpg', difference)
return "please goddamitt"
if __name__ == '__main__':
app.run(debug=True)
```
And if it isn't that above node error, its a different node error in the form of html(cant seem to reproduce it right now but I got it before with no difference in code). But I do know for sure it has something to do with the images I'm sending.
Related
I am trying to use webcam to collect a photo and resizing it using cv2.resize(). But I am getting this error. I am using google colab. tensorflow==2.4.1 tensorflow-gpu==2.4.1 opencv-python matplotlib. cv2.resize suppose to take numpy array and dimension details as parameter. But everytime I am using it giving me this error. When I comment it out code works fine.
def take_photo(filename='photo.jpg', quality=0.8):
js = Javascript('''
async function takePhoto(quality) {
const div = document.createElement('div');
const capture = document.createElement('button');
capture.textContent = 'Capture';
div.appendChild(capture);
const video = document.createElement('video');
video.style.display = 'block';
const stream = await navigator.mediaDevices.getUserMedia({video: true});
document.body.appendChild(div);
div.appendChild(video);
video.srcObject = stream;
await video.play();
// Resize the output to fit the video element.
google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);
// Wait for Capture to be clicked.
await new Promise((resolve) => capture.onclick = resolve);
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
canvas.getContext('2d').drawImage(video, 0, 0);
stream.getVideoTracks()[0].stop();
div.remove();
return canvas.toDataURL('image/jpeg', quality);
}
''')
display(js)
# get photo data
data = eval_js('takePhoto({})'.format(quality))
# get OpenCV format image
img = js_to_image(data)
# grayscale img
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
print(type(gray))
#print(gray.shape)
# get face bounding box coordinates using Haar Cascade
faces = face_cascade.detectMultiScale(gray)
# draw face bounding box on image
for (x,y,w,h) in faces:
img = cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
# save image
cv2.imwrite(filename, img)
return filename
## js_to_data function
# function to convert the JavaScript object into an OpenCV image
def js_to_image(js_reply):
"""
Params:
js_reply: JavaScript object containing image from webcam
Returns:
img: OpenCV BGR image
"""
# decode base64 image
image_bytes = b64decode(js_reply.split(',')[1])
# convert bytes to numpy array
jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)
#Resize image
jpg_as_np= cv2.resize(jpg_as_np,(250,250))
jpg_as_np.shape
# decode numpy array into OpenCV BGR image
img = cv2.imdecode(jpg_as_np, flags=1) **#error here**
return img
# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream
def bbox_to_bytes(bbox_array):
"""
Params:
bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.
Returns:
bytes: Base64 image byte string
"""
# convert array into PIL image
bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')
iobuf = io.BytesIO()
# format bbox into png for return
bbox_PIL.save(iobuf, format='png')
# format return string
bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))
return bbox_bytes
from numpy.lib import type_check
try:
filename = take_photo()
print(type(filename))
print('Saved to {}'.format(filename))
Error: OpenCV(4.6.0) /io/opencv/modules/imgcodecs/src/loadsave.cpp:818: error: (-215:Assertion failed) buf.checkVector(1, CV_8U) > 0 in function 'imdecode_'
I am working on reactjs with python api and openCV which after uploading photo returns result with green rectangle around the face. So working on it, on clicking upload photo it returns 422(unprocessable entity). I have three main part Upload.js for frontend uploading part, main.py image api and face_detector.py for opencv part.
Upload.js
import React, { useState } from 'react'
import './Upload.css'
import axios from 'axios';
const Upload = () => {
const [file, setFile] = useState();
const handleChange = (event) => {
setFile(URL.createObjectURL(event.target.files[0]))
}
const submitForm = () => {
const formData = new FormData();
formData.append('file', file);
axios
.post('http://127.0.0.1:8000/images', formData, {
headers: {
accept: 'multipart/form-data',
}
})
.then(() => {
alert('file upload succcess');
})
.catch(() => alert("File Upload Error"))
return formData
}
return (
<>
<input className='img_choose' type="file" onChange={handleChange} />
<img src={file} className='prev_img' alt='img' />
<button className='btn_upload' onClick={submitForm}>Upload</button>
</>
);
}
export default Upload
main.py
from fastapi import FastAPI, File, UploadFile
from fastapi.responses import Response
from random import randint
from starlette.requests import Request
import uuid
from fastapi.middleware.cors import CORSMiddleware
app = FastAPI()
db = []
origins = [
"http://localhost:3000",
"http://127.0.0.1:8000/"
]
app.add_middleware(
CORSMiddleware,
allow_origins=origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
#app.get("/main")
def main():
return{"message":"Welcome"}
#app.post("/images/")
async def create_upload_file(file: UploadFile = File(...)):
file.filename = f"{uuid.uuid4()}.jpg"
contents = await file.read() # <-- Important!
db.append(contents)
return {"filename": file.filename}
#app.get("/images/")
async def read_random_file():
# get a random file from the image db
random_index = randint(0, len(db) - 1)
response = Response(content=db[random_index])
return response
Face_detector.py
import cv2
import urllib.request
import numpy as np
url = [
"http://127.0.0.1:8000/images/"
]
def url_to_image(url):
# download the image, convert it to a NumPy array, and then read
# it into OpenCV format
resp = urllib.request.urlopen(url)
image = np.asarray(bytearray(resp.read()), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
# return the image
return image
for url in url:
trained_face_data = cv2.CascadeClassifier(
'haarcascade_frontalface_default.xml')
x = y = w = h = int
image = url_to_image(url)
face_coordinates = trained_face_data.detectMultiScale(image,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30, 30),
flags=cv2.CASCADE_SCALE_IMAGE)
for (x, y, w, h) in face_coordinates:
cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
cv2.imshow("Image", image)
cv2.waitKey(0)
This is where you are doing it wrong.
setFile(URL.createObjectURL(event.target.files[0]))
You are attaching the file URL in the formData instead of the file.
use this instead
setFile(event.target.files[0])
I want to classify a video using opencv methods by using flask, the video to be classified is live-stream that is from the user.
I googled a lot, then finally I found some code and did this:-
from flask import Flask, render_template
from flask_socketio import SocketIO, emit
import time
import io
from PIL import Image
import base64,cv2
import numpy as np
# import pyshine as ps
from flask_cors import CORS,cross_origin
import imutils
# import dlib
from engineio.payload import Payload
# detector = dlib.get_frontal_face_detector()
# predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
Payload.max_decode_packets = 2048
app = Flask(__name__)
socketio = SocketIO(app,cors_allowed_origins='*' )
CORS(app)
#app.route('/', methods=['POST', 'GET'])
def index():
return render_template('index.html')
def readb64(base64_string):
idx = base64_string.find('base64,')
base64_string = base64_string[idx+7:]
sbuf = io.BytesIO()
sbuf.write(base64.b64decode(base64_string, ' /'))
pimg = Image.open(sbuf)
return cv2.cvtColor(np.array(pimg), cv2.COLOR_RGB2BGR)
def moving_average(x):
return np.mean(x)
#socketio.on('catch-frame')
def catch_frame(data):
emit('response_back', data)
global fps,prev_recv_time,cnt,fps_array
fps=30
prev_recv_time = 0
cnt=0
fps_array=[0]
#socketio.on('image')
def image(data_image):
global fps,cnt, prev_recv_time,fps_array
recv_time = time.time()
text = 'FPS: '+str(fps)
frame = (readb64(data_image))
# frame = changeLipstick(frame,[255,0,0])
# frame = ps.putBText(frame,text,text_offset_x=20,text_offset_y=30,vspace=20,hspace=10, font_scale=1.0,background_RGB=(10,20,222),text_RGB=(255,255,255))
frame = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
imgencode = cv2.imencode('.jpeg', frame,[cv2.IMWRITE_JPEG_QUALITY,40])[1]
# base64 encode
stringData = base64.b64encode(imgencode).decode('utf-8')
b64_src = 'data:image/jpeg;base64,'
stringData = b64_src + stringData
# emit the frame back
emit('response_back', stringData)
fps = 1/(recv_time - prev_recv_time)
fps_array.append(fps)
fps = round(moving_average(np.array(fps_array)),1)
prev_recv_time = recv_time
#print(fps_array)
cnt+=1
if cnt==30:
fps_array=[fps]
cnt=0
def getMaskOfLips(img,points):
""" This function will input the lips points and the image
It will return the mask of lips region containing white pixels
"""
mask = np.zeros_like(img)
mask = cv2.fillPoly(mask,[points],(255,255,255))
return mask
def changeLipstick(img,value):
""" This funciton will take img image and lipstick color RGB
Out the image with a changed lip color of the image
"""
img = cv2.resize(img,(0,0),None,1,1)
imgOriginal = img.copy()
imgColorLips=imgOriginal
imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = detector(imgGray)
for face in faces:
x1,y1 = face.left(),face.top()
x2,y2 = face.right(),face.bottom()
facial_landmarks = predictor(imgGray,face)
points =[]
for i in range(68):
x = facial_landmarks.part(i).x
y = facial_landmarks.part(i).y
points.append([x,y])
points = np.array(points)
imgLips = getMaskOfLips(img,points[48:61])
imgColorLips = np.zeros_like(imgLips)
imgColorLips[:] =value[2],value[1],value[0]
imgColorLips = cv2.bitwise_and(imgLips,imgColorLips)
value = 1
value=value//10
if value%2==0:
value+=1
kernel_size = (6+value,6+value) # +1 is to avoid 0
weight = 1
weight = 0.4 + (weight)/400
imgColorLips = cv2.GaussianBlur(imgColorLips,kernel_size,10)
imgColorLips = cv2.addWeighted(imgOriginal,1,imgColorLips,weight,0)
return imgColorLips
if __name__ == '__main__':
socketio.run(app,port=9990 ,debug=True)
This is in main.py of flask
In the templates folder I have made index.html with
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
<style>
#video {
transform: rotateY(180deg);
-webkit-transform:rotateY(180deg); /* Safari and Chrome */
-moz-transform:rotateY(180deg); /* Firefox */
}
</style>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src='https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.0.0/socket.io.js'></script>
</head>
<body>
<div id="container">
<video autoplay playsinline id="videoElement"></video>
<canvas id="canvas" width="400" height="300"></canvas>
</div>
<div class = 'video'>
<img id="photo" width="400" height="300">
<h1>video</h1>
</div>
<script type="text/javascript" charset="utf-8">
var socket = io.connect(window.location.protocol + '//' + document.domain + ':' + location.port);
socket.on('connect', function(){
console.log("Connected...!", socket.connected)
});
var canvas = document.getElementById('canvas');
var context = canvas.getContext('2d');
const video = document.querySelector("#videoElement");
video.width = 400;
video.height = 300;
if (navigator.mediaDevices.getUserMedia) {
navigator.mediaDevices.getUserMedia({ video: true })
.then(function (stream) {
video.srcObject = stream;
video.play();
})
.catch(function (err0r) {
});
}
const FPS = 6;
setInterval(() => {
width=video.width;
height=video.height;
context.drawImage(video, 0, 0, width , height );
var data = canvas.toDataURL('image/jpeg', 0.5);
context.clearRect(0, 0, width,height );
socket.emit('image', data);
}, 1000/FPS);
socket.on('response_back', function(image){
photo.setAttribute('src', image );
});
</script>
</body>
</html>
The problem is: I need to make sure that it works on the user's device, so when I change socketio.run() and add host='0.0.0.0' It stops having access to my webcam
How can I solve this?
also I am unable to show back the video :(
The Error in Console
polling-xhr.js:264 GET http://localhost:9990/socket.io/?EIO=3&transport=polling&t=N_GnQuZ 400 (BAD REQUEST)
Hey,
First you are using a very old socketio client version.
https://cdn.socket.io/4.4.1/socket.io.min.js
use a newer version of socketio client, thats the reason for 400 Error, if you are going to the preview window of the request on dev tools (chrome) you have notice that there are a not supported version message.
Why you dont see the webcam!? is because you have no security context, when you go away from localhost like 0.0.0.0 you need a security context on chrome based browser it means you need HTTPS otherwise "navigator.mediaDevices.getUserMedia" will return undefined.
See this post
I've created a simple Python application that uses the CV2 computer vision library to recognise a template image on a webpage.
I give the application a template image that it needs to recognise on the source image. In this case, the source image is a screenshot of the website www.google.com and the template image is the Google search button.
Template image
I thought the application worked at first, but it's drawing the rectangle completely in the wrong place on the input (source) image. I've added a picture below of where the application located the template image.
Result
Here's the source code.
Main Application Source
import cv2
import numpy
from io import BytesIO
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
class Automate:
def __init__(self):
chrome_options = Options()
chrome_options.add_argument("kiosk")
self.driver = webdriver.Chrome(ChromeDriverManager("93.0.4577.63").install(), options=chrome_options)
#self.driver = webdriver.Chrome(executable_path='./chromedriver',options=chrome_options)
self.screenShot = None
self.finalImage = None
def open_webpage(self, url):
print(f"Open webpage {url}")
self.driver.get(url)
def close_webpage(self):
Event().wait(5)
self.driver.close()
print("Closing webpage")
def snap_screen(self):
print("Capturing screen")
self.screenShot = "screenshot.png"
self.driver.save_screenshot(self.screenShot)
print("done.")
def match(self, image, template):
# convert images to greyscale.
src = cv2.cvtColor(cv2.imread(image), cv2.COLOR_BGR2GRAY)
temp = cv2.cvtColor(cv2.imread(template), cv2.COLOR_BGR2GRAY)
cv2.imshow("out", temp)
cv2.waitKey(0)
height, width = src.shape
H, W = temp.shape
result = cv2.matchTemplate(src, temp, cv2.cv2.TM_CCOEFF_NORMED)
minVal, maxVal, minLoc, maxLoc = cv2.minMaxLoc(result)
location = maxLoc
bottomRight = (location[0] + W, location[1] + H)
src2 = cv2.imread(image)
cv2.rectangle(src2, location, bottomRight, (0, 0, 255), 5)
cv2.imshow("output", src2)
cv2.waitKey(0)
cv2.destroyAllWindows()
def main():
url = "http://www.google.com"
auto = Automate()
auto.open_webpage(url)
auto.snap_screen()
auto.close_webpage()
match_image = "images/templates/google-button.png"
# Match screenshot with template image.
auto.check_match(
image=auto.screenShot,
template=match_image
)
I'd appreciate any help or advice on how to solve this issue.
Update
Following the advice given by user zteffi, I resized my template image to the correct image dimensions. After doing this, the match template function works as expected.
You want to make sure that your template image is a close as possible to the actual size of the image you want to be located in the base image. In my case, this was around 150 x 150 or 200 x 200 so that it will be easier to find the button.
The end goal of the project is to take in a capture of the screen and output circles in the screenshots locations(midpoint and radius). So very start of this project is capturing the screen and sending it through a circle finding function.
I started here: Screen Capture with OpenCV and Python-2.7
This works for its functionality and on my machine does the cv2.imshow sucessfully displays the screenshots as it should. However, I want it to work with this example: https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_houghcircles/py_houghcircles.html
Basically, the code in that example works for cv2.imshow, however, I want it to work with cv2.imread so it's compatible with the example I want to copy.
I've tried a few basic things to no avail, for reference see below!
Attempt 1: http://prntscr.com/n8tpyh
printscreen_pil = Imagegrab()
printscreen_numpy = np.array(printscreen_pil.getdata(),dtype='uint8')
img = cv2.imread(printscreen_humpy , 0)
Errors on cv2.imread with message
TypeError: bad argument type for built-in operation
Attempt 2: http://prntscr.com/n8tqtp
from mss import mss
mon = {'top':160 , 'left': 160 , 'width': 200, 'height': 200 }
sct = mss()
sct.get_pixels(mon)
Errors on sc.getpixels with message AttributeError: 'MSS' object has no attribute `get_pixels'
Attempt 3: https://prnt.sc/n8tw7w
img_grab = ImageGrab.grab(bbox=(0,0,500,500))
img = np.array(img_grab)
#img = cv2.imread(img)
img = cv2.medianBlur(img,5)
cimg = cv2.cvtColor(img,cv2.COLOR_GRAY2BGR)
Errors on cimg = cv2.cvtColor(img,cv2.COLOR_GRAY2BGR) with a very long message which I will summarize into (-2:Unspecified error) and Invalid number of channels in input image: 'VScn::contains(scn)' where 'scn' is 3 for more info please open the screenshot, if not drop a comment and I'll type that chunk for ya!
Attempt 4: http://prntscr.com/n8u0jc
cords = {'top':40 , 'left': 0 , 'width': 800, 'height': 640 }
with mss() as sct :
img = np.array(sct.grab(cords))
img = cv2.medianBlur(img,5)
cimg = cv2.cvtColor(img , cv2.COLOR_GRAY2BGR)
Errors on cimg = cv2.cvtColor(img , cv2.COLOR_GRAY2BGR) with message I will again summarize to Invalid number of channels in input image: `VScn::contains(scn)' where 'scn' is 4
Thanks Everyone!!!!
Based on your attempt 4, use this to define cimg:
cimg = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)