I have a flask application which reads frame from camera and streams it to the website.
Camera.py
from threading import Thread
from copy import deepcopy
import queue
import cv2
class Camera(Thread):
def __init__(self, cam, normalQue, detectedQue):
Thread.__init__(self)
self.__cam = cam
self.__normalQue = normalQue
self.__detectedQue = detectedQue
self.__shouldStop = False
def __del__(self):
self.__cam.release()
print('Camera released')
def run(self):
while True:
rval, frame = self.__cam.read()
if rval:
frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
_, jpeg = cv2.imencode('.jpg', frame)
self.__normalQue.put(jpeg.tobytes())
self.__detectedQue.put(deepcopy(jpeg.tobytes()))
if self.__shouldStop:
break
def stopCamera(self):
self.__shouldStop = True
From what you can see I am just reading the frame, resizing it and storing in two different ques. Nothing too complex.
I also have two two classes responsible for mjpeg stream:
NormalVideoStream.py
from threading import Thread
import traceback
import cv2
class NormalVideoStream(Thread):
def __init__(self, framesQue):
Thread.__init__(self)
self.__frames = framesQue
self.__img = None
def run(self):
while True:
if self.__frames.empty():
continue
self.__img = self.__frames.get()
def gen(self):
while True:
try:
if self.__img is None:
print('Normal stream frame is none')
continue
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
except:
traceback.print_exc()
print('Normal video stream genenation exception')
and
DetectionVideoStream.py
from threading import Thread
import cv2
import traceback
class DetectionVideoStream(Thread):
def __init__(self, framesQue):
Thread.__init__(self)
self.__frames = framesQue
self.__img = None
self.__faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
def run(self):
while True:
if self.__frames.empty():
continue
self.__img = self.__detectFace()
def gen(self):
while True:
try:
if self.__img is None:
print('Detected stream frame is none')
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
except:
traceback.print_exc()
print('Detection video stream genenation exception')
def __detectFace(self):
retImg = None
try:
img = self.__frames.get()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = self.__faceCascade.detectMultiScale(gray, 1.1, 4)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
(_, encodedImage) = cv2.imencode('.jpg', img)
retImg = encodedImage.tobytes()
except:
traceback.print_exc()
print('Face detection exception')
return retImg
From what you can see in both streams I am reading camera frames from ques in infinite loop. Both classes also have gen() method which generates frame to site itself. Only difference is that in detection stream I am also doing face recognition.
Now in my main file:
main.py
from flask import Blueprint, render_template, Response, abort, redirect, url_for
from flask_login import login_required, current_user
from queue import Queue
from . import db
from .Camera import Camera
from .NormalVideoStream import NormalVideoStream
from .DetectionVideoStream import DetectionVideoStream
from .models import User
import cv2
main = Blueprint('main', __name__)
# Queues for both streams
framesNormalQue = Queue(maxsize=0)
framesDetectionQue = Queue(maxsize=0)
print('Queues created')
# RPi camera instance
camera = Camera(cv2.VideoCapture(0), framesNormalQue, framesDetectionQue)
camera.start()
print('Camera thread started')
# Streams
normalStream = NormalVideoStream(framesNormalQue)
detectionStream = DetectionVideoStream(framesDetectionQue)
print('Streams created')
normalStream.start()
print('Normal stream thread started')
detectionStream.start()
print('Detection stream thread started')
#main.route('/')
def index():
return render_template('index.html')
#main.route('/profile', methods=["POST", "GET"])
def profile():
if not current_user.is_authenticated:
abort(403)
return render_template('profile.html', name=current_user.name, id=current_user.id, detectionState=current_user.detectionState)
#main.route('/video_stream/<int:stream_id>')
def video_stream(stream_id):
if not current_user.is_authenticated:
abort(403)
print(f'Current user detection: {current_user.detectionState}')
global detectionStream
global normalStream
stream = None
if current_user.detectionState:
stream = detectionStream
print('Stream set to detection one')
else:
stream = normalStream
print('Stream set to normal one')
return Response(stream.gen(), mimetype='multipart/x-mixed-replace; boundary=frame')
#main.route('/detection')
def detection():
if not current_user.is_authenticated:
abort(403)
if current_user.detectionState:
current_user.detectionState = False
else:
current_user.detectionState = True
user = User.query.filter_by(id=current_user.id)
user.detectionState = current_user.detectionState
db.session.commit()
return redirect(url_for('main.profile', id=current_user.id, user_name=current_user.name))
#main.errorhandler(404)
def page_not_found(e):
return render_template('404.html'), 404
#main.errorhandler(403)
def page_forbidden(e):
return render_template('403.html'), 403
I am creating camera, ques and streams object globally. Also when user logs in on website, he will be able to see live video stream. There is also a button which changes the stream which is currently presented.
Whole project is working well with one exception: when I change stream to detection one it will have huge lag (around 10/15 seconds) which makes whole thing unfunctional. Tried to search a bug/optimalization in my own but can't find anything. On purpose I am running everything on separate threads to unoverload app but it looks like this is not enought. Lag on a level of 1 - 2 seconds will be acceptable, but not 10+. So guys, maybe you can see some bug here? Or know how to optimalize it?
Also need to mention that whole app is running on RPi 4B 4GB and I am accessing website on my desktop. Default server is changed to the Nginx and Gunicorn. From what I can see Pi's CPU usage is 100% when app is working. When testing on default server behaviout is the same. Guess that 1,5 GHz CPU have enough power to run it more smoothly.
One option is using VideoStream
The reason VideoCapture is so slow because the VideoCapture pipeline spends the most time on the reading and decoding the next frame. While the next frame is being read, decode, and returned the OpenCV application is completely blocked.
VideoStream solves the problem by using a queue structure, concurrently read, decode, and return the current frame.
VideoStream supports both PiCamera and webcam.
All you need to is:
Install imutils:
For virtual environment: pip install imutils
For anaconda environment: conda install -c conda-forge imutils
Initialize VideoStream on main.py
import time
from imutils.video import VideoStream
vs = VideoStream(usePiCamera=True).start() # For PiCamera
# vs = VideoStream(usePiCamera=False).start() # For Webcam
camera = Camera(vs, framesNormalQue, framesDetectionQue)
In your Camera.py
In run(self) method:
* ```python
def run(self):
while True:
frame = self.__cam.read()
frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
_, jpeg = cv2.imencode('.jpg', frame)
self.__normalQue.put(jpeg.tobytes())
self.__detectedQue.put(deepcopy(jpeg.tobytes()))
if self.__shouldStop:
break
```
One of the issue which even I had was regrading the encoding and decoding. Like the encoder of Opencv is too slow so try to use encoder from simplejpeg. Use pip3 install simplejpeg and with respect to using cv2.imencode() use simplejpeg.encode_jpeg()
I'm not really surprised about your problem, in general "detection" using a lot of your computation time, because
performing a cascaded classification algorithm is a demanding computational task.
I found a source which compares cascaded classification algos for there performance link
An easy solution, would be to reduce the frame rate, when
processing your detection.
An easy implementation to reduce performance demand could be something like a skip counter e.g.
frameSkipFactor = 3 # use every third frame
frameCounter = 0
if (frameCounter%frameSkipFactor==0):
#process
else:
print("ignore frame", frameCounter)
frameCounter+=1
Nevertheless you will have a lag, because the detection calculation
will produce always a time offset.
I you planning to construct a "real time" classification camera system, please look for another class of classification algos,
which are more designed for this use-case.
I followed an discussion here: real time class algos
Another solution could be using a bigger "hardware hammer" than the rpi e.g. an GPU implementation of the algo via Cuda etc.
What I have found that main reason of getting slow frames is that you have higher resolution and frame rates.
To tackle this problem you can change the resolution to something 640 width by 480 height with fps 30 or less upto 5 fps (if you only need face detection) and can implement resizing of OpenCV (cv2.resize() function) by 0.25 resizing factor of fx and fy. Do this if you don't want higher resolution streams. Works smoothly with opencv. I wanted to try out that VideoStream code (from imutils) as it is also used by Adrian Rosebrock of PyImageSearch. I will use in later projects.
For reference, I am posting code snippet in the following. Special Thanks to Adrian Rosebrock and ageitey face_recognition as their code helped me making it.
class Camera(object):
SHRINK_RATIO = 0.25
FPS = 5
FRAME_RATE = 5
WIDTH = 640
HEIGHT = 480
def __init__(self):
""" initializing camera with settings """
self.cap = cv2.VideoCapture(0)
self.cap.set(3,Camera.WIDTH)
self.cap.set(4,Camera.HEIGHT)
self.cap.set(5,Camera.FPS)
self.cap.set(7,Camera.FRAME_RATE)
def get_frame(self):
""" get frames from the camera """
success, frame = self.cap.read()
if success == True:
# Resizing to 0.25 scale
rescale_frame = cv2.resize(frame, None, fx= Camera.SHRINK_RATIO, fy=Camera.SHRINK_RATIO)
cascPath = "haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascPath)
gray = cv2.cvtColor(rescale_frame, cv2.COLOR_BGR2GRAY)
# detecting faces
faces = faceCascade.detectMultiScale(
gray,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30,30)
)
# Draw a rectangle around the faces
if len(faces) != 0:
for (x, y, w, h) in faces:
x *=4
y *=4
w *=4
h *=4
# Draw rectangle across faces in current frame
cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
# return frame outside if-statement
return frame
Also keep in mind that JPEG is the accelerated codec, use:
cv2.imencode(".jpg",frame)[1].tobytes()
Related
I'm pretty new to opencv and I've been trying to build an automatic stat tracker for a game I'm playing. I want to collect videos to test the object recognition and stat tracking code, and thought I might as well use the capture code I've already written to make the test videos. So I've been using the pywin32 api to make images for opencv to process, and thought I'd just directly use those images to save to video.
The problem I'm having is that the videos being made are invalid and useless.
So now that I've gone through the background I can move on to the actual questions.
Is there a specific format the opencv images need to be in before they can be saved? or is there a way to possibly feed the images through the VideoCapture class in a non clunky way (like recapturing the cv.imshow window)? I don't want to directly use the VideoCapture class, because it requires the specified window to be always on top. I might also just be doing it wrong.
Code included below.
class VideoScreenCapture:
frame_time_limit = 0
time_elapsed = 0
frame_width = None
frame_height = None
frame_size = None
fps = None
window_capture = None
filming = False
output_video = None
scaling_factor = 2/3
def __init__(self, window_capture=None, fps=5):
"""
window_capture: a custom window capture object that is associated with the window to be captured
fps: the frames per second of the resulting video
"""
self.fps = fps
# calculate the amount of time per frame (inverse of fps)
self.frame_time_limit = 1/self.fps
self.window_capture = window_capture
# get the size (in pixels) of the window getting captured
self.frame_width, self.frame_height, self.frame_size = self.window_capture.get_frame_dimentions()
# initialize video writer
self.output_video = cv.VideoWriter(
f'assets{sep}Videos{sep}test.avi',
cv.VideoWriter_fourcc(*'XVID'),
self.fps,
self.frame_size)
def update(self, dt):
"""dt: delta time -> the time since the last update"""
# only updates if the camera is on
if self.filming:
# update the elapsed time
self.time_elapsed += dt
# if the elapsed time is more than the time segment calculated for the given fps
if self.time_elapsed >= self.frame_time_limit:
self.time_elapsed -= self.frame_time_limit
# window capture takes screenshot of the capture window and resizes it to the wanted side
frame = self.window_capture.get_screenshot()
frame = cv.resize(
frame,
dsize=(0, 0),
fx=self.scaling_factor,
fy=self.scaling_factor)
# the image is then saved to video
self.output_video.write(frame)
# show the image in a separate window
cv.imshow("Computer Vision", frame)
"""methods for stopping and starting the video capture"""
def start(self):
self.filming = True
def stop(self):
self.filming = False
"""releases video writer: use when done"""
def release_vw(self):
self.output_video.release()
This while loop that runs the code
vsc.start()
loop_time = time()
while cv.waitKey(1) != ord('q'):
vsc.update(time() - loop_time)
loop_time = time()
I have been trying to solve this one for quite a while now and cannot figure it out. Would appreciate some help with it. So I have a FastAPI server in which I have deployed a Drowsiness Detection Model/Script (dlib, opencv2, scipy). Now what I am trying to achieve is - Start and stop the DDM via API Endpoints. So the problem is - the uvicorn server is single-threaded, so when I run the DDM it will run in the same thread and when I try to stop the DDM it stops the entire server process (which is not something I want). I have tried forking the process and running the DDM on that process but it gives an error and crashes. I think using multithreading might help, I am not sure. Also if it does help me solve my issue I don't know how exactly to approach it. Relevant Code :
# Drowsiness Detection Script
def eye_aspect_ratio(eye):
A = distance.euclidean(eye[1], eye[5])
B = distance.euclidean(eye[2], eye[4])
C = distance.euclidean(eye[0], eye[3])
ear = (A + B) / (2.0 * C)
return ear
def detect_drowsiness(monitor: bool):
pid_file = open("intelligence/drowsiness_detection/dataset/pid.txt", "w")
pid_str = str(os.getpid())
pid_file.write(pid_str)
pid_file.close()
thresh = 0.25
frame_check = 18
detect = dlib.get_frontal_face_detector()
# Dat file is the crux of the code
predict = dlib.shape_predictor(
"intelligence/drowsiness_detection/dataset/shape_predictor_68_face_landmarks.dat")
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["right_eye"]
cap = cv2.VideoCapture(0)
flag = 0
while monitor:
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
subjects = detect(gray, 0)
for subject in subjects:
shape = predict(gray, subject)
shape = face_utils.shape_to_np(
shape) # converting to NumPy Array
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
ear = (leftEAR + rightEAR) / 2.0
if ear < thresh:
flag += 1
print("Detecting,{}".format(flag))
if flag >= frame_check:
print("ALERT - Drowsy")
else:
flag = 0
cap.release()
# Drowsiness detection for a user
# router.get("/face/drowsy/start", response_description="Drowsiness monitoring for the user")
async def start_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(detect_drowsiness, True)
return("Drowsiness monitoring ON")
# router.get("/face/drowsy/stop", response_description="Drowsiness monitoring for the user")
async def stop_drowsiness_detection():
pid_file_path = f"intelligence/drowsiness_detection/dataset/pid.txt"
pid_file = open(pid_file_path, "r")
if not os.path.exists(pid_file_path):
return("Please start monitoring first")
pid_str = pid_file.read()
remove_file(pid_file_path)
os.kill(int(pid_str), signal.SIGKILL)
return("Drowsiness monitoring OFF")
Possible workaround :
# Drowsiness Detection Script
def eye_aspect_ratio(eye):
A = distance.euclidean(eye[1], eye[5])
B = distance.euclidean(eye[2], eye[4])
C = distance.euclidean(eye[0], eye[3])
ear = (A + B) / (2.0 * C)
return ear
class DrowsinessDetector(Process):
running = Event()
def stop_monitoring(self):
if self.running.is_set():
self.running.clear()
def start_monitoring(self):
if self.running.is_set():
return
self.running.set()
self.detect_drowsiness()
def detect_drowsiness(self):
thresh = 0.25
frame_check = 18
detect = dlib.get_frontal_face_detector()
# Dat file is the crux of the code
predict = dlib.shape_predictor("./shape_predictor_68_face_landmarks.dat")
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["right_eye"]
cap = cv2.VideoCapture(0)
flag = 0
while self.running.is_set():
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
subjects = detect(gray, 0)
for subject in subjects:
shape = predict(gray, subject)
shape = face_utils.shape_to_np(shape) # converting to NumPy Array
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
ear = (leftEAR + rightEAR) / 2.0
if ear < thresh:
flag += 1
print("Detecting - {}".format(flag))
if flag >= frame_check:
print("ALERT - Drowsy")
else:
flag = 0
cap.release()
# Drowsiness detection for a user
drowsy = DrowsinessDetector()
#router.get("/face/drowsy/start", response_description="Drowsiness monitoring for the user")
async def start_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(drowsy.start_monitoring())
return "Drowsiness monitoring ON"
#router.get("/face/drowsy/stop", response_description="Drowsiness monitoring for the user")
async def stop_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(drowsy.stop_monitoring())
return "Drowsiness monitoring OFF"
I got this solution from Reddit but for some reason, it doesn't work. Any help will be much appreciated.
You can also just put your non async code into a standard sync route def. (This is actually the encouraged approach from FastAPI) FastAPI will then run that code in an external threadpool and manage it all for you. From there you could simply check the status of literally anything(files, redis, inMem dict, pub/sub) from within your while loop to stop the drowsiness detector.
https://fastapi.tiangolo.com/async/#path-operation-functions
While not explicitly mentioned in the FastAPI documentation, BackgroundTasks.background_tasks will create a new thread on the same process.
Using the first code you posted - when you store the PID (process ID) into a file in the detect_drowsiness() function, and then kill the process on stop_drowsiness_detection() route/function, you are effectively killing the very process that is running FastAPI.
In the background tasks section of FastAPI, in caveat, they mention:
If you need to perform heavy background computation and you don't necessarily need it to be run by the same process (for example, you don't need to share memory, variables, etc), you might benefit from using other bigger tools like Celery.
Relating to the second code you posted, the use of multiprocessing there seems to be towards the right direction. Without more details on why that specific implementation is not working, it's hard to help you further.
I've been working on this project to detect qr-code and make attendance and have the outputs in sql database and so far I've gotten everything to work out,
The Problem is I actually want the frame to close when a single QR code is detected, decoded and all of the above process is completed also The camera works and the frame shows up but it is very slow when detecting the QR code.
What actually happens is after the frame is open it constantly and after it detects a QR code, it actually logs multiple (around 15) instances of a QR being detected after like 2 seconds. Also the frame is still up and I can still detect Images.
The frame closes, only after pressing the waitkey which is 27 or 'Esc'
So I am actually looking for 2 Things:
How to close the frame after detecting a QR Code?
(additional question) how do I detect only one qr code (whether the frame closes or not). So when I scan the QR code, either the frame closes and I am left with one decoded data OR I scan the QR code and the frame remains open, until I hit 'Esc' and I am left with one decoded data.
here is the full code for reference:
import cv2
import os
import csv
import sqlite3
import vobject
from datetime import datetime
import numpy as np
from pyzbar import pyzbar
path = 'qr-codes'
images = []
classNames = []
lists = os.listdir(path)
for cl in lists:
curImage = cv2.imread(f'{path}/{cl}')
images.append(curImage)
classNames.append(os.path.splitext(cl)[0])
def main():
camera = cv2.VideoCapture(0)
ret, frame = camera.read()
while ret:
ret, frame = camera.read()
recognizer = parse_vcard(frame)
win_name = 'Face and QR Detection'
cv2.imshow(win_name, recognizer)
if cv2.waitKey(1) & 0xFF == 27:
break
camera.release()
cv2.destroyAllWindows()
def parse_vcard(frame):
barcodes = pyzbar.decode(frame)
for barcode in barcodes:
x, y, w, h = barcode.rect
cv2.rectangle(frame, (x, y),(x+w, y+h), (0, 255, 0), 2)
mydata = barcode.data.decode('utf-8')
vcard = vobject.readOne(mydata)
make_attendence(vcard)
return frame
def make_attendence(vcard):
name = str(vcard.contents['n'][0].value)
# print(type(Employee.name))
profession = str(vcard.contents['title'][0].value)
now = datetime.now()
date = datetime.date(now)
attendance_time = now.strftime('%H:%M:%S')
leave_time = '21:00:00'
connection = sqlite3.connect('employee.db')
conn = connection.cursor()
# conn.execute("""CREATE TABLE employees (
# name text,
# profission text,
# date date,
# attendance_time time,
# leave_time time
# )""")
conn.execute("INSERT INTO employees VALUES \
(:name, :profession, :date, :attendance_time, :leave_time)", {
'name': name,
'profession': profession,
'date': date,
'attendance_time': attendance_time,
'leave_time': leave_time})
connection.commit()
connection.close()
if __name__ == '__main__':
main()
do not reconnect to the database for every QR code and every picture you take.
connect once. keep the connection.
the imshow window is a window. a "frame" is commonly the term for one picture/image from a video. you can't "close" a frame.
you can close imshow windows with destroyAllWindows or individual imshow windows using destroyWindow
use python's "pdb" debugger. use a profiler to investigate the time cost of your code. use time.perf_counter() and do the measurements yourself, if you don't find a profiler you like.
I have face_recognition python script, and running fine if executed, but it randomly show error if execute:
Traceback (most recent call last):
File "faces_test.py", line 38, in <module>
rgb_frame = frame[:, :, ::-1]
TypeError: 'NoneType' object is not subscriptable
That error show randomly sometimes running sometimes got that error. I don't understand what's happen, but I change face_recognition tolerance at api.py from 0.6 to 0.4 before. I'm not sure that chage make random error to opencv
I want to running my script and don't get any random error like that, any solution?
environment version:
python = 3.8.2
opencv-contrib-python 4.2.0.32
opencv-python 4.2.0.32
face-recognition 1.3.0
face-recognition-models 0.3.0
My python script:
import face_recognition
import cv2
import numpy as np
# This is a super simple (but slow) example of running face recognition on live video from your webcam.
# There's a second example that's a little more complicated but runs faster.
# PLEASE NOTE: This example requires OpenCV (the `cv2` library) to be installed only to read from your webcam.
# OpenCV is *not* required to use the face_recognition library. It's only required if you want to run this
# specific demo. If you have trouble installing it, try any of the other demos that don't require it instead.
# Get a reference to webcam #0 (the default one)
video_capture = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
# Load a sample picture and learn how to recognize it.
obama_image = face_recognition.load_image_file("./training/obama.jpg")
obama_face_encoding = face_recognition.face_encodings(obama_image)[0]
# Load a second sample picture and learn how to recognize it.
biden_image = face_recognition.load_image_file("./training/biden.jpg")
biden_face_encoding = face_recognition.face_encodings(biden_image)[0]
# Create arrays of known face encodings and their names
known_face_encodings = [
obama_face_encoding,
biden_face_encoding
]
known_face_names = [
"obama",
"biden"
]
while True:
# Grab a single frame of video
ret, frame = video_capture.read()
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_frame = frame[:, :, ::-1]
# Find all the faces and face enqcodings in the frame of video
face_locations = face_recognition.face_locations(rgb_frame)
face_encodings = face_recognition.face_encodings(rgb_frame, face_locations)
# Loop through each face in this frame of video
for (top, right, bottom, left), face_encoding in zip(face_locations, face_encodings):
# See if the face is a match for the known face(s)
matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
name = "Unknown"
# If a match was found in known_face_encodings, just use the first one.
# if True in matches:
# first_match_index = matches.index(True)
# name = known_face_names[first_match_index]
# Or instead, use the known face with the smallest distance to the new face
face_distances = face_recognition.face_distance(known_face_encodings, face_encoding)
best_match_index = np.argmin(face_distances)
#print(face_distances)
#print(best_match_index)
if matches[best_match_index]:
name = known_face_names[best_match_index]
print(name)
# Draw a box around the face
cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2)
# Draw a label with a name below the face
cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (0, 0, 255), cv2.FILLED)
font = cv2.FONT_HERSHEY_DUPLEX
cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1)
# Display the resulting image
cv2.imshow('Video', frame)
# Hit 'q' on the keyboard to quit!
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Release handle to the webcam
video_capture.release()
cv2.destroyAllWindows()
sorry for the late response, but if you're still working through coding with OpenCV maybe this will help.
First Comment
You are probably not getting any video coming through the webcam feed.
For example, look at this section of code
# Get a reference to webcam #0 (the default one)
video_capture = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
Sometimes the camera will initialize in slot '1' instead of '0'. Its important to take into consideration that these initializations can bounce around. If you want to ensure that the camera is being recognized correctly, you can always explicitly define your camera location in a variable and also do some error handling.
Example for problem one:
def setup_camera():
"""
Initialize camera by calling cv2.VideoCapture
Produces a variable that you can read from using self.capture.read
If camera doesn't connect prints error and Camera not conntected
"""
device = '/dev/v4l/by-id/usb-HD_Camera_Manufacturer_USB_2.0_Camera-video-index0'
try:
capture = cv2.VideoCapture(device)
print("Camera Connection Successful")
self.capture.release()
except IOError as e:
print(e)
print("Camera Not Connected")
This will at least get you started and will tell you if the camera feed is not recording.
If you want to be able to find Cameras easily on your system (and you're using linux - I'm an ubuntu guy so sorry if you are windows) you can run a simple package like v4l-utils to track down the cameras. Something like the following code:
sudo apt-get install v4l-utils
v4l2-ctl --list-devices
Should give you a response like the following:
USB 2.0 Camera: HD USB Camera (usb-0000:00:1a.0-1.5):
/dev/video0
You can then run a command with udevadm to get more info on the /dev/video0:
udevadm info --query=all --name=/dev/video0
Which will return something like the following:
P: /devices/pci0000:00/0000:00:1a.0/usb1/1-1/1-1.5/1-1.5:1.0/video4linux/video0
N: video0
S: v4l/by-id/usb-HD_Camera_Manufacturer_USB_2.0_Camera-video-index0
S: v4l/by-path/pci-0000:00:1a.0-usb-0:1.5:1.0-video-index0
E: COLORD_DEVICE=1
E: COLORD_KIND=camera
E: DEVLINKS=/dev/v4l/by-path/pci-0000:00:1a.0-usb-0:1.5:1.0-video-index0 /dev/v4l/by-id/usb-HD_Camera_Manufacturer_USB_2.0_Camera-video-index0
E: DEVNAME=/dev/video0
E: DEVPATH=/devices/pci0000:00/0000:00:1a.0/usb1/1-1/1-1.5/1-1.5:1.0/video4linux/video0
E: ID_BUS=usb
E: ID_FOR_SEAT=video4linux-pci-0000_00_1a_0-usb-0_1_5_1_0
E: ID_MODEL=USB_2.0_Camera
E: ID_MODEL_ENC=USB\x202.0\x20Camera
E: ID_MODEL_ID=9230
E: ID_PATH=pci-0000:00:1a.0-usb-0:1.5:1.0
E: ID_PATH_TAG=pci-0000_00_1a_0-usb-0_1_5_1_0
E: ID_REVISION=0100
E: ID_SERIAL=HD_Camera_Manufacturer_USB_2.0_Camera
E: ID_TYPE=video
E: ID_USB_DRIVER=uvcvideo
E: ID_USB_INTERFACES=:0e0100:0e0200:
E: ID_USB_INTERFACE_NUM=00
E: ID_V4L_CAPABILITIES=:capture:
E: ID_V4L_PRODUCT=USB 2.0 Camera: HD USB Camera
E: ID_V4L_VERSION=2
E: ID_VENDOR=HD_Camera_Manufacturer
E: ID_VENDOR_ENC=HD\x20Camera\x20Manufacturer
E: ID_VENDOR_ID=05a3
E: MAJOR=81
E: MINOR=0
E: SUBSYSTEM=video4linux
E: TAGS=:uaccess:seat:
E: USEC_INITIALIZED=23692957
As you can see, the second value in DEVLINKS provides a hardware location that you can use to call your camera.
Second Comment
OpenCV has built in functions to do conversions on frames. Your method and the example I'll use are both in this post. In order to look more into things, lets give this a try.
For Example -
The method of code you are using is fine:
# Convert the image from BGR color (which OpenCV uses) to RGB color (which face_recognition uses)
rgb_frame = frame[:, :, ::-1]
Try using this example for problem two if it does not work:
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
Conclusions
I hope that this response helped. I don't think its a conversion problem but most likely a camera initialization issue. Try getting a more accurate camera feed location and this will provide you with a more robust way to call the camera. Have a nice day!
The main part is to test if the object is None as seen in the comment 'The error message says that video_capture.read() returned None'
video_capture.read()
If the type is none, just break out of the for loop as shown below.
for i in range(1,length-1):
ret, frame = video_capture.read()
print(type(frame))
if frame is None:
print('Noneeeee')
break
A entire code sample is shown below. The reference for the entire code is - https://expoleet.medium.com/flask-opencv-face-recognition-b9cbc3d1d280
import os
import sys
from flask import Flask, request, redirect, url_for, render_template, flash, jsonify
from werkzeug.utils import secure_filename
import face_recognition
import cv2
import numpy as np
from flask_cors import CORS, cross_origin
UPLOAD_FOLDER = '/root/backendOne/pyflaskone/upload'
ALLOWED_EXTENSIONS = set(['txt','mp4'])
app = Flask(__name__)
CORS(app, support_credentials=True)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.secret_key = "secret key"
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS
NEWPATH=""
#app.route("/", methods=['GET', 'POST'])
def index():
if request.method == 'POST':
print('request recieved....')
image_file = request.files['video']
print(image_file)
filename = "aaa.mp4"
image_file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
print('file savedd.....', filename)
NEWPATH=videotest(os.path.join(app.config['UPLOAD_FOLDER'], filename))
if(NEWPATH == "Unknown"):
flash('No matches for attendance')
else:
flash('File successfully uploaded', NEWPATH)
print(NEWPATH,'NEWPATH....')
return {"status": "Success", "userName": NEWPATH}
#else
if request.method == 'GET':
return """
<!doctype html>
<title>Upload new File</title>
<h1>Upload new File</h1>
<form action="" method=post enctype=multipart/form-data>
<p><input type=file name=file>
<input type=submit value=Upload>
</form>
<p>%s</p>
""" % "<br>".join(os.listdir(app.config['UPLOAD_FOLDER'],))
def videotest(filename):
video_capture = cv2.VideoCapture(filename)
length = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
fps = int(video_capture.get(cv2.CAP_PROP_FPS))
harmesh = face_recognition.load_image_file("harmesh.jpg")
hfencoding = face_recognition.face_encodings(harmesh)[0]
prateek = face_recognition.load_image_file("prateek.jpeg")
pfencoding = face_recognition.face_encodings(prateek)[0]
krishna = face_recognition.load_image_file("BusinessCards.jpg")
kfencoding = face_recognition.face_encodings(krishna)[0]
balaji = face_recognition.load_image_file("Balaji-Photo.jpg")
bfencoding = face_recognition.face_encodings(balaji)[0]
known_face_encodings = [
hfencoding,
pfencoding,
kfencoding,
bfencoding
]
known_face_names = [
"Harmesh",
"Prateek",
"Krrrr",
"Balaji"
]
name = "Unknown"
width = int(video_capture.get(3)) # float
height = int(video_capture.get(4))
for i in range(1,length-1):
ret, frame = video_capture.read()
print(type(frame))
if frame is None:
print('Noneeeee')
break
rgb_frame = frame[:, :, ::-1]
face_locations = face_recognition.face_locations(rgb_frame)
face_encodings = face_recognition.face_encodings(rgb_frame, face_locations)
for (top, right, bottom, left), face_encoding in zip(face_locations, face_encodings):
matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
face_distances = face_recognition.face_distance(known_face_encodings, face_encoding)
best_match_index = np.argmin(face_distances)
print(best_match_index)
if matches[best_match_index]:
name = known_face_names[best_match_index]
print('if matchesss', name)
video_capture.release()
#out.release()
cv2.destroyAllWindows()
print('final name', name)
return name
if __name__ == "__main__":
app.run(host ='0.0.0.0')
I am trying to make an application with a flask that
Captures a video stream from a webcam
Applies an object detection algorithm on each frame
Concurrently displays the frames with bounding boxes and data provided by the above function in the form of a video
The problem is that due to a function operation, the result renders with a bit of lag. To overcome this I tried to run the function over another thread.
Can you please help in running the function over each frame of the video and displaying the resultant frames in the form of a video?
My main function looks somewhat like :
def detect_barcode():
global vs, outputFrame, lock
total = 0
# loop over frames from the video stream
while True:
frame = vs.read()
frame = imutils.resize(frame, width=400)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)
timestamp = datetime.datetime.now()
cv2.putText(frame, timestamp.strftime(
"%A %d %B %Y %I:%M:%S%p"), (10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)
if total > frameCount:
# detect barcodes in the image
barcodes = md.detect(gray)
# check to see if barcode was found
if barcode in barcodes:
x,y,w,h = cv2.boundRect(barcode)
cv2.rectangle(frame,x,y,(x+w),(y+h),(255,0,0),2)
total += 1
# lock
with lock:
outputFrame = frame.copy()
if __name__ == '__main__':
# start a thread that will perform motion detection
t = threading.Thread(target=detect_barcode)
t.daemon = True
t.start()
app.run(debug=True)
vs.stop()
I literally just did something like this. My solution was asyncio's concurrent.futures module.
import concurrent.futures as futures
Basically, you create your executor and 'submit' your blocking method as a parameter. It returns something called Future, which you can check the result of (if any) every frame.
executor = futures.ThreadPoolExecutor()
would initialize the executor
future_obj = executor.submit(YOUR_FUNC, *args)
would start execution. Sometime later on you could check its status
if future_obj.done():
print(future_obj.result())
else:
print("task running")
# come back next frame and check again