I have been trying to solve this one for quite a while now and cannot figure it out. Would appreciate some help with it. So I have a FastAPI server in which I have deployed a Drowsiness Detection Model/Script (dlib, opencv2, scipy). Now what I am trying to achieve is - Start and stop the DDM via API Endpoints. So the problem is - the uvicorn server is single-threaded, so when I run the DDM it will run in the same thread and when I try to stop the DDM it stops the entire server process (which is not something I want). I have tried forking the process and running the DDM on that process but it gives an error and crashes. I think using multithreading might help, I am not sure. Also if it does help me solve my issue I don't know how exactly to approach it. Relevant Code :
# Drowsiness Detection Script
def eye_aspect_ratio(eye):
A = distance.euclidean(eye[1], eye[5])
B = distance.euclidean(eye[2], eye[4])
C = distance.euclidean(eye[0], eye[3])
ear = (A + B) / (2.0 * C)
return ear
def detect_drowsiness(monitor: bool):
pid_file = open("intelligence/drowsiness_detection/dataset/pid.txt", "w")
pid_str = str(os.getpid())
pid_file.write(pid_str)
pid_file.close()
thresh = 0.25
frame_check = 18
detect = dlib.get_frontal_face_detector()
# Dat file is the crux of the code
predict = dlib.shape_predictor(
"intelligence/drowsiness_detection/dataset/shape_predictor_68_face_landmarks.dat")
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["right_eye"]
cap = cv2.VideoCapture(0)
flag = 0
while monitor:
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
subjects = detect(gray, 0)
for subject in subjects:
shape = predict(gray, subject)
shape = face_utils.shape_to_np(
shape) # converting to NumPy Array
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
ear = (leftEAR + rightEAR) / 2.0
if ear < thresh:
flag += 1
print("Detecting,{}".format(flag))
if flag >= frame_check:
print("ALERT - Drowsy")
else:
flag = 0
cap.release()
# Drowsiness detection for a user
# router.get("/face/drowsy/start", response_description="Drowsiness monitoring for the user")
async def start_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(detect_drowsiness, True)
return("Drowsiness monitoring ON")
# router.get("/face/drowsy/stop", response_description="Drowsiness monitoring for the user")
async def stop_drowsiness_detection():
pid_file_path = f"intelligence/drowsiness_detection/dataset/pid.txt"
pid_file = open(pid_file_path, "r")
if not os.path.exists(pid_file_path):
return("Please start monitoring first")
pid_str = pid_file.read()
remove_file(pid_file_path)
os.kill(int(pid_str), signal.SIGKILL)
return("Drowsiness monitoring OFF")
Possible workaround :
# Drowsiness Detection Script
def eye_aspect_ratio(eye):
A = distance.euclidean(eye[1], eye[5])
B = distance.euclidean(eye[2], eye[4])
C = distance.euclidean(eye[0], eye[3])
ear = (A + B) / (2.0 * C)
return ear
class DrowsinessDetector(Process):
running = Event()
def stop_monitoring(self):
if self.running.is_set():
self.running.clear()
def start_monitoring(self):
if self.running.is_set():
return
self.running.set()
self.detect_drowsiness()
def detect_drowsiness(self):
thresh = 0.25
frame_check = 18
detect = dlib.get_frontal_face_detector()
# Dat file is the crux of the code
predict = dlib.shape_predictor("./shape_predictor_68_face_landmarks.dat")
(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_68_IDXS["right_eye"]
cap = cv2.VideoCapture(0)
flag = 0
while self.running.is_set():
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
subjects = detect(gray, 0)
for subject in subjects:
shape = predict(gray, subject)
shape = face_utils.shape_to_np(shape) # converting to NumPy Array
leftEye = shape[lStart:lEnd]
rightEye = shape[rStart:rEnd]
leftEAR = eye_aspect_ratio(leftEye)
rightEAR = eye_aspect_ratio(rightEye)
ear = (leftEAR + rightEAR) / 2.0
if ear < thresh:
flag += 1
print("Detecting - {}".format(flag))
if flag >= frame_check:
print("ALERT - Drowsy")
else:
flag = 0
cap.release()
# Drowsiness detection for a user
drowsy = DrowsinessDetector()
#router.get("/face/drowsy/start", response_description="Drowsiness monitoring for the user")
async def start_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(drowsy.start_monitoring())
return "Drowsiness monitoring ON"
#router.get("/face/drowsy/stop", response_description="Drowsiness monitoring for the user")
async def stop_drowsiness_detection(background_tasks: BackgroundTasks):
background_tasks.add_task(drowsy.stop_monitoring())
return "Drowsiness monitoring OFF"
I got this solution from Reddit but for some reason, it doesn't work. Any help will be much appreciated.
You can also just put your non async code into a standard sync route def. (This is actually the encouraged approach from FastAPI) FastAPI will then run that code in an external threadpool and manage it all for you. From there you could simply check the status of literally anything(files, redis, inMem dict, pub/sub) from within your while loop to stop the drowsiness detector.
https://fastapi.tiangolo.com/async/#path-operation-functions
While not explicitly mentioned in the FastAPI documentation, BackgroundTasks.background_tasks will create a new thread on the same process.
Using the first code you posted - when you store the PID (process ID) into a file in the detect_drowsiness() function, and then kill the process on stop_drowsiness_detection() route/function, you are effectively killing the very process that is running FastAPI.
In the background tasks section of FastAPI, in caveat, they mention:
If you need to perform heavy background computation and you don't necessarily need it to be run by the same process (for example, you don't need to share memory, variables, etc), you might benefit from using other bigger tools like Celery.
Relating to the second code you posted, the use of multiprocessing there seems to be towards the right direction. Without more details on why that specific implementation is not working, it's hard to help you further.
Related
I was actually trying to modify some yolov5 script. Here I'm trying to pass an array between threads.
def detection(out_q):
while(cam.isOpened()):
ref, img = cam.read()
img = cv2.resize(img, (640, 320))
result = model(img)
yoloBbox = result.xywh[0].numpy() # yolo format
bbox = result.xyxy[0].numpy() # pascal format
for i in bbox:
out_q.put(i) # 'i' is the List of length 6
def resultant(in_q):
while(cam.isOpened()):
ref, img =cam.read()
img = cv2.resize(img, (640, 320))
qbbox = in_q.get()
print(qbbox)
if __name__=='__main__':
q = Queue(maxsize = 10)
t1 = threading.Thread(target= detection, args = (q, ))
t2 = threading.Thread(target= resultant, args = (q, ))
t1.start()
t2.start()
t1.join()
t2.join()
I tried with this but it's giving me errors like:
Assertion fctx->async_lock failed at libavcodec/pthread_frame.c:155
so is there any other method to pass the array?
any kind of tutorial/ solution is appreciated. If there is any misunderstanding with my question, please let me know.
Thanks a lot!!
Update:::
I was trying like this..
def detection(ns, event):#
## a = np.array([1, 2, 3]) -
#### a= list(a) | #This is working
## ns.value = a |
## event.set() -
while(cam.isOpened()):
ref, img = cam.read()
img = cv2.resize(img, (640, 320))
result = model(img)
yoloBbox = result.xywh[0].numpy() # yolo format
bbox = result.xyxy[0].numpy() # pascal format
for i in bbox:
arr = np.squeeze(np.array(i))
print("bef: ", arr) -
ns.value = arr | # This is not working
event.set() -
def transfer(ns, event):
event.wait()
print(ns.value)
if __name__=='__main__':
## detection()
manager = multiprocessing.Manager()
namespace = manager.Namespace()
event=multiprocessing.Event()
p1 = multiprocessing.Process(target=detection, args=
(namespace, event),)
p2= multiprocessing.Process(target=transfer, args=(namespace,
event),)
p1.start()
p2.start()
p1.join()
p2.join()
The output from the above "arr" = [ 0 1.8232
407.98 316.46 0.92648 0]
but all I got is blank. no error, no warning, only blank.
I tested arr is having value.
I tested the list, np array all are shareing data which is marked as working.
But why that the data from "arr" array is blank (after sharing)
so what should I do?
so is there any other method to pass the array?
Yes, you could use multiprocessing.shared_memory, it is part of standard library since python3.8, and PyPI has backport allowing to use it in python3.6 and python3.7. See example in linked docs to learn how to use multiprocessing.shared_memory with numpy.ndarray
The answer provided by #Daweo suggesting use of shared memory is correct.
However, it's also worth considering using a lock to 'protect' access to the numpy array (which is not thread-safe).
See:- this
Okay guys, Thanks for the help. I used multiprocessing queue to share data.
Then I transfered my program multiprocessing to threading.
def capture(q):
cap =
cv2.VideoCapture(0)
while True:
ref, frame = cap.read()
frame = cv2.resize(frame, (640, 480))
q.put(frame)
def det(q):
model = torch.hub.load('ultralytics/yolov5','yolov5s',device='cpu')
model.conf = 0.30 # model confidence level
model.classes = [0] # model classes (where 0 = person, 2 = car)
model.iou = 0.55 # bounding box accuracy
while True:
mat = q.get()
det = model(mat)
bbox = det.xyxy[0].numpy()
for i in bbox:
print(i)
I have a question regarding threading and queueing in python. I have a class that recognises faces and another script that is supposed to retrieve these recognised faces. And I'm unable to retrieve the recognised faces, once recognition starts
I posted another question that relates to this one, but here it's more about the actual user recognition part (this just in case someone stumbles over my other question and thinks this may be a duplicate).
So as said, I have a class that uses imutile and face_recognition to do just that - do face recognition. My issue with the class is that, once it's started it does recognise faces perfectly but it's not possible from an outside class (from within another script) to call any other method for example to retrieve a dict with the currently identified faces. I think this is problably because, once the actual recognition is called, the call to other methods within this class is not going through because of threading???. I attached the complete code for reference below.
Here is the code that is supposed to start the recogniser class and retrieve the results from within another script:
class recognizerAsync(Thread):
def __init__(self):
super(recognizerAsync,self).__init__()
print("initiating recognizer class from recognizerAsync")
if (use_user == True):
#myRecognizer = recognizer(consoleLog, debugMsg, run_from_flask)
self.myRecognizer = the_recognizer.recognizerAsync(False, True, True)
#face_thread = recognizerAsync(consoleLog, debugMsg, False)
def run(self):
print("starting up recogizer from callRecognizerThread")
self.myRecognizer.run()
if (use_user == True):
self.myRecognizer.start()
while (True):
if ( not q.full() ):
#fd = random.randint(1,10)
fd = self.myRecognizer.returnRecognizedFaces()
print(f"PRODUCER: putting into queue {fd}")
q.put(fd)
#else:
# print("ERROR :: Queue q is full")
time.sleep(10)
And I start this like so in the very end:
mirror = GUI(window)
mirror.setupGUI()
window.after(1000, mirror.updateNews)
face_thread = recognizerAsync()
face_thread.start()
window.mainloop()
My question is, how would I need to change either the recogniser classier the recognizerAsync class in the other script, so that while the method faceRecognizer() is running indefinitely, one can still call other methods - specifically returnRecognizedFaces()???
Thank you very much, folks.
#!/usr/bin/env python3
# import the necessary packages for face detection
from imutils.video import VideoStream
from imutils.video import FPS
import face_recognition
import imutils
import pickle
import time
import base64
import json
from threading import Thread
class recognizerAsync(Thread):
# How long in ms before a person detection is considered a new event
graceTimeBeforeNewRecognition = 6000 #60000
# the time in ms when a person is detected
timePersonDetected = 0
# ceate an empty faces dictionary to compare later with repetive encounterings
faces_detected_dict = {"nil": 0}
# Determine faces from encodings.pickle file model created from train_model.py
encodingsP = "" # "encodings.pickle"
# load the known faces and embeddings along with OpenCV's Haar
# cascade for face detection - i do this in the class initialization
#data = pickle.loads(open(encodingsP, "rb").read())
data = ''
# print dictionary of recognized faces on the console?
print_faces = False
debug_mnessagesa = False
called_from_flask = True # this changes the path to the
def __init__(self, print_val=False, debug_val=False, called_from_flask=True):
super(recognizerAsync,self).__init__()
self.print_faces = print_val
self.debug_messages = debug_val
if (called_from_flask == False):
encodingsP = "encodings.pickle"
else:
encodingsP = "Recognizer/encodings.pickle"
# load the known faces and embeddings along with OpenCV's Haar
# cascade for face detection
self.data = pickle.loads(open(encodingsP, "rb").read())
if (self.debug_messages == True):
print("Faces class initialized")
def run(self):
self.faceRecognizer()
def returnRecognizedFaces(self):
if (self.debug_messages == True):
print("from returnRecognizedFaces: returning: " + str((({k:self.faces_detected_dict[k] for k in self.faces_detected_dict if k!='nil'}))))
# print(f"from returnRecognizedFaces: returning: {self.faces_detected_dict}")
return(({k:self.faces_detected_dict[k] for k in self.faces_detected_dict if k!='nil'}))
def faceRecognizer(self):
try:
# initialize the video stream and allow the camera sensor to warm up
# Set the ser to the followng
# src = 0 : for the build in single web cam, could be your laptop webcam
# src = 2 : I had to set it to 2 inorder to use the USB webcam attached to my laptop
#vs = VideoStream(src=2,framerate=10).start()
vs = VideoStream(src=0,framerate=10).start()
#vs = VideoStream(usePiCamera=True).start()
time.sleep(2.0)
# start the FPS counter
fps = FPS().start()
if (self.debug_messages == True):
print("starting face detection - press Ctrl C to stop")
# loop over frames from the video file stream
while (True):
# grab the frame from the threaded video stream and resize it
# to 500px (to speedup processing)
frame = vs.read()
try:
frame = imutils.resize(frame, width=500)
except:
# Error: (h, w) = image.shape[:2]
# AttributeError: 'NoneType' object has no attribute 'shape'
break
# Detect the fce boxes
boxes = face_recognition.face_locations(frame)
# compute the facial embeddings for each face bounding box
encodings = face_recognition.face_encodings(frame, boxes)
names = []
# loop over the facial embeddings
for encoding in encodings:
# attempt to match each face in the input image to our known encodings
matches = face_recognition.compare_faces(self.data["encodings"], encoding)
name = "unknown" #if face is not recognized, then print Unknown
timePersonDetected = time.time()*1000.0
# check to see if we have found a match
if (True in matches):
# find the indexes of all matched faces then initialize a
# dictionary to count the total number of times each face
# was matched
matchedIdxs = [i for (i, b) in enumerate(matches) if b]
counts = {}
# loop over the matched indexes and maintain a count for
# each recognized face face
for i in matchedIdxs:
name = self.data["names"][i]
counts[name] = counts.get(name, 0) + 1
# determine the recognized face with the largest number
# of votes (note: in the event of an unlikely tie Python
# will select first entry in the dictionary)
name = max(counts, key=counts.get)
# If someone in your dataset is identified, print their name on the screen and provide them through rest
if (max(self.faces_detected_dict, key=self.faces_detected_dict.get) != name or timePersonDetected > self.faces_detected_dict[name] + self.graceTimeBeforeNewRecognition):
# put the face in the dictionary with time detected so we can prrovide this info
# in the rest endpoint for others - this is not really used internally,
# except for the timePersonDetected time comparison above
self.faces_detected_dict[name] = timePersonDetected
# update the list of names
names.append(name)
# exemplary way fo cleaning up the dict and removing the nil entry - kept here for reference:
#new_dict = ({k:self.faces_detected_dict[k] for k in self.faces_detected_dict if k!='nil'})
self.last_recognized_face = name
if (self.print_faces == True):
print(self.last_recognized_face)
# clean up the dictionary
new_dict = {}
for k, v in list(self.faces_detected_dict.items()):
if (v + self.graceTimeBeforeNewRecognition) < (time.time()*1000.0) and str(k) != 'nil':
if (self.debug_messages == True):
print('entry ' + str(k) + " dropped due to age")
else:
new_dict[k] = v
self.faces_detected_dict = new_dict
if (self.debug_messages == True):
print(f"faces dict: {self.faces_detected_dict}")
# update the FPS counter
fps.update()
time.sleep(1)
except KeyboardInterrupt:
if (self.debug_messages == True):
print("Ctrl-C received - cleaning up and exiting")
pass
I have a flask application which reads frame from camera and streams it to the website.
Camera.py
from threading import Thread
from copy import deepcopy
import queue
import cv2
class Camera(Thread):
def __init__(self, cam, normalQue, detectedQue):
Thread.__init__(self)
self.__cam = cam
self.__normalQue = normalQue
self.__detectedQue = detectedQue
self.__shouldStop = False
def __del__(self):
self.__cam.release()
print('Camera released')
def run(self):
while True:
rval, frame = self.__cam.read()
if rval:
frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
_, jpeg = cv2.imencode('.jpg', frame)
self.__normalQue.put(jpeg.tobytes())
self.__detectedQue.put(deepcopy(jpeg.tobytes()))
if self.__shouldStop:
break
def stopCamera(self):
self.__shouldStop = True
From what you can see I am just reading the frame, resizing it and storing in two different ques. Nothing too complex.
I also have two two classes responsible for mjpeg stream:
NormalVideoStream.py
from threading import Thread
import traceback
import cv2
class NormalVideoStream(Thread):
def __init__(self, framesQue):
Thread.__init__(self)
self.__frames = framesQue
self.__img = None
def run(self):
while True:
if self.__frames.empty():
continue
self.__img = self.__frames.get()
def gen(self):
while True:
try:
if self.__img is None:
print('Normal stream frame is none')
continue
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
except:
traceback.print_exc()
print('Normal video stream genenation exception')
and
DetectionVideoStream.py
from threading import Thread
import cv2
import traceback
class DetectionVideoStream(Thread):
def __init__(self, framesQue):
Thread.__init__(self)
self.__frames = framesQue
self.__img = None
self.__faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
def run(self):
while True:
if self.__frames.empty():
continue
self.__img = self.__detectFace()
def gen(self):
while True:
try:
if self.__img is None:
print('Detected stream frame is none')
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
except:
traceback.print_exc()
print('Detection video stream genenation exception')
def __detectFace(self):
retImg = None
try:
img = self.__frames.get()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = self.__faceCascade.detectMultiScale(gray, 1.1, 4)
for (x, y, w, h) in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
(_, encodedImage) = cv2.imencode('.jpg', img)
retImg = encodedImage.tobytes()
except:
traceback.print_exc()
print('Face detection exception')
return retImg
From what you can see in both streams I am reading camera frames from ques in infinite loop. Both classes also have gen() method which generates frame to site itself. Only difference is that in detection stream I am also doing face recognition.
Now in my main file:
main.py
from flask import Blueprint, render_template, Response, abort, redirect, url_for
from flask_login import login_required, current_user
from queue import Queue
from . import db
from .Camera import Camera
from .NormalVideoStream import NormalVideoStream
from .DetectionVideoStream import DetectionVideoStream
from .models import User
import cv2
main = Blueprint('main', __name__)
# Queues for both streams
framesNormalQue = Queue(maxsize=0)
framesDetectionQue = Queue(maxsize=0)
print('Queues created')
# RPi camera instance
camera = Camera(cv2.VideoCapture(0), framesNormalQue, framesDetectionQue)
camera.start()
print('Camera thread started')
# Streams
normalStream = NormalVideoStream(framesNormalQue)
detectionStream = DetectionVideoStream(framesDetectionQue)
print('Streams created')
normalStream.start()
print('Normal stream thread started')
detectionStream.start()
print('Detection stream thread started')
#main.route('/')
def index():
return render_template('index.html')
#main.route('/profile', methods=["POST", "GET"])
def profile():
if not current_user.is_authenticated:
abort(403)
return render_template('profile.html', name=current_user.name, id=current_user.id, detectionState=current_user.detectionState)
#main.route('/video_stream/<int:stream_id>')
def video_stream(stream_id):
if not current_user.is_authenticated:
abort(403)
print(f'Current user detection: {current_user.detectionState}')
global detectionStream
global normalStream
stream = None
if current_user.detectionState:
stream = detectionStream
print('Stream set to detection one')
else:
stream = normalStream
print('Stream set to normal one')
return Response(stream.gen(), mimetype='multipart/x-mixed-replace; boundary=frame')
#main.route('/detection')
def detection():
if not current_user.is_authenticated:
abort(403)
if current_user.detectionState:
current_user.detectionState = False
else:
current_user.detectionState = True
user = User.query.filter_by(id=current_user.id)
user.detectionState = current_user.detectionState
db.session.commit()
return redirect(url_for('main.profile', id=current_user.id, user_name=current_user.name))
#main.errorhandler(404)
def page_not_found(e):
return render_template('404.html'), 404
#main.errorhandler(403)
def page_forbidden(e):
return render_template('403.html'), 403
I am creating camera, ques and streams object globally. Also when user logs in on website, he will be able to see live video stream. There is also a button which changes the stream which is currently presented.
Whole project is working well with one exception: when I change stream to detection one it will have huge lag (around 10/15 seconds) which makes whole thing unfunctional. Tried to search a bug/optimalization in my own but can't find anything. On purpose I am running everything on separate threads to unoverload app but it looks like this is not enought. Lag on a level of 1 - 2 seconds will be acceptable, but not 10+. So guys, maybe you can see some bug here? Or know how to optimalize it?
Also need to mention that whole app is running on RPi 4B 4GB and I am accessing website on my desktop. Default server is changed to the Nginx and Gunicorn. From what I can see Pi's CPU usage is 100% when app is working. When testing on default server behaviout is the same. Guess that 1,5 GHz CPU have enough power to run it more smoothly.
One option is using VideoStream
The reason VideoCapture is so slow because the VideoCapture pipeline spends the most time on the reading and decoding the next frame. While the next frame is being read, decode, and returned the OpenCV application is completely blocked.
VideoStream solves the problem by using a queue structure, concurrently read, decode, and return the current frame.
VideoStream supports both PiCamera and webcam.
All you need to is:
Install imutils:
For virtual environment: pip install imutils
For anaconda environment: conda install -c conda-forge imutils
Initialize VideoStream on main.py
import time
from imutils.video import VideoStream
vs = VideoStream(usePiCamera=True).start() # For PiCamera
# vs = VideoStream(usePiCamera=False).start() # For Webcam
camera = Camera(vs, framesNormalQue, framesDetectionQue)
In your Camera.py
In run(self) method:
* ```python
def run(self):
while True:
frame = self.__cam.read()
frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
_, jpeg = cv2.imencode('.jpg', frame)
self.__normalQue.put(jpeg.tobytes())
self.__detectedQue.put(deepcopy(jpeg.tobytes()))
if self.__shouldStop:
break
```
One of the issue which even I had was regrading the encoding and decoding. Like the encoder of Opencv is too slow so try to use encoder from simplejpeg. Use pip3 install simplejpeg and with respect to using cv2.imencode() use simplejpeg.encode_jpeg()
I'm not really surprised about your problem, in general "detection" using a lot of your computation time, because
performing a cascaded classification algorithm is a demanding computational task.
I found a source which compares cascaded classification algos for there performance link
An easy solution, would be to reduce the frame rate, when
processing your detection.
An easy implementation to reduce performance demand could be something like a skip counter e.g.
frameSkipFactor = 3 # use every third frame
frameCounter = 0
if (frameCounter%frameSkipFactor==0):
#process
else:
print("ignore frame", frameCounter)
frameCounter+=1
Nevertheless you will have a lag, because the detection calculation
will produce always a time offset.
I you planning to construct a "real time" classification camera system, please look for another class of classification algos,
which are more designed for this use-case.
I followed an discussion here: real time class algos
Another solution could be using a bigger "hardware hammer" than the rpi e.g. an GPU implementation of the algo via Cuda etc.
What I have found that main reason of getting slow frames is that you have higher resolution and frame rates.
To tackle this problem you can change the resolution to something 640 width by 480 height with fps 30 or less upto 5 fps (if you only need face detection) and can implement resizing of OpenCV (cv2.resize() function) by 0.25 resizing factor of fx and fy. Do this if you don't want higher resolution streams. Works smoothly with opencv. I wanted to try out that VideoStream code (from imutils) as it is also used by Adrian Rosebrock of PyImageSearch. I will use in later projects.
For reference, I am posting code snippet in the following. Special Thanks to Adrian Rosebrock and ageitey face_recognition as their code helped me making it.
class Camera(object):
SHRINK_RATIO = 0.25
FPS = 5
FRAME_RATE = 5
WIDTH = 640
HEIGHT = 480
def __init__(self):
""" initializing camera with settings """
self.cap = cv2.VideoCapture(0)
self.cap.set(3,Camera.WIDTH)
self.cap.set(4,Camera.HEIGHT)
self.cap.set(5,Camera.FPS)
self.cap.set(7,Camera.FRAME_RATE)
def get_frame(self):
""" get frames from the camera """
success, frame = self.cap.read()
if success == True:
# Resizing to 0.25 scale
rescale_frame = cv2.resize(frame, None, fx= Camera.SHRINK_RATIO, fy=Camera.SHRINK_RATIO)
cascPath = "haarcascade_frontalface_default.xml"
faceCascade = cv2.CascadeClassifier(cascPath)
gray = cv2.cvtColor(rescale_frame, cv2.COLOR_BGR2GRAY)
# detecting faces
faces = faceCascade.detectMultiScale(
gray,
scaleFactor=1.1,
minNeighbors=5,
minSize=(30,30)
)
# Draw a rectangle around the faces
if len(faces) != 0:
for (x, y, w, h) in faces:
x *=4
y *=4
w *=4
h *=4
# Draw rectangle across faces in current frame
cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
# return frame outside if-statement
return frame
Also keep in mind that JPEG is the accelerated codec, use:
cv2.imencode(".jpg",frame)[1].tobytes()
I am trying to make an application with a flask that
Captures a video stream from a webcam
Applies an object detection algorithm on each frame
Concurrently displays the frames with bounding boxes and data provided by the above function in the form of a video
The problem is that due to a function operation, the result renders with a bit of lag. To overcome this I tried to run the function over another thread.
Can you please help in running the function over each frame of the video and displaying the resultant frames in the form of a video?
My main function looks somewhat like :
def detect_barcode():
global vs, outputFrame, lock
total = 0
# loop over frames from the video stream
while True:
frame = vs.read()
frame = imutils.resize(frame, width=400)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)
timestamp = datetime.datetime.now()
cv2.putText(frame, timestamp.strftime(
"%A %d %B %Y %I:%M:%S%p"), (10, frame.shape[0] - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)
if total > frameCount:
# detect barcodes in the image
barcodes = md.detect(gray)
# check to see if barcode was found
if barcode in barcodes:
x,y,w,h = cv2.boundRect(barcode)
cv2.rectangle(frame,x,y,(x+w),(y+h),(255,0,0),2)
total += 1
# lock
with lock:
outputFrame = frame.copy()
if __name__ == '__main__':
# start a thread that will perform motion detection
t = threading.Thread(target=detect_barcode)
t.daemon = True
t.start()
app.run(debug=True)
vs.stop()
I literally just did something like this. My solution was asyncio's concurrent.futures module.
import concurrent.futures as futures
Basically, you create your executor and 'submit' your blocking method as a parameter. It returns something called Future, which you can check the result of (if any) every frame.
executor = futures.ThreadPoolExecutor()
would initialize the executor
future_obj = executor.submit(YOUR_FUNC, *args)
would start execution. Sometime later on you could check its status
if future_obj.done():
print(future_obj.result())
else:
print("task running")
# come back next frame and check again
I am trying to get my script to run over multiple cores. When I start the script it works but only seems to run on one core. I am running this on a Raspberry Pi 3 b+ with 4 cores, but only one is being used when I check using top.
I have tried both Pool and Process from the multiprocessing module. I do not have a programming background but have written basic scrips with Python before, though this is the first time trying to use multiprocessing.
Sorry for sharing all this code but I think its the best way to show what I have tried.
import multiprocessing as mp
from multiprocessing import Process
import cv2
import face_pi_helper as fph
def put_frame_on_q(q_vid_cap, vid_cap):
for i in range(5):
frame = fph.fetch_frame(vid_cap)
q_vid_cap.put(frame)
def detect_from_q(q_vid_cap, q_detections, net):
print(fph.time_stamp(), "[INFO] starting detect from q...")
while True:
frame = q_vid_cap.get()
if frame is None:
print(fph.time_stamp(), "[WARNING] No frame")
else:
detections = tuple(fph.detect(frame, net))
d = detections[1]
for f in d:
conf = float(f[2])
if conf > 0.9:
print(fph.time_stamp(), "[INFO] face found")
q_detections.put(detections)
def identify_from_q(q_detections,
vid_cap,
known_face_encodings,
known_face_names,
q_vid_cap
):
print(fph.time_stamp(), "[INFO] starting identify from q...")
while True:
detections = q_detections.get()
if detections is None:
print(fph.time_stamp(), "[WARNING] No detections")
else:
frame = detections[0]
fph.identify(
frame,
detections[1],
known_face_encodings,
known_face_names
)
put_frame_on_q(q_vid_cap, vid_cap)
if __name__ == "__main__":
topology = "models/intel/face-detection-retail-0004.xml"
weights = "models/intel/face-detection-retail-0004.bin"
image = "images/adam.jpg"
db_file = "data/allstars_faces_plus.json"
print(fph.time_stamp(), "[INFO] starting app...")
net = fph.load_model(topology, weights)
known_face_encodings, known_face_names = fph.load_face_db(db_file)
q_vid_cap = mp.Queue(1)
q_detections = mp.Queue(1)
vid_cap = cv2.VideoCapture(0)
put_frame_on_q(q_vid_cap, vid_cap)
p_detect = Process(
target = detect_from_q,
args = (q_vid_cap, q_detections, net, ))
print(fph.time_stamp(), "[INFO] starting detect from q process...")
p_detect.start()
p_identify = Process(
target = identify_from_q,
args = (
q_detections,
vid_cap,
known_face_encodings,
known_face_names,
q_vid_cap,
)
)
print(fph.time_stamp(),
"[INFO] starting identify from q process..."
)
p_identify.start()
p_detect.join()
p_identify.join()
I would expect this to run on 3 cores but it only runs on 1.