We Keep Coding

Python multiprocessing queue using a lot of resources with opencv

I am using multiprocessing to get frames of a video using Opencv in python.
My class looks like this :-
import cv2
from multiprocessing import Process, Queue
class StreamVideos:
def __init__(self):
self.image_data = Queue()
def start_proces(self):
p = Process(target=self.echo)
p.start()
def echo(self):
cap = cv2.VideoCapture('videoplayback.mp4')
while cap.isOpened():
ret,frame = cap.read()
self.image_data.put(frame)
# print("frame")
I start the process "echo" using :-
p = Process(target=self.echo)
p.start()
the echo function looks like this :-
def echo(self):
cap = cv2.VideoCapture('videoplayback.mp4')
while cap.isOpened():
ret,frame = cap.read()
self.image_data.put(frame)
in which i am using queue where i put these frames
self.image_data.put(frame)
and then in another process I start reviving these frames
self.obj = StreamVideos()
def start_process(self):
self.obj.start_proces()
p = Process(target=self.stream_videos)
p.start()
def stream_videos(self):
while True:
self.img = self.obj.image_data.get()
print(self.img)
but as soon as I start putting frames to queue, the ram gets filled very quickly and the system gets stuck. The video I am using is just 25 fps and 39mb in size, so it does not make any sense.
One thing I noticed is that the "echo" process is putting a lot of frames in the queue before the "stream_videos" process retrives it.
What could be the root of this problem?
Thanks in advance.
Expectations: -
Able to retrieve the frames continuosly.
Tried :-
Not putting frames in queue, in which case the ram is not filled.

The following is a general purpose single producer/multiple consumer implementation. The producer (class StreamVideos) creates a shared memory array whose size is the number of bytes in the video frame. One or more consumers (you specify the number of consumers to StreamVideos) can then call StreamVideos.get_next_frame() to retrieve the next frame. This method converts the shared array back into a numpy array for subsequent processing. The producer will only read the next frame into the shared array after all consumers have called get_next_frame:
#!/usr/bin/env python3
import multiprocessing
import numpy as np
import ctypes
import cv2
class StreamVideos:
def __init__(self, path, n_consumers):
"""
path is the path to the video:
n_consumers is the number of tasks to which we will be sreaming this.
"""
self._path = path
self._event = multiprocessing.Event()
self._barrier = multiprocessing.Barrier(n_consumers + 1, self._reset_event)
# Discover how large a framesize is by getting the first frame
cap = cv2.VideoCapture(self._path)
ret, frame = cap.read()
if ret:
self._shape = frame.shape
frame_size = self._shape[0] * self._shape[1] * self._shape[2]
self._arr = multiprocessing.RawArray(ctypes.c_ubyte, frame_size)
else:
self._arr = None
cap.release()
def _reset_event(self):
self._event.clear()
def start_streaming(self):
cap = cv2.VideoCapture(self._path)
while True:
self._barrier.wait()
ret, frame = cap.read()
if not ret:
# No more readable frames:
break
# Store frame into shared array:
temp = np.frombuffer(self._arr, dtype=frame.dtype)
temp[:] = frame.flatten(order='C')
self._event.set()
cap.release()
self._arr = None
self._event.set()
def get_next_frame(self):
# Tell producer that this consumer is through with the previous frame:
self._barrier.wait()
# Wait for next frame to be read by the producer:
self._event.wait()
if self._arr is None:
return None
# Return shared array as a numpy array:
return np.ctypeslib.as_array(self._arr).reshape(self._shape)
def consumer(producer, id):
frame_name = f'Frame - {id}'
while True:
frame = producer.get_next_frame()
if frame is None:
break
cv2.imshow(frame_name, frame)
cv2.waitKey(1)
cv2.destroyAllWindows()
def main():
producer = StreamVideos('videoplayback.mp4', 2)
consumer1 = multiprocessing.Process(target=consumer, args=(producer, 1))
consumer1.start()
consumer2 = multiprocessing.Process(target=consumer, args=(producer, 2))
consumer2.start()
"""
# Run as a child process:
producer_process = multiprocessing.Process(target=producer.start_streaming)
producer_process.start()
producer_process.join()
"""
# Run in main process:
producer.start_streaming()
consumer1.join()
consumer2.join()
if __name__ == '__main__':
main()

Send Thermal Image with mlx90640 via TCP

I have a problem with my program I am trying to send the images I collect from the MLX90640 thanks to the Raspberry to process them in a remote PC.
I am using a Raspberry 4 as a client and the data is routed to a PC. I am using the socket to start the server which is to receive and the images and thermal images. For the images connected to the camera, I took care of it my problem is to transfer the thermal images. I am currently using a wifi connection that I share with my cellphone for the tests.If necessary I will post the server code. But I have this error message I have tried many solutions and I have not found it. In fact, the Raspberry is the client and the PC is the server. So I collect data from the raspberry to transmit it to the PC for processing. I want to detect the temperature of the face and for that the MLX90640 which is connected to the Raspberry must send the thermal data. Knowing that it collects 768 values, so I want these values ​​to be transmitted or the maximum value to be returned to the PC. Can someone help me
import cv2
import io
import socket
import struct
import time
import pickle
import zlib
import adafruit_mlx90640
import board
import busio
import numpy as np
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('192.168.43.134', 8485))
connection = client_socket.makefile('wb')
i2c = busio.I2C(board.SCL, board.SDA, frequency=800000)
mlx = adafruit_mlx90640.MLX90640(i2c)
print("MLX addr detected on I2C")
print([hex(i) for i in mlx.serial_number])
mlx.refresh_rate = adafruit_mlx90640.RefreshRate.REFRESH_4_HZ
frame1 = np.zeros((24*32,))
#max_t=0
#moy = 0
#cam = cv2.VideoCapture(0)
#mlx.set(3, 32);
#mlx.set(4, 24);
img_counter = 0
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 90]
while True:
frame = mlx.getFrame(frame1)
result, frame = cv2.imencode('.jpg', frame, encode_param)
# data = zlib.compress(pickle.dumps(frame, 0))
data = pickle.dumps(frame, 0)
size = len(data)
print("{}: {}".format(img_counter, size))
client_socket.sendall(struct.pack(">L", size) + data)
img_counter += 1
```Traceback (most recent call last): File "client1.py", line 37, in <module> result, frame = cv2.imencode('.jpg', frame, encode_param) cv2.error: OpenCV(4.1.1) /home/pi/opencv/modules/imgcodecs/src/grfmt_base.cpp:145: error: (-10:Unknown error code -10) Raw image encoder error: Empty JPEG image (DNL not supported) in function 'throwOnEror'

Do you manage to get thermal at Raspberry pi? I did a similar approach but i am not using thermal camera. If your problem is not able to transfer image from raspberry pi to your computer
Server code at Raspberry pi
#!/usr/bin/env python3
import os
import datetime
import numpy as np
import cv2
import sys
import socket
import select
import queue
import pickle
import struct
import time
from threading import Thread
class WebcamVideoStream:
def __init__(self, src=0):
self.stream = cv2.VideoCapture(src)
cv2.VideoWriter_fourcc('M','J','P','G')
self.stream .set(cv2.CAP_PROP_BUFFERSIZE,1)
self.stream .set(5, 60)
self.stream .set(3,640)
self.stream .set(4,480)
(self.grabbed, self.frame) = self.stream.read()
self.stopped = False
def start(self):
Thread(target=self.update, args=()).start()
return self
def update(self):
while True:
if self.stopped:
return
(self.grabbed, self.frame) = self.stream.read()
time.sleep(0.1)
def read(self):
img= cv2.cvtColor(self.frame , cv2.COLOR_BGR2GRAY)
data = pickle.dumps(img)
return data
def stop(self):
self.stopped = True
def commandParser(cmd, stream):
reply = ""
if(cmd == "getimage"):
reply = stream.read()
time.sleep(0.1)
else:
reply = '/n'.encode()
return(reply)
if __name__ == '__main__':
camera_idx = 0
for i in range(3):
stream = cv2.VideoCapture(i)
test,frame = stream.read()
stream.release()
if test == True:
camera_idx = i
break
#stream = cv2.VideoCapture(camera_idx)
vs = WebcamVideoStream(src=camera_idx).start()
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 8080
server.bind(('192.168.128.14', port))
server.listen(5)
inputs = [server]
outputs = []
message_queues = {}
cmd =""
while inputs:
readable, writable, exceptional = select.select(inputs, outputs, inputs, 1)
for s in readable:
if s is server:
connection, client_address = s.accept()
inputs.append(connection)
message_queues[connection] = queue.Queue(1024)
else:
data = s.recv(4096)
if data:
cmd = data.decode()
message_queues[s].put(commandParser(data.decode(), vs))
if s not in outputs:
outputs.append(s)
else:
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del message_queues[s]
for s in writable:
try:
next_msg = message_queues[s].get_nowait()
except queue.Empty:
outputs.remove(s)
else:
if(cmd == "getimage"):
size = len(next_msg)
s.sendall(struct.pack(">L", size) + next_msg)
else:
s.send("ABCDEFGHIJKLMNONOOO".encode())
for s in exceptional:
print ('handling exceptional condition for', s.getpeername())
inputs.remove(s)
if s in outputs:
outputs.remove(s)
s.close()
del message_queues[s]
vs.stop()
Client Code at PC
#!/usr/bin/env python3
import os
import datetime
import numpy as np
import cv2
import socket
import socket
import sys
import pickle
import struct ## new
import zlib
import time
server_address = ('192.168.128.14', 8080)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#print ('connecting to %s port %s' % server_address)
s.connect(server_address)
cv2.namedWindow('Streaming')
payload_size = struct.calcsize(">L")
while True:
s.send("getimage".encode())
data = b""
while len(data) < payload_size:
data += s.recv(4096)
packed_msg_size = data[:payload_size]
data = data[payload_size:]
msg_size = struct.unpack(">L", packed_msg_size)[0]
while len(data) < msg_size:
data += s.recv(4096)
frame_data = data[:msg_size]
data = data[msg_size:]
frame=pickle.loads(frame_data, fix_imports=True, encoding="bytes")
cv2.imshow('Streaming',frame)
cv2.waitKey(1)
#cv2.imwrite("test.tiff", frame)
s.close()

how to do real time video/audio streaming in django

i want to stream the video and audio (and some real time data which i will get from precessing every fram) from surveillance camera into a django website ... i found this code that help me send frames to the client
'''
from django.shortcuts import render
from django.http import HttpResponse, StreamingHttpResponse
import cv2
import time
from django.views.decorators import gzip
class VideoCamera(object):
def __init__(self):
self.video = cv2.VideoCapture('./streaming/video.mp4')
def __del__(self):
self.video.release()
def get_frame(self):
ret, image = self.video.read()
ret, jpeg = cv2.imencode('.jpg', image)
return jpeg.tobytes()
def gen(camera):
while True:
frame = camera.get_frame()
yield(b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
#gzip.gzip_page
def index(request):
try:
return StreamingHttpResponse(gen(VideoCamera()), content_type="multipart/x-mixed-replace;boundary=frame")
except HttpResponseServerError as e:
print("aborted")
but i dont know how to handle the audio and data and the synchronization . i want to know what technology i have to use and if there any tutorial or ideas about it. i really don't know what to read and how to start (i'm using django).

Follow for more details :
Github : https://github.com/JRodrigoF/AVrecordeR
class AudioRecorder():
# Audio class based on pyAudio and Wave
def __init__(self):
self.open = True
self.rate = 44100
self.frames_per_buffer = 1024
self.channels = 2
self.format = pyaudio.paInt16
self.audio_filename = "temp_audio.wav"
self.audio = pyaudio.PyAudio()
self.stream = self.audio.open(format=self.format,
channels=self.channels,
rate=self.rate,
input=True,
frames_per_buffer = self.frames_per_buffer)
self.audio_frames = []
# Audio starts being recorded
def record(self):
self.stream.start_stream()
while(self.open == True):
data = self.stream.read(self.frames_per_buffer)
self.audio_frames.append(data)
if self.open==False:
break
# Finishes the audio recording therefore the thread too
def stop(self):
if self.open==True:
self.open = False
self.stream.stop_stream()
self.stream.close()
self.audio.terminate()
waveFile = wave.open(self.audio_filename, 'wb')
waveFile.setnchannels(self.channels)
waveFile.setsampwidth(self.audio.get_sample_size(self.format))
waveFile.setframerate(self.rate)
waveFile.writeframes(b''.join(self.audio_frames))
waveFile.close()
pass
# Launches the audio recording function using a thread
def start(self):
audio_thread = threading.Thread(target=self.record)
audio_thread.start()
Actully I don't have much idea with open cv2. But I think we cannot record video with audio simultaneously. You have to capture audio and video in thread.

Python: Automatically reconnect IP camera

My IP camera seems to be a little unstable and disconnects randomly. I'd like my script to be able to determine when its disconnected and attempt to reconnect a few times, probably waiting 5-10 seconds between attempts. I've tried a few things, but nothing is working.
This is my basic script, when ret is false the script ends:
#!/usr/local/bin/python3
import cv2
import time
import datetime
print("start time: " + datetime.datetime.now().strftime("%A %d %B %Y %I:%M:%S%p"))
cap = cv2.VideoCapture('rtsp://<ip><port>/live0.264')
while(True):
# Capture frame-by-frame
ret, frame = cap.read()
# Confirm we have a valid image returned
if not ret:
print("disconnected!")
break
# Our operations on the frame come here
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2BGRA)
# Display the resulting frame
cv2.imshow('frame', gray)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
print("end time: " + time.strftime("%X"))
# When everything is done, release the capture
cap.release()
cv2.destroyAllWindows()
Edit: I would also like the script to try to reconnect to the camera in the event that my network goes down temporarily or anything like that as well.

I was finally able to solve this myself. Hopefully this is useful for anyone else looking to do the same thing.
This is actually a shell of a more complex script that has logic for motion detection and video recording when motion is detected. Everything is working very well with this basic logic (and my crappy IP camera) although I am still doing testing.
#!/usr/local/bin/python3
import cv2
import datetime
import time
def reset_attempts():
return 50
def process_video(attempts):
while(True):
(grabbed, frame) = camera.read()
if not grabbed:
print("disconnected!")
camera.release()
if attempts > 0:
time.sleep(5)
return True
else:
return False
recall = True
attempts = reset_attempts()
while(recall):
camera = cv2.VideoCapture("rtsp://<ip><port>/live0.264")
if camera.isOpened():
print("[INFO] Camera connected at " +
datetime.datetime.now().strftime("%m-%d-%Y %I:%M:%S%p"))
attempts = reset_attempts()
recall = process_video(attempts)
else:
print("Camera not opened " +
datetime.datetime.now().strftime("%m-%d-%Y %I:%M:%S%p"))
camera.release()
attempts -= 1
print("attempts: " + str(attempts))
# give the camera some time to recover
time.sleep(5)
continue

More detailed description:
https://github.com/Combinacijus/various-code-samples/tree/master/Python/OpenCV/ip_cam_reconnecting
Wrote a class to deal with IP camera disconnecting randomly. Main idea is to check if cap.read() returns a frame and if it doesn't it tries to reconnect to the camera.
import cv2
import requests # NOTE: Only used for forceful reconnection
import time # NOTE: Only used for throttling down printing when connection is lost
class IPVideoCapture:
def __init__(self, cam_address, cam_force_address=None, blocking=False):
"""
:param cam_address: ip address of the camera feed
:param cam_force_address: ip address to disconnect other clients (forcefully take over)
:param blocking: if true read() and reconnect_camera() methods blocks until ip camera is reconnected
"""
self.cam_address = cam_address
self.cam_force_address = cam_force_address
self.blocking = blocking
self.capture = None
self.RECONNECTION_PERIOD = 0.5 # NOTE: Can be changed. Used to throttle down printing
self.reconnect_camera()
def reconnect_camera(self):
while True:
try:
if self.cam_force_address is not None:
requests.get(self.cam_force_address)
self.capture = cv2.VideoCapture(self.cam_address)
if not self.capture.isOpened():
raise Exception("Could not connect to a camera: {0}".format(self.cam_address))
print("Connected to a camera: {}".format(self.cam_address))
break
except Exception as e:
print(e)
if self.blocking is False:
break
time.sleep(self.RECONNECTION_PERIOD)
def read(self):
"""
Reads frame and if frame is not received tries to reconnect the camera
:return: ret - bool witch specifies if frame was read successfully
frame - opencv image from the camera
"""
ret, frame = self.capture.read()
if ret is False:
self.reconnect_camera()
return ret, frame
if __name__ == "__main__":
CAM_ADDRESS = "http://192.168.8.102:4747/video" # NOTE: Change
CAM_FORCE_ADDRESS = "http://192.168.8.102:4747/override" # NOTE: Change or omit
cap = IPVideoCapture(CAM_ADDRESS, CAM_FORCE_ADDRESS, blocking=True)
# cap = IPVideoCapture(CAM_ADDRESS) # Minimal init example
while True:
ret, frame = cap.read()
if ret is True:
cv2.imshow(CAM_ADDRESS, frame)
if cv2.waitKey(1) == ord("q"):
break

HOW TO DO Socket Multiple Clients

this is a part from a code that making GUI for process list. I'm trying to do a Socket Multiple Clients but it is'nt working. Can some one help me with this?
import socket, pickle
import threading
import select
BUF_SIZE = 8192
class Network(threading.Thread):
def __init__(self, frame):
threading.Thread.__init__(self)
self.frame = frame
self.server_sock = socket.socket()
def run(self, client_sock=None):
self.server_sock.bind(('', 1729))
self.server_sock.listen(5)
self.open_client_sockets = []
while True:
self.rlist, self.wlist, self.xlist = select.select([self.server_sock] + self.open_client_sockets, [], [])
if self.currect_sock is self.server_sock:
client_sock, client_address = self.server_sock.accept()
self.open_client_sockets.append(self.new_sock)
data = client_sock.recv(BUF_SIZE)
processes_list = pickle.loads(data)
for process_details in processes_list:
self.frame.add_line(process_details)
else:
data = self.client_sock.recv(BUF_SIZE)
processes_list = pickle.loads(data)
for process_details in processes_list:
self.frame.add_line(process_details)
client_sock.close()
self.server_sock.close()
self.server_sock.close()