I'm doing some python work for real-time video analysis.
About for 1 month... I've revised same code everyday, but there was no improvement.
So I decided to write a question here.
**My Question : **
I finally got a real-time data (max intensity) from video camera, but I failed to draw a real-time graph using this data.
I tried to use Pipe(multiprocessing), but everytime I try this the python says '!_src.empty() in function 'cv::cvtColor', and other errors...
This is a code for extracting a max intensity from camera(video) in realtime.
import numpy as np
import cv2
import matplotlib.pyplot as plt
cap = cv2.VideoCapture(0)
fps = cap.get(cv2.CAP_PROP_FPS)
count = 0
while(True):
ret, frame = cap.read()
vid = cv2.cvtColor(frame, cv2.IMREAD_COLOR)
cv2.imshow('Interference',vid)
h, s, v = cv2.split(vid)
if(int(cap.get(1)) % fps == 0):
for i in np.arange(fps):
print(np.max(v))
if cv2.waitKey(1) & 0xFF == ord('q'):
break
And this is my failed code to extract the realtime data and make a realtime graph.
import numpy as np
import cv2
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from itertools import count
from multiprocessing import Process, Pipe
cap = cv2.VideoCapture(0)
fps = cap.get(cv2.CAP_PROP_FPS)
plt.style.use('fivethirtyeight')
count = 0
while(True):
ret, frame = cap.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
cv2.imshow('gray', gray)
h, s, v = cv2.split(vid)
def func0(pipe):
if(int(cap.get(1)) % fps == 0):
px = np.max(v)
for i in np.arange(fps):
pipe.send(px)
pipe.close(px)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def func1(pipe):
x_val = []
y_val = []
def animate():
x_val.append(count())
y_val.append(pipe)
plt.cla()
plt.plot(x_val, y_val)
ani = FuncAnimation(plt.gcf(),animate,interval = 1000)
plt.tight_layout()
plt.show()
if __name__ == '__main__':
a_pipe, b_pipe = Pipe()
p0 = Process(target=func0, args=(a_pipe,))
p0.start()
p1 = Process(target=func1, args=(b_pipe.recv(),))
p1.start()
p0.join()
p1.join()
How can I try to draw a realtime graph from this data ?
If you give me any hint of this, I will really appreciated about it.
Thank you.
I tried some code for extracting a realtime data and making a realtime graph.
Finally I succeed to extract a realtime data from video, but failed the other one.
Can you please help me to make a realtime graph using the extracted data in realtime?
Related
I am trying to extract keypoints using mediapipe to train an LSTM model to detect sign language actions(only for a set of adjectives for now). The training dataset is being parsed and passed to VideoCapture, which is then used to extract keypoints through mediapipe's holistic model, which are then saved as numpy arrays for each frame in .npy files in respective directories. However, due to varying video lengths, the total number of frames captured and thus the number of numpy arrays generated are varying for each video, which is not suitable for the training architecture. It would be straightforward to simply break out of the loop when 30 frames have been captured, but that would exract data only for first 30 frames, which is not desirable. Is there a way to not skip over data whilst getting only a fixed number of frames per video.
import cv2
import numpy as np
import os
from matplotlib import pyplot as plt
import time
import mediapipe as mp
DATASET_PATH = "/home/kuna71/Dev/HearMySign/Datasets/Adjectives_1of8/Adjectives"
KEYPOINT_PATH = "/home/kuna71/Dev/HearMySign/Keypoints"
sequence_len = 30
mp_holistic = mp.solutions.holistic
mp_drawing = mp.solutions.drawing_utils
def mediapipe_detection(image, model):
print("In mediapipe_detection function")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # COLOR CONVERSION BGR 2 RGB
image.flags.writeable = False # Image is no longer writeable (just to save on some memory)
results = model.process(image) # Make prediction
image.flags.writeable = True # Image is now writeable
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) # COLOR COVERSION RGB 2 BGR
return image, results
def extract_keypoints(results):
pose = np.array([[res.x, res.y, res.z, res.visibility] for res in results.pose_landmarks.landmark]).flatten() if results.pose_landmarks else np.zeros(33*4)
face = np.array([[res.x, res.y, res.z] for res in results.face_landmarks.landmark]).flatten() if results.face_landmarks else np.zeros(468*3)
lh = np.array([[res.x, res.y, res.z] for res in results.left_hand_landmarks.landmark]).flatten() if results.left_hand_landmarks else np.zeros(21*3)
rh = np.array([[res.x, res.y, res.z] for res in results.right_hand_landmarks.landmark]).flatten() if results.right_hand_landmarks else np.zeros(21*3)
return np.concatenate([pose, face, lh, rh])
#opencv code to capture frames
directories = os.listdir(DATASET_PATH)
for d in directories:
vids = os.listdir(os.path.join(DATASET_PATH, d))
for v in vids:
videopath = os.path.join(DATASET_PATH, d, v)
print("\n\n\n\n\n__________________CHANGING VIDEO__________________\n\n\n\n\n")
print("\n\n________VIDEOPATH:"+videopath)
cap = cv2.VideoCapture(videopath)
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
print("LENGTH:" + str(length))
# Set mediapipe model
with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
count =0
while cap.isOpened():
count=count+1
print("Reading feed")
ret, frame = cap.read()
if(frame is None):
break
image, result = mediapipe_detection(frame, holistic)
result_test = extract_keypoints(result)
print(result_test)
if not os.path.exists(os.path.join(KEYPOINT_PATH, d, v)):
os.makedirs(os.path.join(KEYPOINT_PATH, d, v))
NPY_PATH = os.path.join(KEYPOINT_PATH, d, v, str(count))
np.save(NPY_PATH, result_test)
# Show to screen
cv2.imshow('OpenCV Feed', image)
# Break gracefully
if cv2.waitKey(10) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
I'm trying to make a python program with OpenCV, which opens the webcam and takes several images with different exposures in real time (40ms,95ms,150ms) and averages them in the end.
I tried to create a loop in which I change the exposure time, update the rendering (frame) and save it in a list, but the problem is that the display remains static and the rendering hardly changes (which gives after merging the images an image whose exposure time is almost 40)
I supposed that after setting exposure time, the frame update needs some time so I added the method time.sleep to suspend the execution for 3 seconds, but it was in vain.
Here is my code
import numpy as np
import cv2
import os
import time
capture = cv2.VideoCapture(0, cv2.CAP_V4L2)
while True:
(grabbed, frame) = capture.read()
if not grabbed:
break
# Resize frame
width = 1500
height = 1000
dim = (width, height)
frame = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
cv2.imshow('RGB', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
if cv2.waitKey(1) == ord('h') or cv2.waitKey(1) == ord('H'):
#repertory = input("Enter the name of the directory: ")
if not os.path.exists(repertory):
os.mkdir(repertory)
exposure = [40,95,150]
ims = []
for i in exposure:
capture.set(cv2.CAP_PROP_EXPOSURE, i) # Setting Exposure
(grabbed, frame) = capture.read() # Updating frame
if grabbed:
cv2.imshow('RGB', frame) #Display
ims.append(frame)
# Convert to numpy
ims = np.array([np.array(im) for im in ims])
# average et conversion en uint8
imave = np.average(ims, axis=0)
imave = imave.astype(np.uint8)
# image HDR
cv2.imwrite(repertory + '/' + repertory + '_HDR8.jpg', imave)
capture.release()
cv2.destroyAllWindows()
Is there an optimal solution that allows to take pictures with differents exposure time in real time and in an automatic way?
I am trying to write a script to manipulate video from a webcam. I am trying to do this through OpenCV with Python, but I am running into some issues.
If I run the video capture stream with no pixel manipulation applied, the stream works fine and has a smooth frame rate. However, I applied a threshold loop as a test, and my stream undergoes major lag and updates once every few seconds. Any ideas if it is possible to optimise this? Ideally, I am looking to get a 30 fps stream with the video manipulation applied. Here is the code:
import cv2
import numpy as np
cap = cv2.VideoCapture(0)
T = 100
while True:
ret, frame = cap.read()
height, width, channels = frame.shape
for x in range(width):
for y in range(height):
if frame[y,x,0] < T:
frame[y,x]=0
else:
frame[y,x]=255
cv2.imshow('frame', frame)
if cv2.waitKey(1) == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
accessing Pixel by pixel in image processing in general is very bad practice as it slow the performance very much, packages like opencv and numpy has optimized this by doing matrix operations allowing your program to be much more faster, here is a sample code that will perform your task but much more faster.
import cv2
import numpy as np
cap = cv2.VideoCapture(0)
T = 100
while True:
ret, frame = cap.read()
height, width, channels = frame.shape
B,G,R = cv2.split(frame)
# for x in range(width):
# for y in range(height):
# if frame[y,x,0] < T:
# frame[y,x]=0
# else:
# frame[y,x]=255
_,B = cv2.threshold(B,T,255,cv2.THRESH_BINARY)
frame = cv2.merge((B,G,R))
cv2.imshow('frame', frame)
if cv2.waitKey(1) == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
I made a convolutional neural network, that predicts faces and returns coordinates (y1, x1, y2, x2). Iam able to create rectangle that serves as mask that covers the desired coordinates. I need a way to cover the images in real time. Is there a way to get live image sequence without saving the frames, just overwriting them, and how do i extract the coordinates in openCV? I was using pyplot and was saving the images, it is slow and ineffective.
Yeah, so I managed to come up with a solution, but I found out that, 1 frame takes about 0.54s to compute, so 2FPS, not great for live streaming, so I am switching to haarcascade.
Code below is used to configure and call the model.
from numpy import expand_dims
from mrcnn.config import Config
from mrcnn.model import MaskRCNN
from mrcnn.model import mold_image
import cv2
import time
# define the prediction configuration
class PredictionConfig(Config):
# define the name of the configuration
NAME = "face_cfg"
# number of classes (background + face)
NUM_CLASSES = 1 + 1
# simplify GPU config
GPU_COUNT = 1
IMAGES_PER_GPU = 1
def classify_image(image,model,cfg):
# convert pixel values (e.g. center)
scaled_image = mold_image(image, cfg)
# convert image into one sample
sample = expand_dims(scaled_image, 0)
# make prediction
tic = time.time()
yhat = model.detect(sample, verbose=0)[0]
print(time.time() - tic)
return yhat['rois']
def image_bnd_highlight(image,coordinates):
for box in coordinates:
# get coordinates
y1, x1, y2, x2 = box
# create the shape
new_img = cv2.rectangle(image,(x1,y1),(x2,y2),(255,255,255),5)
return new_img
# create config
cfg: PredictionConfig = PredictionConfig()
# define the model
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
# load model weights
model_path = 'mask_rcnn_face_cfg_0029.h5'
model.load_weights(model_path, by_name=True)
definitive_model = model
Then I call my functions, that I created above.
import cv2 as cv
import acapture
from RealTime import definitive_model
from RealTime import cfg
from RealTime import classify_image
from RealTime import image_bnd_highlight
import time
# cap = acapture.open(0)
cap = cv.VideoCapture(0)
cap.set(3,128) #set frame width
cap.set(4,128) #set frame height
cap.set(cv.CAP_PROP_FPS, 2) #adjusting fps to 2
# cap.set(cv.CAP_PROP_BUFFERSIZE,3)
# if not cap.isOpened():
# print("Cannot open camera")
# exit()
while True:
# Capture frame-by-frame
ret, frame = cap.read()
# if frame is read correctly ret is True
if not ret:
print("Can't receive frame (stream end?). Exiting ...")
break
# let's resize our image to be 150 pixels wide, but in order to
# prevent our resized image from being skewed/distorted, we must
# first calculate the ratio of the *new* width to the *old* width
r = 150.0 / frame.shape[1]
dim = (150, int(frame.shape[0] * r))
# perform the actual resizing of the image
resized = cv.resize(frame, dim, interpolation=cv.INTER_AREA)
# tic = time.time()
coords = classify_image(resized,definitive_model,cfg)
# print(time.time() - tic)
image = image_bnd_highlight(resized,coords)
# Display the resulting frame
cv.imshow('frame', image)
if cv.waitKey(1) == ord('q'):
break
# When everything done, release the capture
cap.release()
cv.destroyAllWindows()
What I need to do is fairly simple:
load a 5 frames video file
detect background
On every frames, one by one :
subtract background (create foreground mask)
do some calculations on foreground mask
save both original frame and foreground mask
Just to see the 5 frames and the 5 corresponding fgmasks :
import numpy as np
import cv2
cap = cv2.VideoCapture('test.avi')
fgbg = cv2.BackgroundSubtractorMOG()
while(True):
# Capture frame-by-frame
ret, frame = cap.read()
fgmask = fgbg.apply(frame)
# Display the fgmask frame
cv2.imshow('fgmask',fgmask)
# Display original frame
cv2.imshow('img', frame)
k = cv2.waitKey(0) & 0xff
if k == 5:
break
cap.release()
cv2.destroyAllWindows()
Every frame gets opened and displayed correctly but the showed fgmask do not correspond to the showed original frame. Somewhere in the process, the order of the fgmasks gets mixed up.
The background does get subtracted correctly but I don't get the 5 expected fgmasks.
What am I missing ? I feel like this should be straightforward : the while loop runs over the 5 frames of the video and fgbg.apply apply the background subtraction function to each frame.
OpenCV version that I use is opencv-2.4.9-3
As bikz05 suggested, running average method worked pretty good on my 5 images sets. Thanks for the tip !
import cv2
import numpy as np
c = cv2.VideoCapture('test.avi')
_,f = c.read()
avg1 = np.float32(f)
avg2 = np.float32(f)
# loop over images and estimate background
for x in range(0,4):
_,f = c.read()
cv2.accumulateWeighted(f,avg1,1)
cv2.accumulateWeighted(f,avg2,0.01)
res1 = cv2.convertScaleAbs(avg1)
res2 = cv2.convertScaleAbs(avg2)
cv2.imshow('img',f)
cv2.imshow('avg1',res1)
cv2.imshow('avg2',res2)
k = cv2.waitKey(0) & 0xff
if k == 5:
break