I have some code in which you input a video, and then use background subtraction via cv2 to produce contours over a certain threshold, drawing a bounding box over them. On its own this acts simply to identify objects/motion in the video. I then go on to track the change in x and y over coordinate points of the contours to print on screen what direction the object is moving, drawing on a motion track behind the object to the frame.
However, the code cannot distinguish the different contours as separate objects. When only one object is present, it works great. When it detects more than one object/contour, the motion is all over the place as you would expect.
I have been doing some research and it seems optical flow might be the best solution, but I'm not really sure if it applies to this situation, nor how to integrate it into the code I already have.
Is optical flow the best solution, and how can I implement it into the code? I have read this page but it doesn't follow the background subtraction or contour finding that my code currently follows. The result I want is objects/contours being tracked separately, so I can filter out any moving in a fashion/direction I don't want. Below is a small bits from my code to demonstrate my method. I follow these outlines: https://pyimagesearch.com/2015/09/21/opencv-track-object-movement/
Any help appreciated! Thanks!
# example of bg subtraction
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (21, 21), 0)
frameDelta = cv2.absdiff(firstFrame, gray)
(_, thresh) = cv2.threshold(frameDelta, desired_th, 255, cv2.THRESH_BINARY)
thresh = cv2.dilate(thresh, kernel, iterations=2)
cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts) #cnts ie contours, getting the outline
# motion
c = max(cnts, key=cv2.contourArea) #find the largest contour (based on its area)
((x, y), radius) = cv2.minEnclosingCircle(c) #compute the minimum enclosing circle and the centroid of the object.
M = cv2.moments(c)
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
if radius > 10: #minimum pixel radius
cv2.circle(frame, (int(x), int(y)), int(radius),(0, 255, 255), 2) #draw the minimum enclosing circle surrounding the objec
cv2.circle(frame, center, 5, (0, 0, 255), -1) #draw the centroid
pts.appendleft(center) #update the list of pts containing the center (x, y)-coordinates of the object
for i in np.arange(1, len(pts)): # loop over the set of tracked points
if pts[i - 1] is None or pts[i] is None:
continue
if counter >= 5 and i == 10 and pts[i-10] is not None: #check if enough points have been accumulated in buffer
dX = pts[i-10][0] - pts[i][0] #compute difference between x and y
dY = pts[i-10][1] - pts[i][1]
(dirX, dirY) = ("", "") #reinitialise direction in txt variables
if np.abs(dX) > 10: #ensure significant movement in x direction
dirX = "East" if np.sign(dX) == 1 else "West"
if np.abs(dY) > 10: #eg 20 as in, 20 pixel difference. make smaller to detect smaller movements, and vice versa
dirY = "South" if np.sign(dY) == 1 else "North" # Python reverses eg top of frame is pixel 0 and bottom is eg 780
if dirX != "" and dirY != "": #handles when both directions are not empty
direction = "{}-{}".format(dirY, dirX) #ie north west
else: # otherwise, only one direction is non-empty
direction = dirX if dirX != "" else dirY
#compute the thickness of the line and draw the connecting lines
thickness = int(np.sqrt(buffer / float(i + 1)) * 2.5)
cv2.line(frame, pts[i - 1], pts[i], (0, 0, 255), thickness)
Related
I want to detect the circle and the five squares in this image:
This is the relevant part of the code I currently use:
# detect shapes in black-white RGB formatted cv2 image
def detect_shapes(img, approx_poly_accuracy=APPROX_POLY_ACCURACY):
res_dict = {
"rectangles": [],
"squares": []
}
vis = img.copy()
shape = img.shape
height, width = shape[0], shape[1]
total_area = height * width
# Morphological closing: get rid of holes
# img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5)))
# Morphological opening: get rid of extensions at the border of the objects
# img = cv2.morphologyEx(img, cv2.MORPH_OPEN, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (121, 121)))
img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# cv2.imshow('intermediate', img)
# cv2.waitKey(0)
contours, hierarchy = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
logging.info("Number of found contours for shape detection: {0}".format(len(contours)))
# vis = img.copy()
# cv2.drawContours(vis, contours, -1, (0, 255, 0), 2)
cv2.imshow('vis', vis)
cv2.waitKey(0)
for contour in contours:
area = cv2.contourArea(contour)
if area < MIN_SHAPE_AREA:
logging.warning("Area too small: {0}. Skipping.".format(area))
continue
if area > MAX_SHAPE_AREA_RATIO * total_area:
logging.warning("Area ratio too big: {0}. Skipping.".format(area / total_area))
continue
approx = cv2.approxPolyDP(contour, approx_poly_accuracy * cv2.arcLength(contour, True), True)
cv2.drawContours(vis, [approx], -1, (0, 0, 255), 2)
la = len(approx)
# find the center of the shape
M = cv2.moments(contour)
if M['m00'] == 0.0:
logging.warning("Unable to compute shape center! Skipping.")
continue
x = int(M['m10'] / M['m00'])
y = int(M['m01'] / M['m00'])
if la < 3:
logging.warning("Invalid shape detected! Skipping.")
continue
if la == 3:
logging.info("Triangle detected at position {0}".format((x, y)))
elif la == 4:
logging.info("Quadrilateral detected at position {0}".format((x, y)))
if approx.shape != (4, 1, 2):
raise ValueError("Invalid shape before reshape to (4, 2): {0}".format(approx.shape))
approx = approx.reshape(4, 2)
r_check, data = check_rect_or_square(approx)
blob_data = {"position": (x, y), "approx": approx}
blob_data.update(data)
if r_check == 2:
res_dict["squares"].append(blob_data)
elif r_check == 1:
res_dict["rectangles"].append(blob_data)
elif la == 5:
logging.info("Pentagon detected at position {0}".format((x, y)))
elif la == 6:
logging.info("Hexagon detected at position {0}".format((x, y)))
else:
logging.info("Circle, ellipse or arbitrary shape detected at position {0}".format((x, y)))
cv2.drawContours(vis, [contour], -1, (0, 255, 0), 2)
cv2.imshow('vis', vis)
cv2.waitKey(0)
logging.info("res_dict: {0}".format(res_dict))
return res_dict
The problem is: if I set the approx_poly_accuracy parameter too high, the circle is detected as a polygon (Hexagon or Octagon, for example). If I set it too low, the squares are not detected as squares, but as Pentagons, for example:
The red lines are the approximated contours, the green lines are the original contours. The text is detected as a completely wrong contour, it should never be approximated to this level (I don't care about the text so much, but if it is detected as a polygon with less than 5 vertices, it will be a false positive).
For a human, it is obvious that the left object is a circle and that the five objects on the right are squares, so there should be a way to make the computer realize that with high accuracy too. How do I modify this code to properly detect all objects?
What I already tried:
Apply filters like MedianFilter. It made things worse, because the rounded edges of the squares promoted them being detected as a polygon with more than four vertices.
Variate the approx_poly_accuracy parameter. There is no value that fits my purposes, considering that some other images might even have a some more noise.
Find an implementation of the RDP algorithm that allows me to specify an EXACT number of output points. This would allow me to to compute the suggested polygons for a certain number of points (for example in the range 3..10) and then calculate (A_1 + A_2) / A_common - 1 to use the area instead of the arc length as an accuracy, which would probably lead to a better result. I have not yet found a good implementation for that. I will now try to use a numerical solver method to dynamically figure out the correct epsilon parameter for RDP. The approach is not really clean and efficient though. I will post the results here as soon as available. If someone has a better approach, please let me know.
A possible approach would involve the calculation of some blob descriptors and filter blobs according to those properties. For example, you can compute the blob's aspect ratio, the (approximated) number of vertices and area. The steps are very straightforward:
Load the image and convert it to grayscale.
(Invert) Threshold the image. Let’s make sure the blobs are colored in white.
Get the binary image’s contours.
Compute two features: aspect ratio and number of vertices
Filter the blobs based on those features
Let’s see the code:
# Imports:
import cv2
import numpy as np
# Load the image:
fileName = "yh6Uz.png"
path = "D://opencvImages//"
# Reading an image in default mode:
inputImage = cv2.imread(path + fileName)
# Prepare a deep copy of the input for results:
inputImageCopy = inputImage.copy()
# Grayscale conversion:
grayscaleImage = cv2.cvtColor(inputImage, cv2.COLOR_BGR2GRAY)
# Threshold via Otsu:
_, binaryImage = cv2.threshold(grayscaleImage, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
# Find the blobs on the binary image:
contours, hierarchy = cv2.findContours(binaryImage, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# Store the bounding rectangles here:
circleData = []
squaresData = []
Alright. So far, I’ve loaded, thresholded and computed contours on the input image. Additionally, I’ve prepared two lists to store the bounding box of the squares and the circle. Let’s create the feature filter:
for i, c in enumerate(contours):
# Get blob perimeter:
currentPerimeter = cv2.arcLength(c, True)
# Approximate the contour to a polygon:
approx = cv2.approxPolyDP(c, 0.04 * currentPerimeter, True)
# Get polygon's number of vertices:
vertices = len(approx)
# Get the polygon's bounding rectangle:
(x, y, w, h) = cv2.boundingRect(approx)
# Compute bounding box area:
rectArea = w * h
# Compute blob aspect ratio:
aspectRatio = w / h
# Set default color for bounding box:
color = (0, 0, 255)
I loop through each contour and calculate the current blob’s perimeter and polygon approximation. This info is used to approximately compute the blob vertices. The aspect ratio calculation is very easy. I first get the blob’s bounding box and get its dimensions: top left corner (x, y), width and height. The aspect ratio is just the width divided by the height.
The squares and the circle a very compact. These means that their aspect ratio should be close to 1.0. However, the squares have exactly 4 vertices, while the (approximated) circle has more. I use this info to build a very basic feature filter. It first checks aspect ratio, area and then number of vertices. I use the difference between the ideal feature and the real feature. The parameter delta adjusts the filter tolerance. Be sure to also filter tiny blobs, use the area for this:
# Set minimum tolerable difference between ideal
# feature and actual feature:
delta = 0.15
# Set the minimum area:
minArea = 400
# Check features, get blobs with aspect ratio
# close to 1.0 and area > min area:
if (abs(1.0 - aspectRatio) < delta) and (rectArea > minArea):
print("Got target blob.")
# If the blob has 4 vertices, it is a square:
if vertices == 4:
print("Target is square")
# Save bounding box info:
tempTuple = (x, y, w, h)
squaresData.append(tempTuple)
# Set green color:
color = (0, 255, 0)
# If the blob has more than 6 vertices, it is a circle:
elif vertices > 6:
print("Target is circle")
# Save bounding box info:
tempTuple = (x, y, w, h)
circleData.append(tempTuple)
# Set blue color:
color = (255, 0, 0)
# Draw bounding rect:
cv2.rectangle(inputImageCopy, (int(x), int(y)), (int(x + w), int(y + h)), color, 2)
cv2.imshow("Rectangles", inputImageCopy)
cv2.waitKey(0)
This is the result. The squares are identified with a green rectangle and the circle with a blue one. Additionally, the bounding boxes are stored in squaresData and circleData respectively:
I am trying to track a basketball through a short clip using OpenCV. I am using code to help me try to find the correct upper and lower bounds for the color code, but the ball is of very similar color to the game clock near the bottom of the video. How can I cut this off in my object tracking code, so that the program does not simply track the clock? I am using code from this tutorial: https://www.pyimagesearch.com/2015/09/14/ball-tracking-with-opencv/
The code where I think this change would be made is in the following block:
# find contours in the mask and initialize the current
# (x, y) center of the ball
cnts = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
center = None
# only proceed if at least one contour was found
if len(cnts) > 0:
# find the largest contour in the mask, then use
# it to compute the minimum enclosing circle and
# centroid
c = max(cnts, key=cv2.contourArea)
((x, y), radius) = cv2.minEnclosingCircle(c)
M = cv2.moments(c)
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
# only proceed if the radius meets a minimum size
if radius > 10:
# draw the circle and centroid on the frame,
# then update the list of tracked points
cv2.circle(frame, (int(x), int(y)), int(radius),
(0, 255, 255), 2)
cv2.circle(frame, center, 5, (0, 0, 255), -1)
I know I haven't provided a MWE but I'm not sure how to do that in this case. I think my question is at least straightforward, if not simple.
Im working in python and opencv library.
I can threshold a camera capture, find contours (more than one) and draw.
But I have a problem. I try to identify those contours with an unique id or tag. (for example: Id: 1 , Id:2) to track them.
I need this contours use a persistent id.
The goal is draw a line and count more than one contour and sometimes more than one near contours converts in a big one.
Note: Im working with a depth camera and my image its an array of depth.
add a piece of code.
Thanks in advance.
countours = cv2.findContours(mask, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[1]
# only proceed if at least one contour was found
if len(countours) > 0:
# find the largest contour in the mask, then use
# it to compute the minimum enclosing circle and
# centroid
for (i,c) in enumerate(countours):
((x, y), radius) = cv2.minEnclosingCircle(c)
M = cv2.moments(c)
if M["m00"] > 0:
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
centerString = str(center)
x = (int(M["m10"] / M["m00"]))
y = (int(M["m01"] / M["m00"]))
else:
center = int(x), int(y)
if radius > 10:
# draw the circle and centroid on the frame,
cv2.circle(frame, (int(x), int(y)), int(radius),
(0, 255, 255), 2)
cv2.circle(frame, center, 5, (0, 0, 255), -1)
# then update the ponter trail
if self.previous_position:
cv2.line(self.trail, self.previous_position, center,
(255, 255, 255), 2)
cv2.add(self.trail, frame, frame)
print center
self.previous_position = center
if len(countours) < 1:
center = 0
self.trail = numpy.zeros((self.cam_height, self.cam_width, 3),
numpy.uint8)
self.previous_position = None
Two options. First off, the contours are already in a Python list, so the indices of that list can be used to enumerate them. In fact you're already doing that in some sense with (i,c) in enumerate(countours). You can just use the index i to 'color' each contour with the value i drawing on a blank image, and then you'll know which contour is which just by examining the image. The other, probably better option IMO, is to use cv2.connectedComponents() to label the binary images instead of the contours of the binary image. Also pre-labeling you might try morphological operations to close up blobs.
EDIT: I follow your recommendation and I have new problems :)
Label is not persistent. When I move both contours, numbers of labels change.
Add some pictures and my code.
Hand label 0 and circle 1
Hand label 1 and circle 0
while True:
cnt += 1
if (cnt % 10) == 0:
now = time.time()
dt = now - last
fps = 10/dt
fps_smooth = (fps_smooth * smoothing) + (fps * (1.0-smoothing))
last = now
c = dev.color
cad = dev.cad
dev.wait_for_frames()
c = cv2.cvtColor(c, cv2.COLOR_RGB2GRAY)
depth = dev.depth * dev.depth_scale * 1000
print depth
#d = cv2.applyColorMap(depth.astype(np.uint8), cv2.COLORMAP_SUMMER)
print depth
res = np.float32(dev.depth)
depth = 255 * np.logical_and(depth >= 100, depth <= 500)
res2 = depth.astype(np.uint8)
kernel = np.ones((3, 3), np.uint8)
#res2 = cv2.blur(res2,(15,15))
res2 = cv2.erode(res2, kernel, iterations=4)
res2 = cv2.dilate(res2, kernel, iterations=8)
im_floodfill = res2.copy()
h, w = res2.shape[:2]
mask2 = np.zeros((h + 2, w + 2), np.uint8)
cv2.floodFill(im_floodfill, mask2, (0, 0), 255)
im_floodfill_inv = cv2.bitwise_not(im_floodfill)
im_out = res2 | im_floodfill_inv
im_out = cv2.blur(im_out,(5,5))
label = cv2.connectedComponentsWithStats(im_out)
n = label[0] - 1
cog = np.delete(label[3], 0, 0)
for i in xrange(n):
# im = cv2.circle(im,(int(cog[i][0]),int(cog[i][1])), 10, (0,0,255), -1)
cv2.putText(im_out, str(i), (int(cog[i][0]), int(cog[i][1])), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)
cv2.imshow("Depth", res)
cv2.imshow("OUT", im_out)
cv2.imshow( "C", res2)
I'm using OpenCV 2.4 to do some tracking, and I can get a contour of the shape I want, which is a T.
Input image:
I can use cv2.minAreaRect(my_t_contour) and get the angle of that rect, but that only gives me 0-180 degrees. But this is a T shape though, so I want to be able to tell 0-360. I was thinking of:
Split the contour into two rects
Get a line through the rects (either using skeletonize > HoughLinesP)
Determine which line is which, determine their gradient (using the coordinates I get from HoughLinesP) and then determine the direction of the T.
But I'm stuck at number 1, how can I split a contour into two shapes?
Method 1: draw center of contour and center of minAreaRect of contour
dst = cv2.cvtColor(r_target, cv2.COLOR_BGR2GRAY)
dst = cv2.GaussianBlur(dst, (11, 11), 0)
ret,dst = cv2.threshold(dst,110,255,cv2.THRESH_BINARY_INV)
cnts = cv2.findContours(dst, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
for c in cnts:
# get minAreaRect around contour and draw its center in red
rect = cv2.minAreaRect(c)
cv2.circle(r_target, (int(rect[0][0]), int(rect[0][1])), 7, (0, 0, 255), -1)
# get moments of contour to get center and draw it in white
M = cv2.moments(c)
cX = int(M["m10"] / M["m00"])
cY = int(M["m01"] / M["m00"])
cv2.circle(r_target, (cX, cY), 7, (255, 255, 255), -1)
Next step would probably calculate a simple gradient between the centers to determine the angle.
Method 2: skeletonize the image and get lines using HoughLinesP.
dst = cv2.cvtColor(r_target, cv2.COLOR_BGR2GRAY)
dst = cv2.GaussianBlur(dst, (11, 11), 0)
ret,dst = cv2.threshold(dst,110,255,cv2.THRESH_BINARY)
dst = 1 - dst / 255
dst = skimage.morphology.skeletonize(dst).astype(np.uint8)
rho = 1
theta = np.pi / 180
threshold = 1
minLineLength = 30
maxLineGap = 15
lines = cv2.HoughLinesP(dst, rho, theta, threshold, minLineLength=minLineLength, maxLineGap=maxLineGap)
for line in lines[0]:
cv2.line(r_target, (line[0], line[1]), (line[2], line[3]), (0, 255, 0), 1, 8)
But the lines don't come out nicely. This is how the skeleton looks like:
I'm still experimenting with the variables but is there a specific thought process around using HoughLinesP?
As a variant you can use PCA, find first component direction, and use it as an searced angle. You can check here for an example: http://docs.opencv.org/trunk/d1/dee/tutorial_introduction_to_pca.html
So i have taken the code from Github of #bradmontgomer and trying to modify it. The code first converts the frame into HSV color space, split the video frame into color channels and then Performs an AND on HSV components to identify the laser. I am having trouble in finding the contours of the detected laser point. heres my code;
def threshold_image(self, channel):
if channel == "hue":
minimum = self.hue_min
maximum = self.hue_max
elif channel == "saturation":
minimum = self.sat_min
maximum = self.sat_max
elif channel == "value":
minimum = self.val_min
maximum = self.val_max
(t, tmp) = cv2.threshold(
self.channels[channel], # src
maximum, # threshold value
0, # we dont care because of the selected type
cv2.THRESH_TOZERO_INV #t type
)
(t, self.channels[channel]) = cv2.threshold(
tmp, # src
minimum, # threshold value
255, # maxvalue
cv2.THRESH_BINARY # type
)
if channel == 'hue':
# only works for filtering red color because the range for the hue is split
self.channels['hue'] = cv2.bitwise_not(self.channels['hue'])
def detect(self, frame):
# resize the frame, blur it, and convert it to the HSV
# color space
frame = imutils.resize(frame, width=600)
hsv_img = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
# split the video frame into color channels
h, s, v = cv2.split(hsv_img)
self.channels['hue'] = h
self.channels['saturation'] = s
self.channels['value'] = v
# Threshold ranges of HSV components; storing the results in place
self.threshold_image("hue")
self.threshold_image("saturation")
self.threshold_image("value")
# Perform an AND on HSV components to identify the laser!
self.channels['laser'] = cv2.bitwise_and(
self.channels['hue'],
self.channels['value']
)
self.channels['laser'] = cv2.bitwise_and(
self.channels['saturation'],
self.channels['laser']
)
# Merge the HSV components back together.
hsv_image = cv2.merge([
self.channels['hue'],
self.channels['saturation'],
self.channels['value'],
])
thresh = cv2.threshold(self.channels['laser'], 25, 255, cv2.THRESH_BINARY)[1]
#find contours in the mask and initialize the current
#(x, y) center of the ball
#cnts = cv2.findContours(self.channels['laser'].copy(), cv2.RETR_EXTERNAL,
#cv2.CHAIN_APPROX_SIMPLE)
(_, cnts, _) = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
center = None
# only proceed if at least one contour was found
if len(cnts) > 0:
# find the largest contour in the mask, then use
# it to compute the minimum enclosing circle and
# centroid
c = max(cnts, key=cv2.contourArea)
((x, y), radius) = cv2.minEnclosingCircle(c)
M = cv2.moments(c)
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
# only proceed if the radius meets a minimum size
if radius > 10:
# draw the circle and centroid on the frame,
# then update the list of tracked points
cv2.circle(frame, (int(x), int(y)), int(radius),
(0, 255, 255), 2)
cv2.circle(frame, center, 5, (0, 0, 255), -1)
cv2.imshow('LaserPointer', self.channels['laser'])
################################################
return hsv_image
I am getting the cnts greater then 0 in line "if len(cnts) > 0:", but can't see a circle drawn in the laser pointer.
There was another function (display()) that was displaying laser frame (self.channel['laser']),
def display(self, img, frame):
"""Display the combined image and (optionally) all other image channels
NOTE: default color space in OpenCV is BGR.
"""
cv2.imshow('RGB_VideoFrame', frame)
cv2.imshow('LaserPointer', self.channels['laser'])
I commented out these cv2.iamshow lines from this function and then I was able to see circle around the laser pointer. This is because now frame from cv2.iamshow line inside function "detect(self, frame):" was executed. I then applied further codings on the pointer to detect its location.