We Keep Coding

Detecting handwritten boxes using OpenCV

I have the following image:
I want to extract the boxed diagrams as so:
Here's what I've attempted:
import cv2
import matplotlib.pyplot as plt
# Load the image
image = cv2.imread('diagram.jpg')
# Convert to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# Apply thresholding to create a binary image
_, thresh = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY_INV)
# Find contours
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# Draw the contours
cv2.drawContours(image, contours, -1, (0, 0, 255), 2)
# Show the final image
plt.imshow(image), plt.show()
However, I've realized it'll be difficult to extract the diagrams because the contours aren't closed:
I've tried using morphological closing to close the gaps in the box edges:
# Define a rectangular kernel for morphological closing
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
# Perform morphological closing to close the gaps in the box edges
closed = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
But this changes almost nothing. How should I approach this problem?

We may replace morphological closing with dilate then erode, but filling the contours between the dilate and erode.
For filling the gaps, the kernel size should be much larger than 5x5 (I used 51x51).
Assuming the handwritten boxes are colored, we may convert from BGR to HSV, and apply the threshold on the saturation channel of HSV:
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) # Convert from BGR to HSV color space
gray = hsv[:, :, 1] # Use saturation from HSV channel as "gray".
_, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU) # Apply automatic thresholding (use THRESH_OTSU).
Apply dilate with large kernel, and use drawContours for filling the contours:
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (51, 51)) # Use relatively large kernel for closing the gaps
dilated = cv2.dilate(thresh, kernel) # Dilate with large kernel
contours, hierarchy = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(dilated, contours, -1, 255, -1)
Apply erode after filling the contours
Erode after dilate is equivalent to closing, but here we are closing after filling.
closed = cv2.erode(dilated, kernel)
Code sample:
import cv2
import numpy as np
# Load the image
image = cv2.imread('diagram.png')
hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) # Convert from BGR to HSV color space
# Convert to grayscale
#gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = hsv[:, :, 1] # Use saturation from HSV channel as "gray".
# Apply thresholding to create a binary image
_, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU) # Apply automatic thresholding (use THRESH_OTSU).
thresh = np.pad(thresh, ((100, 100), (100, 100))) # Add zero padding (required due to large dilate kernels).
# Define a rectangular kernel for morphological operations.
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (51, 51)) # Use relatively large kernel for closing the gaps
dilated = cv2.dilate(thresh, kernel) # Dilate with large kernel
# Fill the contours, before applying erode.
contours, hierarchy = cv2.findContours(dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(dilated, contours, -1, 255, -1)
closed = cv2.erode(dilated, kernel) # Apply erode after filling the contours.
closed = closed[100:-100, 100:-100] # Remove the padding.
# Find contours
contours, hierarchy = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
# Draw the contours
cv2.drawContours(image, contours, -1, (255, 0, 0), 2)
# Show images for testing
# plt.imshow(image), plt.show()
cv2.imshow('gray', gray)
cv2.imshow('thresh', thresh)
cv2.imshow('dilated', dilated)
cv2.imshow('closed', closed)
cv2.imshow('image', image)
cv2.waitKey()
cv2.destroyAllWindows()
Result:
gray (saturation channel):
thresh:
dilated (after filling):
closed:

Just need to dilate the image to make the rectangle closed, then define a threshold for the area of the contours:
import cv2
# Load the image
image = cv2.imread('diagram.jpg')
# Convert to grayscale
gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
# Apply thresholding to create a binary image
ret,thresh = cv2.threshold(gray,200,255,1)
# Need to dilate the image to make the contours closed
dilate = cv2.dilate(thresh,None)
erode = cv2.erode(dilate,None)
# Find contours
contours,hierarchy = cv2.findContours(erode,cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
for i,cnt in enumerate(contours):
# Check if it is an external contour and its area is more than 8000
if hierarchy[0,i,3] == -1 and cv2.contourArea(cnt)>8000:
x,y,w,h = cv2.boundingRect(cnt)
cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,0),2)
cv2.imwrite('template {0}.jpg'.format(i), image[y:y+h,x:x+w])
cv2.imshow('img',image)
You will get :

Find contours in a specific area

I'm new to image processing. I have the following image :
I'm trying to get the contours shown in the green area (bottom of picture) but I only detect the large blue form. I tried to skeleton the forms but without success and this is the "best" result I could get. How can I achieve that ?
import cv2
import numpy as np
im = cv2.imread('picture.jpg', cv2.IMREAD_UNCHANGED)
imgray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(imgray, 127, 255, 0)
cv2.imshow("Thresh", thresh)
cv2.waitKey(0)
kernel = np.ones((15,15),np.uint8)
opening = cv2.morphologyEx(imgray, cv2.MORPH_OPEN, kernel)
gradient = cv2.morphologyEx(opening, cv2.MORPH_GRADIENT, kernel)
opening1 = cv2.morphologyEx(gradient, cv2.MORPH_OPEN, kernel)
closing = cv2.morphologyEx(opening1, cv2.MORPH_CLOSE, kernel)
opening2 = cv2.morphologyEx(closing, cv2.MORPH_OPEN, kernel)
cv2.imshow("Closing", closing)
cv2.waitKey(0)
#Detecting contours
contours, hierarchy = cv2.findContours(image=opening2, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
cv2.drawContours(image=im, contours=contours, contourIdx=-1, color=(255, 255, 0), thickness=2, lineType=cv2.LINE_AA)
cv2.imshow("Image final", im)
cv2.waitKey(0)

Not able to create mask on text in image

import cv2
import numpy as np
# Load image, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread('1.png')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (7,7), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
# Create rectangular structuring element and dilate
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
dilate = cv2.dilate(thresh, kernel, iterations=4)
cv2.imshow('dilate', dilate)
cv2.waitKey()
I am trying to mask text in image using this code. But it is working for only one image.

How to extract the stain from the image?

I have an image that has a stain (the green stain on the surface of the water), and my goal is to extract that stain. It was guiding me with this solution but it doesn't extract it properly. Is there any way to extract using Python?
Input image:
The attempt:
img = cv2.imread('/content/001.jpg')
# blur
blur = cv2.GaussianBlur(img, (3,3), 0)
# convert to hsv and get saturation channel
sat = cv2.cvtColor(blur, cv2.COLOR_BGR2HSV)[:,:,1]
# threshold saturation channel
thresh = cv2.threshold(sat, 90, 255, cv2.THRESH_BINARY)[1]
# apply morphology close and open to make mask
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
morph = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=1)
mask = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel, iterations=1)
# do OTSU threshold to get circuit image
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
otsu = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
# write black to otsu image where mask is black
otsu_result = otsu.copy()
otsu_result[mask==0] = 0
# write black to input image where mask is black
img_result = img.copy()
img_result[mask==0] = 0
The result:

I followed your approach but used the LAB color space instead.
img = cv2.imread(image_path)
# blur
blur = cv2.GaussianBlur(img, (3,3), 0)
# convert to LAB space and get a-channel
lab = cv2.cvtColor(blur, cv2.COLOR_BGR2LAB)
a = lab[:,:,1]
thresh = cv2.threshold(a, 95, 255, cv2.THRESH_BINARY_INV)[1]
thresh = cv2.threshold(lab[:,:,1], 95, 255, cv2.THRESH_BINARY)[1]
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel, iterations=1)
inv_morph = cv2.bitwise_not(morph)
img_result = img.copy()
img_result[inv_morph==0] = 0
cv2.imshow('Final result', img_result)
The result is not accurate. You can alter the threshold value and/or morphological kernel to get the desired result.

How to detect a contour or spot on the edge?

I'm trying to detect the black spots on the following image.
I use adaptive thresholding and use find contours in opencv. This method is successful for detecting most of the black spots inside the gray background. However, it's not able to detect the spots on the edges, simply because contour detection thinks the spots are part of the black background, see here:
Here is the code I used to get these contours:
import cv2
image_path = "cropped.png"
img = cv2.imread(image_path)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# do adaptive threshold on gray image
thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 101, 3)
# apply morphology open then close
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (1, 1))
blob = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (10, 10))
blob = cv2.morphologyEx(blob, cv2.MORPH_CLOSE, kernel)
# invert blob
blob = (255 - blob)
# Get contours
cnts, hierarchy = cv2.findContours(blob, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
result1 = img.copy()
cv2.drawContours(result1, cnts, -1, (0, 0, 255), 3)
cv2.imwrite("_Fail_Blob.png", result1)
Any suggestions on how to detect the black spots on the edges? Eventually looking for an algorithm to be able to output sth like the following:

You can use morphological operations for select spot:
By example:
import cv2
fn = 'IdTPp.jpg'
img = cv2.imread(fn)
img=cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
se=cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (45,45))
img2=cv2.morphologyEx(img, cv2.MORPH_CLOSE, se)
img3=cv2.absdiff(img, img2)
cv2.imshow("detected circles", img3)