We Keep Coding

Detect squares (paintings) in images and draw contour around them using python

I'm trying to detect and draw a rectangular contour on every painting on for example this image:
I followed some guides and did the following:
Grayscale conversion
Applied median blur
Sharpen image
Applied adaptive Threshold
Applied Morphological Gradient
Find contours
Draw contours
And got the following result:
I know it's messy but is there a way to somehow detect and draw a contour around the paintings better?
Here is the code I used:
path = '<PATH TO THE PICTURE>'
#reading in and showing original image
image = cv2.imread(path)
image = cv2.resize(image,(880,600)) # resize was nessecary because of the large images
cv2.imshow("original", image)
cv2.waitKey(0)
cv2.destroyAllWindows()
# grayscale conversion
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
cv2.imshow("painting_gray", gray)
cv2.waitKey(0)
cv2.destroyAllWindows()
# we need to find a way to detect the edges better so we implement a couple of things
# A little help was found on stackoverflow: https://stackoverflow.com/questions/55169645/square-detection-in-image
median = cv2.medianBlur(gray,5)
cv2.imshow("painting_median_blur", median) #we use median blur to smooth the image
cv2.waitKey(0)
cv2.destroyAllWindows()
# now we sharpen the image with help of following URL: https://www.analyticsvidhya.com/blog/2021/08/sharpening-an-image-using-opencv-library-in-python/
kernel = np.array([[0, -1, 0],
[-1, 5,-1],
[0, -1, 0]])
image_sharp = cv2.filter2D(src=median, ddepth=-1, kernel=kernel)
cv2.imshow('painting_sharpend', image_sharp)
cv2.waitKey(0)
cv2.destroyAllWindows()
# now we apply adapptive thresholding
# thresholding: https://opencv24-python-tutorials.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_thresholding/py_thresholding.html#adaptive-thresholding
thresh = cv2.adaptiveThreshold(src=image_sharp,maxValue=255,adaptiveMethod=cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
thresholdType=cv2.THRESH_BINARY,blockSize=61,C=20)
cv2.imshow('thresholded image', thresh)
cv2.waitKey(0)
cv2.destroyAllWindows()
# lets apply a morphological transformation
kernel = np.ones((7,7),np.uint8)
gradient = cv2.morphologyEx(thresh, cv2.MORPH_GRADIENT, kernel)
cv2.imshow('dilated image', gradient)
cv2.waitKey(0)
cv2.destroyAllWindows()
# # lets now find the contours of the image
# # find contours: https://docs.opencv.org/4.x/dd/d49/tutorial_py_contour_features.html
contours, hierarchy = cv2.findContours(gradient, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
print("contours: ", len(contours))
print("hierachy: ", len(hierarchy))
print(hierarchy)
cv2.drawContours(image, contours, -1, (0,255,0), 3)
cv2.imshow("contour image", image)
cv2.waitKey(0)
cv2.destroyAllWindows()
Tips, help or code is appreciated!

Here's a simple approach:
Obtain binary image. We load the image, grayscale, Gaussian blur, then Otsu's threshold to obtain a binary image.
Two pass dilation to merge contours. At this point, we have a binary image but individual separated contours. Since we can assume that a painting is a single large square contour, we can merge small individual adjacent contours together to form a single contour. To do this, we create a vertical and horizontal kernel using cv2.getStructuringElement then dilate to merge them together. Depending on the image, you may need to adjust the kernel sizes or number of dilation iterations.
Detect paintings. Now we find contours and filter using contour area using a minimum threshold area to filter out small contours. Finally we obtain the bounding rectangle coordinates and draw the rectangle with cv2.rectangle.
Code
import cv2
# Load image, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread('1.jpeg')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (13,13), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]
# Two pass dilate with horizontal and vertical kernel
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (9,5))
dilate = cv2.dilate(thresh, horizontal_kernel, iterations=2)
vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,9))
dilate = cv2.dilate(dilate, vertical_kernel, iterations=2)
# Find contours, filter using contour threshold area, and draw rectangle
cnts = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
area = cv2.contourArea(c)
if area > 20000:
x,y,w,h = cv2.boundingRect(c)
cv2.rectangle(image, (x, y), (x + w, y + h), (36, 255, 12), 3)
cv2.imshow('thresh', thresh)
cv2.imshow('dilate', dilate)
cv2.imshow('image', image)
cv2.waitKey()

So here is the actual size of the portrait frame.
So here is small code.
#!/usr/bin/python 37
#OpenCV 4.3.0, Raspberry Pi 3/B/4B-w/4/8GB RAM, Buster,v10.
#Date: 3rd, June, 2020
import cv2
# Load the image
img = cv2.imread('portrait.jpeg')
# convert to grayscale
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
edged = cv2.Canny(img, 120,890)
# Apply adaptive threshold
thresh = cv2.adaptiveThreshold(edged, 255, 1, 1, 11, 2)
thresh_color = cv2.cvtColor(thresh, cv2.COLOR_GRAY2BGR)
# apply some dilation and erosion to join the gaps - change iteration to detect more or less area's
thresh = cv2.dilate(thresh,None,iterations = 50)
thresh = cv2.erode(thresh,None,iterations = 50)
# Find the contours
contours,hierarchy = cv2.findContours(thresh,
cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
# For each contour, find the bounding rectangle and draw it
for cnt in contours:
area = cv2.contourArea(cnt)
if area > 20000:
x,y,w,h = cv2.boundingRect(cnt)
cv2.rectangle(img,
(x,y),(x+w,y+h),
(0,255,0),
2)
cv2.imshow('img',img)
cv2.waitKey(0)
cv2.destroyAllWindows()
Here is output:

Using OpenCV to find a rotated rectangle of a certain aspect ratio?

I am not having difficulty transforming a found box, it is the fact that I am not able to detect the box in the first place when it is at an angle.
Here is a sample image I want the largest ~1230:123 rectangle in the image the problem is the rectangle can be rotated.
Here is a picture of a rotated barcode that I am unable to detect:
The function I have been using to process uses contour area just looks for the largest rectangle.
What methods should I use to look for a rotated rectangle so that even when rotated I can detect it?
#PYTHON 3.6 Snippet for Image Processing
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# compute the Scharr gradient magnitude representation of the images
# in both the x and y direction using OpenCV 2.4
ddepth = cv2.cv.CV_32F if imutils.is_cv2() else cv2.CV_32F
gradX = cv2.Sobel(gray, ddepth=ddepth, dx=1, dy=0, ksize=-1)
gradY = cv2.Sobel(gray, ddepth=ddepth, dx=0, dy=1, ksize=-1)
# subtract the y-gradient from the x-gradient
gradient = cv2.subtract(gradX, gradY)
gradient = cv2.convertScaleAbs(gradient)
# blur and threshold the image
blurred = cv2.blur(gradient, (8, 8))
(_, thresh) = cv2.threshold(blurred, 225, 255, cv2.THRESH_BINARY)
# construct a closing kernel and apply it to the thresholded image
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (21, 7))
closed = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# perform a series of erosions and dilations
closed = cv2.erode(closed, None, iterations = 4)
closed = cv2.dilate(closed, None, iterations = 4)
# find the contours in the thresholded image, then sort the contours
# by their area, keeping only the largest one
cnts = cv2.findContours(closed.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
c = sorted(cnts, key = cv2.contourArea, reverse = True)[0]
# compute the rotated bounding box of the largest contour
rect = cv2.minAreaRect(c)

You don't need all the preprocessing (like Sobel, erode, dilate) for finding before executing findContours.
findContours works better when contours are full (filled with white color) instead of having just the edges.
I suppose you can keep the code from cv2.findContours to the end, and get the result you are looking for.
You may use the following stages:
Apply binary threshold using Otsu's thresholding (just in case image is not a binary image).
Execute cv2.findContours, and Find the contour with the maximum area.
Use cv2.minAreaRect for finding the minimum area bounding rectangle.
Here is a code sample:
import numpy as np
import cv2
img = cv2.imread('img.png', cv2.IMREAD_GRAYSCALE) # Read input image as gray-scale
ret, img = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) # Apply threshold using Otsu's thresholding (just in case image is not a binary image).
# Find contours in img.
cnts = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2] # [-2] indexing takes return value before last (due to OpenCV compatibility issues).
# Find the contour with the maximum area.
c = max(cnts, key=cv2.contourArea)
# Find the minimum area bounding rectangle
# https://stackoverflow.com/questions/18207181/opencv-python-draw-minarearect-rotatedrect-not-implemented
rect = cv2.minAreaRect(c)
box = cv2.boxPoints(rect)
box = np.int0(box)
# Convert image to BGR (just for drawing a green rectangle on it).
bgr_img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
cv2.drawContours(bgr_img, [box], 0, (0, 255, 0), 2)
# Show images for debugging
cv2.imshow('bgr_img', bgr_img)
cv2.waitKey()
cv2.destroyAllWindows()
Result:
Note: The largest contour seems to be a parallelogram and not a perfect rectangle.

How to enclose the irregular figure contour and fill it with with 5px dots in opencv using python?

This is my
I want to get this
but the problem is I am not able to enclose the contour and how should I add these dots?
Does Open cv have any such function to handle this?
So basically,
The first problem is how to enclose this image
Second, how to add Dots.
Thank you

Here is one way to do that in Python/OpenCV. However, I cannot close your dotted outline without connecting separate regions. But it will give you some idea how to proceed with most of what you want to do.
If you manually add a few more dots to your input image where there are large gaps, then the morphology kernel can be made smaller such that it can connected the regions without merging separate parts that should remain isolated.
Read the input
Convert to grayscale
Threshold to binary
Apply morphology close to try to close the dotted outline. Unfortunately it connected separate regions.
Get the external contours
Draw white filled contours on a black background as a mask
Draw a single black circle on a white background
Tile out the circle image to the size of the input
Mask the tiled circle image with the filled contour image
Save results
Input:
import cv2
import numpy as np
import math
# read input image
img = cv2.imread('island.png')
hh, ww = img.shape[:2]
# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# threshold
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY)[1]
# use morphology to close figure
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (35,35))
morph = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, )
# find contours and bounding boxes
mask = np.zeros_like(thresh)
contours = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
for cntr in contours:
cv2.drawContours(mask, [cntr], 0, 255, -1)
# create a single tile as black circle on white background
circle = np.full((11,11), 255, dtype=np.uint8)
circle = cv2.circle(circle, (7,7), 3, 0, -1)
# tile out the tile pattern to the size of the input
numht = math.ceil(hh / 11)
numwd = math.ceil(ww / 11)
tiled_circle = np.tile(circle, (numht,numwd))
tiled_circle = tiled_circle[0:hh, 0:ww]
# composite tiled_circle with mask
result = cv2.bitwise_and(tiled_circle, tiled_circle, mask=mask)
# save result
cv2.imwrite("island_morph.jpg", morph)
cv2.imwrite("island_mask.jpg", mask)
cv2.imwrite("tiled_circle.jpg", tiled_circle)
cv2.imwrite("island_result.jpg", result)
# show images
cv2.imshow("morph", morph)
cv2.imshow("mask", mask)
cv2.imshow("tiled_circle", tiled_circle)
cv2.imshow("result", result)
cv2.waitKey(0)
Morphology connected image:
Contour Mask image:
Tiled circles:
Result:

Finding the corners of a rectangle

I'm trying to get the corners of this rectangle:
.
I tried using cv2.cornerHarris(rectangle, 2, 3, 0.04), but the left edges are not showed due to image brightness, I guess. So I tried applying a threshold before using cornerHarris, but the image produced showed a lot of vertices along the edges, not being possible to filter the corners.
I know that I need to filter it before using cornerHarris, but I don't know how. Could someone help me with this problem?
Ps. I've already tried to use blur, but it also doesn't work.
import cv2
import numpy as np
import matplotlib.pyplot as plt
rectangle = cv2.imread('rectangle.png', cv2.IMREAD_GRAYSCALE)
rectangle = np.where(rectangle > np.mean(rectangle), 255, 0).astype(np.uint8)
dst_rectangle = cv2.cornerHarris(rectangle, 2, 3, 0.04)
dst_rectangle = cv2.dilate(dst_rectangle, None)
mask = np.where(dst_rectangle > 0.01*np.max(dst_rectangle), 255, 0).astype(np.uint8)
points = np.nonzero(mask)
plt.imshow(dst_rectangle, cmap='gray')
plt.plot(points[1], points[0], 'or')
plt.show()

I would approach it differently by getting the corners of the rotated bounding box of the contour after adaptive thresholding. Here is my code in Python/OpenCV.
Input:
import cv2
import numpy as np
# read image
img = cv2.imread("rectangle.png")
# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = 255-gray
# do adaptive threshold on gray image
thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 17, 1)
thresh = 255-thresh
# apply morphology
kernel = np.ones((3,3), np.uint8)
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)
# separate horizontal and vertical lines to filter out spots outside the rectangle
kernel = np.ones((7,3), np.uint8)
vert = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel)
kernel = np.ones((3,7), np.uint8)
horiz = cv2.morphologyEx(morph, cv2.MORPH_OPEN, kernel)
# combine
rect = cv2.add(horiz,vert)
# thin
kernel = np.ones((3,3), np.uint8)
rect = cv2.morphologyEx(rect, cv2.MORPH_ERODE, kernel)
# get largest contour
contours = cv2.findContours(rect, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
for c in contours:
area_thresh = 0
area = cv2.contourArea(c)
if area > area_thresh:
area = area_thresh
big_contour = c
# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(big_contour)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
print(box)
# draw rotated rectangle on copy of img
rot_bbox = img.copy()
cv2.drawContours(rot_bbox,[box],0,(0,0,255),2)
# write img with red rotated bounding box to disk
cv2.imwrite("rectangle_thresh.png", thresh)
cv2.imwrite("rectangle_outline.png", rect)
cv2.imwrite("rectangle_bounds.png", rot_bbox)
# display it
cv2.imshow("IMAGE", img)
cv2.imshow("THRESHOLD", thresh)
cv2.imshow("MORPH", morph)
cv2.imshow("VERT", vert)
cv2.imshow("HORIZ", horiz)
cv2.imshow("RECT", rect)
cv2.imshow("BBOX", rot_bbox)
cv2.waitKey(0)
Thresholded Image:
Rectangle Region Extracted:
Rotated Bounding Box on Image:
Rotated Bounding Box Corners:
[[446 335]
[163 328]
[168 117]
[451 124]]
ADDITION:
Here is a slightly shorter version of the code, which is achievable by adding some gaussian blurring before thresholding.
import cv2
import numpy as np
# read image
img = cv2.imread("rectangle.png")
# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray = 255-gray
# blur image
blur = cv2.GaussianBlur(gray, (3,3), 0)
# do adaptive threshold on gray image
thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 75, 2)
thresh = 255-thresh
# apply morphology
kernel = np.ones((5,5), np.uint8)
rect = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
rect = cv2.morphologyEx(rect, cv2.MORPH_CLOSE, kernel)
# thin
kernel = np.ones((5,5), np.uint8)
rect = cv2.morphologyEx(rect, cv2.MORPH_ERODE, kernel)
# get largest contour
contours = cv2.findContours(rect, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
for c in contours:
area_thresh = 0
area = cv2.contourArea(c)
if area > area_thresh:
area = area_thresh
big_contour = c
# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(big_contour)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
for p in box:
pt = (p[0],p[1])
print(pt)
# draw rotated rectangle on copy of img
rot_bbox = img.copy()
cv2.drawContours(rot_bbox,[box],0,(0,0,255),2)
# write img with red rotated bounding box to disk
cv2.imwrite("rectangle_thresh.png", thresh)
cv2.imwrite("rectangle_outline.png", rect)
cv2.imwrite("rectangle_bounds.png", rot_bbox)
# display it
cv2.imshow("IMAGE", img)
cv2.imshow("THRESHOLD", thresh)
cv2.imshow("RECT", rect)
cv2.imshow("BBOX", rot_bbox)
cv2.waitKey(0)
Thresholded Image:
Rectangle Region Extracted:
Rotated Bounding Box on Image:
Rotated Bounding Box Corners:
(444, 335)
(167, 330)
(170, 120)
(448, 125)

Here's a simple approach:
Obtain binary image. We load the image, grayscale, Gaussian blur, then adaptive threshold.
Morphological operations. We create a rectangular kernel and morph open to remove the small noise
Find distorted rectangle contour and draw onto a mask. Find contours, determine rotated bounding box, and draw onto a blank mask
Find corners. We use the Shi-Tomasi Corner Detector already implemented as cv2.goodFeaturesToTrack which is supposedly shows better results compared to the Harris Corner Detector
Here's a visualization of each step:
Binary image
Morph open
Find rotated rectangle contour and draw/fill onto a blank mask
Draw rotated rectangle and corners to get result
Corner coordinates
(448.0, 337.0)
(164.0, 332.0)
(452.0, 123.0)
(168.0, 118.0)
Code
import cv2
import numpy as np
# Load image, grayscale, Gaussian blur, adaptive threshold
image = cv2.imread("1.png")
mask = np.zeros(image.shape, dtype=np.uint8)
gray = 255 - cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (3,3), 0)
thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY_INV, 51, 3)
# Morph open
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)
# Find distorted rectangle contour and draw onto a mask
cnts = cv2.findContours(opening, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
rect = cv2.minAreaRect(cnts[0])
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image,[box],0,(36,255,12),2)
cv2.fillPoly(mask, [box], (255,255,255))
# Find corners on the mask
mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
corners = cv2.goodFeaturesToTrack(mask, maxCorners=4, qualityLevel=0.5, minDistance=150)
for corner in corners:
x,y = corner.ravel()
cv2.circle(image,(x,y),8,(255,120,255),-1)
print("({}, {})".format(x,y))
cv2.imshow("thresh", thresh)
cv2.imshow("opening", opening)
cv2.imshow("mask", mask)
cv2.imshow("image", image)
cv2.waitKey(0)

You can try with an adaptive threshold. Then you may either use cornerHarris if you only need corners, or depending on what you need to do next, you could also find useful findContours, which returns a list of bounding boxes

I was able to locate 3 out of the 4 points, the 4th point can be found easily given the other three points since it's rectangle. Here is my solution:
import cv2
import numpy as np
img = cv2.imread('6dUIr.png',1)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
#smooth the image
kernel = np.ones((5,5),np.float32)/25
gray = cv2.filter2D(gray,-1,kernel)
#histogram equalization
clahe = cv2.createCLAHE(clipLimit=1.45, tileGridSize=(4,4))
cl1 = clahe.apply(gray)
#find edges
edges = cv2.Canny(cl1,4,100)
#find corners
dst = cv2.cornerHarris(edges,2,3,0.04)
#result is dilated for marking the corners, not important
dst = cv2.dilate(dst,None)
# Threshold for an optimal value, it may vary depending on the image.
img[dst>0.25*dst.max()]=[0,0,255]
cv2.imshow('edges', edges)
cv2.imshow('output', img)
# cv2.imshow('Histogram equalized', img_output)
cv2.waitKey(0)
The code has many hard coded thresholds but it's a good start.

How to approximate jagged edges as lines using Python OpenCV?

I am trying to find accurate locations for the corners on ink blotches as seen below:
My idea is to fit lines to the edges and then find where they intersect. As of now, I've tried using cv2.approxPolyDP() with various values of epsilon to approximate the edges, however this doesn't look like the way to go. My cv2.approxPolyDP code gives the following result:
Ideally, this is what I want to produce (drawn on paint):
Are there CV functions in place for this sort of problem? I've considered using Gaussian blurring before the threshold step although that method does not seem like it would be very accurate for corner finding. Additionally, I would like this to be robust to rotated images, so filtering for vertical and horizontal lines won't necessarily work without other considerations.
Code:*
import numpy as np
from PIL import ImageGrab
import cv2
def process_image4(original_image): # Douglas-peucker approximation
# Convert to black and white threshold map
gray = cv2.cvtColor(original_image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
(thresh, bw) = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
# Convert bw image back to colored so that red, green and blue contour lines are visible, draw contours
modified_image = cv2.cvtColor(bw, cv2.COLOR_GRAY2BGR)
contours, hierarchy = cv2.findContours(bw, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(modified_image, contours, -1, (255, 0, 0), 3)
# Contour approximation
try: # Just to be sure it doesn't crash while testing!
for cnt in contours:
epsilon = 0.005 * cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon, True)
# cv2.drawContours(modified_image, [approx], -1, (0, 0, 255), 3)
except:
pass
return modified_image
def screen_record():
while(True):
screen = np.array(ImageGrab.grab(bbox=(100, 240, 750, 600)))
image = process_image4(screen)
cv2.imshow('window', image)
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
screen_record()
A note about my code: I'm using screen capture so that I can process these images live. I have a digital microscope that can display live feed on a screen, so the constant screen recording will allow me to sample from the video feed and locate the corners live on the other half of my screen.

Here's a potential solution using thresholding + morphological operations:
Obtain binary image. We load the image, blur with cv2.bilateralFilter(), grayscale, then Otsu's threshold
Morphological operations. We perform a series of morphological open and close to smooth the image and remove noise
Find distorted approximated mask. We find the bounding rectangle coordinates of the object with cv2.arcLength() and cv2.approxPolyDP() then draw this onto a mask
Find corners. We use the Shi-Tomasi Corner Detector already implemented as cv2.goodFeaturesToTrack() for corner detection. Take a look at this for an explanation of each parameter
Here's a visualization of each step:
Binary image -> Morphological operations -> Approximated mask -> Detected corners
Here are the corner coordinates:
(103, 550)
(1241, 536)
Here's the result for the other images
(558, 949)
(558, 347)
Finally for the rotated image
(201, 99)
(619, 168)
Code
import cv2
import numpy as np
# Load image, bilaterial blur, and Otsu's threshold
image = cv2.imread('1.png')
mask = np.zeros(image.shape, dtype=np.uint8)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.bilateralFilter(gray,9,75,75)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
# Perform morpholgical operations
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (10,10))
opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)
close = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, kernel, iterations=1)
# Find distorted rectangle contour and draw onto a mask
cnts = cv2.findContours(close, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
rect = cv2.minAreaRect(cnts[0])
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image,[box],0,(36,255,12),4)
cv2.fillPoly(mask, [box], (255,255,255))
# Find corners
mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
corners = cv2.goodFeaturesToTrack(mask,4,.8,100)
offset = 25
for corner in corners:
x,y = corner.ravel()
cv2.circle(image,(x,y),5,(36,255,12),-1)
x, y = int(x), int(y)
cv2.rectangle(image, (x - offset, y - offset), (x + offset, y + offset), (36,255,12), 3)
print("({}, {})".format(x,y))
cv2.imshow('image', image)
cv2.imshow('thresh', thresh)
cv2.imshow('close', close)
cv2.imshow('mask', mask)
cv2.waitKey()
Note: The idea for the distorted bounding box came from a previous answer in How to find accurate corner positions of a distorted rectangle from blurry image

After seeing the description of the corners, here is what I would recommend:
by any method, find the gross location of the corners (for instance by looking for the extreme values of (±X+±Y, ±X+±Y) or (±X, ±Y)).
consider a strip than joins two corners, with a certain width. Extract the pixels in that strip, on a portion close to the corner, rotate to make it horizontal and average the values along horizontals.
you will obtain a gray profile that tells you the accurate position of the edge, at the mean between the background and foreground intensities.
repeat on all four edges and at both ends. This will give you four accurate corners, by intersection.