Related
im currently trying to implement a paper of reading pressure gauges. There is a step where i dont know what to do. It says: "The display contour is then rotated to vertically align the long
axis of the ellipse and inscribed in a rectangle used to crop the gauge image." So im not sure how i can rotate this image vertically, for better understanding i will show you a example what the current image is and what it needs to be.
Im currently at step C and need to get the image in position of step D. The text detection in the example is currently not important.
At the moment i have the correct contours and ellipse for the display.
(cnts, boundingBoxes) = sort_contours(cnts)
#find correct contours and fitellipse
if len(cnts) != 0:
for i in range(len(cnts)):
if len(cnts[i]) >= 5: #if contours has more than 5 points
# cv2.drawContours(image,cnts[0],-1,(150,10,255),3)
ellipse = cv2.fitEllipse(cnts[i])
finalElps.append(ellipse) #(centx,centy), (width,height), angle
for i in range(len(finalElps)):
centx = finalElps[i][0][0]
centy = finalElps[i][0][1]
eWidth = finalElps[i][1][0]
eHeight = finalElps[i][1][1]
sfRes = Sf(eWidth, eHeight)
cfRes = Cf(centx, imgCenterX, centy, imgCenterY)
afRes = Af(eWidth,eHeight,imgWidth,imgHeight)
print("SF: " + str(sfRes) + "| " + "CF: " + str(cfRes) + "| Af: " + str(afRes))
if(sfRes < 0.4 and cfRes < 6 and afRes < 0.9):
print(finalElps[i])
cv2.ellipse(image, finalElps[i], (255,0,0), 2)
plt.imshow(image)
sfRes, cfRes and afRes are just calculations to find the right ellipse.
What should be my next step to reach the vertical rotation? I think the correct name for it is "image rectification" but im not 100% sure about it
Here is how to get the image D from the image C:
Find the binary mask
Find an ellipse (center, width, height, angle)
Find 4 points on opposite sides of the ellipse
Use these 4 points to get perspective transform that can be used to warp the rotated gauge image to rectangular image. The final result:
Code:
import cv2
import numpy as np
image = cv2.imread("gauge.png")
# find the parameters of the ellipse
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
mask = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)[1]
points = np.stack(np.nonzero(mask.T)).T
hull = cv2.convexHull(points)
(cx, cy), (width, height), angle = cv2.fitEllipse(hull)
# for visualization
# cv2.ellipse(image, (int(cx), int(cy)), (int(width/2), int(height/2)), angle, 0, 360, (0, 0, 255), 2)
# find the points on the opposite sides of the ellipse
# define rectangular homogenuous coordinates and rotate them using a rotation matrix
mat = cv2.getRotationMatrix2D((cx, cy), -angle, 1)
mat = np.vstack((mat, [0, 0, 1]))
coords = np.array(
[
[cx + width // 2, cy, 1],
[cx - width // 2, cy, 1],
[cx, cy + height // 2, 1],
[cx, cy - height // 2, 1],
]
)
points = (mat # coords.T)[:2].T # drop the homogenuos part
# for visualization
# for px, py in points.astype(int)[:2]:
# cv2.circle(image, (px, py), 10, (0, 0, 255), -1)
# for px, py in points.astype(int)[2:]:
# cv2.circle(image, (px, py), 10, (255, 0, 0), -1)
# define points on the target image to which the ellipse points should be mapped
size = 300
target = np.float32(
[
[size, size // 2],
[0, size // 2],
[size // 2, size],
[size // 2, 0],
]
)
mat = cv2.getPerspectiveTransform(points.astype(np.float32), target)
rect_image = cv2.warpPerspective(image, mat, (size, size))
cv2.imwrite("rect_gauge.png", rect_image)
I am trying to recognize six digits from a meter using python-OpenCV. It's surprising how incredibly hard it is to set morphological operations working in the right way, given the time I have spent adjusting the focus/distance of my raspberry pi camera to the meter screen and I even have bought a separate led lamp to have as much uniform light as possible. This is a template image
and I've tried using and adjusting the code from these two sources: enter link description here and enter link description here reproduced below without any progress. I got stuck right at the start when setting the thresholding options. Thank you in advance for any help.
# Code 1
import cv2
import numpy as np
import pytesseract
# Load the image
img = cv2.imread("test.jpg")
# Color-segmentation to get binary mask
lwr = np.array([43, 0, 71])
upr = np.array([103, 255, 130])
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
msk = cv2.inRange(hsv, lwr, upr)
cv2.imwrite("msk.png", msk)
# Extract digits
krn = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 3))
dlt = cv2.dilate(msk, krn, iterations=5)
res = 255 - cv2.bitwise_and(dlt, msk)
cv2.imwrite("res.png", res)
# Displaying digits and OCR
txt = pytesseract.image_to_string(res, config="--psm 6 digits")
print(''.join(t for t in txt if t.isalnum()))
cv2.imshow("res", res)
cv2.waitKey(0)
cv2.destroyAllWindows()
# code 2
# https://pyimagesearch.com/2017/02/13/recognizing-digits-with-opencv-and-python/
# import the necessary packages
# from imutils.perspective import four_point_transform
from imutils import contours
import imutils
import cv2
import numpy as np
from numpy.linalg import norm
# define the dictionary of digit segments so we can identify
# each digit on the thermostat
DIGITS_LOOKUP = {
(1, 1, 1, 0, 1, 1, 1): 0,
(1, 0, 1, 0, 1, 0, 1): 1,
(1, 0, 1, 1, 1, 0, 1): 2,
(1, 0, 1, 1, 0, 1, 1): 3,
(0, 1, 1, 1, 0, 1, 0): 4,
(1, 1, 0, 1, 0, 1, 1): 5,
(1, 1, 0, 1, 1, 1, 1): 6,
(1, 1, 1, 0, 0, 1, 0): 7,
(1, 1, 1, 1, 1, 1, 1): 8,
(1, 1, 1, 1, 0, 1, 1): 9
}
images = 'test.jpg'
image = cv2.imread(images, 1)
# pre-process the image by resizing it, converting it to
# graycale, blurring it, and computing an edge map
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blurred = cv2.GaussianBlur(gray, (3, 3), 0)
# gray = cv2.medianBlur(blurred, 1)
# threshold the warped image, then apply a series of morphological
# operations to cleanup the thresholded image
(T, thresh) = cv2.threshold(blurred, 0, 255,
cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
cv2.imshow('thresh', thresh)
cv2.waitKey(0)
cv2.destroyAllWindows()
mask = np.zeros((image.shape[0] + 2, image.shape[1] + 2), np.uint8)
cv2.floodFill(thresh, mask, (0, 0), 0)
cv2.floodFill(thresh, mask, (image.shape[1]-1, 0), 0)
cv2.floodFill(thresh, mask, (round(image.shape[1]/2.4), 0), 0)
cv2.floodFill(thresh, mask, (image.shape[1]//2, 0), 0)
cv2.floodFill(thresh, mask, (0, image.shape[0]-1), 0)
cv2.floodFill(thresh, mask, (image.shape[1]-1, image.shape[0]-1), 0)
kernel = np.ones((2, 2), np.uint8)
thresh = cv2.erode(thresh, kernel, iterations=2)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 13))
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=2)
# cv2.imshow('thresh', thresh)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# find contours in the thresholded image, then initialize the
# digit contours lists
cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
digitCnts = []
# loop over the digit area candidates
for c in cnts:
# compute the bounding box of the contour
(x, y, w, h) = cv2.boundingRect(c)
# if the contour is sufficiently large, it must be a digit
if w <= 300 and (h >= 130 and h <= 300):
digitCnts.append(c)
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 255), 2)
# cv2.imshow('image', image)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# sort the contours from left-to-right, then initialize the
# actual digits themselves
digitCnts = contours.sort_contours(digitCnts, method="left-to-right")[0]
digits = []
clao = 0
# loop over each of the digits
for c in digitCnts:
clao = clao + 1
# extract the digit ROI
(x, y, w, h) = cv2.boundingRect(c)
roi = thresh[y:y + h, x:x + w]
# compute the width and height of each of the 7 segments
# we are going to examine
(roiH, roiW) = roi.shape
(dW, dH) = (int(roiW * 0.25), int(roiH * 0.15))
dHC = int(roiH * 0.05)
# define the set of 7 segments
segments = [
((0, 0), (w, dH)), # top
((0, 0), (dW, h // 2)), # top-left
((w - dW, 0), (w, h // 2)), # top-right
((0, (h // 2) - dHC), (w, (h // 2) + dHC)), # center
((0, h // 2), (dW, h)), # bottom-left
((w - dW, h // 2), (w, h)), # bottom-right
((0, h - dH), (w, h)) # bottom
]
on = [0] * len(segments)
# loop over the segments
for (i, ((xA, yA), (xB, yB))) in enumerate(segments):
# extract the segment ROI, count the total number of
# thresholded pixels in the segment, and then compute
# the area of the segment
segROI = roi[yA:yB, xA:xB]
total = cv2.countNonZero(segROI)
area = (xB - xA) * (yB - yA)
# if the total number of non-zero pixels is greater than
# 50% of the area, mark the segment as "on"
if clao == 1:
if total / float(area) > 0.34:
if area < 1500:
on = [1, 0, 1, 0, 1, 0, 1]
else:
on[i] = 1
else:
if total / float(area) > 0.39:
if area < 1500:
on = [1, 0, 1, 0, 1, 0, 1]
else:
on[i] = 1
# lookup the digit and draw it on the image
digit = DIGITS_LOOKUP.get(tuple(on)) or DIGITS_LOOKUP[
min(DIGITS_LOOKUP.keys(), key=lambda key: norm(np.array(key)-np.array(on)))]
# digit = DIGITS_LOOKUP[tuple(on)]
digits.append(digit)
# print(digits)
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 1)
cv2.putText(image, str(digit), (x - 10, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2)
# display the digits
print(digits)
cv2.imshow("Input", image)
cv2.waitKey(0)
cv2.destroyAllWindows()
Update
Apologies for my late reply but I have been quite busy with work.
I have captured 22 images throughout the day and used #fmw42 code (with some amendments) to apply thresholding and morphological operations. I am making the images available here and the code that I am using is available below. Overall the performance is quite robust, although 1s and sometimes 8s get mixed up with 2s. I am happy to accept a code that provides improved performance. Note: I think that one problem is that the vertical lines of the numbers are slightly slanted? Thank you in advance.
import cv2
import numpy as np
from numpy.linalg import norm
from imutils import contours
import imutils
import os
# define the dictionary of digit segments so we can identify
# each digit on the thermostat
DIGITS_LOOKUP = {
(1, 1, 1, 0, 1, 1, 1): 0,
(1, 0, 1, 0, 1, 0, 1): 1,
(1, 0, 1, 1, 1, 0, 1): 2,
(1, 0, 1, 1, 0, 1, 1): 3,
(0, 1, 1, 1, 0, 1, 0): 4,
(1, 1, 0, 1, 0, 1, 1): 5,
(1, 1, 0, 1, 1, 1, 1): 6,
(1, 1, 1, 0, 0, 1, 0): 7,
(1, 1, 1, 1, 1, 1, 1): 8,
(1, 1, 1, 1, 0, 1, 1): 9
}
path_of_the_directory = "/home/myusername/mypathdirectory"
ext = ('.jpg')
for files in os.listdir(path_of_the_directory):
if files.endswith(ext):
# load image
print(files)
img = cv2.imread(path_of_the_directory+files)
# convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# blur
blur = cv2.GaussianBlur(gray, (0,0), sigmaX=51, sigmaY=51)
# divide
divide = cv2.divide(gray, blur, scale=255)
# threshold
thresh = cv2.threshold(divide, 235, 255, cv2.THRESH_BINARY)[1]
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (41,41))
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (41,41))
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)
morph = cv2.bitwise_not(morph) # reverse
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (1, 70))
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)
# write result to disk
cv2.imwrite("digits_division.jpg", divide)
cv2.imwrite("digits_threshold.jpg", thresh)
cv2.imwrite("digits_morph.jpg", morph)
# display it
cv2.imshow("divide", divide)
cv2.imshow("thresh", thresh)
cv2.imshow("morph", morph)
cv2.waitKey(0)
cv2.destroyAllWindows()
# find contours in the thresholded image, then initialize the
# digit contours lists
cnts = cv2.findContours(morph.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
digitCnts = []
# loop over the digit area candidates
for c in cnts:
# compute the bounding box of the contour
(x, y, w, h) = cv2.boundingRect(c)
# if the contour is sufficiently large, it must be a digit
if w >= 60 and (h >= 300 and h <= 800):
digitCnts.append(c)
cv2.rectangle(img, (x, y), (x + w, y + h), (0, 0, 255), 2)
cv2.imshow('image', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
# sort the contours from left-to-right, then initialize the
# actual digits themselves
digitCnts = contours.sort_contours(digitCnts, method="left-to-right")[0]
digits = []
clao = 0
# loop over each of the digits
for c in digitCnts:
clao = clao + 1
# extract the digit ROI
(x, y, w, h) = cv2.boundingRect(c)
roi = morph[y:y + h, x:x + w]
# compute the width and height of each of the 7 segments
# we are going to examine
(roiH, roiW) = roi.shape
(dW, dH) = (int(roiW * 0.25), int(roiH * 0.15))
dHC = int(roiH * 0.05)
# define the set of 7 segments
segments = [
((0, 0), (w, dH)), # top
((0, 0), (dW, h // 2)), # top-left
((w - dW, 0), (w, h // 2)), # top-right
((0, (h // 2) - dHC), (w, (h // 2) + dHC)), # center
((0, h // 2), (dW, h)), # bottom-left
((w - dW, h // 2), (w, h)), # bottom-right
((0, h - dH), (w, h)) # bottom
]
on = [0] * len(segments)
# loop over the segments
for (i, ((xA, yA), (xB, yB))) in enumerate(segments):
# extract the segment ROI, count the total number of
# thresholded pixels in the segment, and then compute
# the area of the segment
segROI = roi[yA:yB, xA:xB]
total = cv2.countNonZero(segROI)
area = (xB - xA) * (yB - yA)
# if the total number of non-zero pixels is greater than
# 50% of the area, mark the segment as "on"
if clao == 1:
if total / float(area) > 0.34:
if area < 1500:
on = [1, 0, 1, 0, 1, 0, 1]
else:
on[i] = 1
else:
if total / float(area) > 0.42:
if area < 1500:
on = [1, 0, 1, 0, 1, 0, 1]
else:
on[i] = 1
# lookup the digit andq draw it on the image
digit = DIGITS_LOOKUP.get(tuple(on)) or DIGITS_LOOKUP[
min(DIGITS_LOOKUP.keys(), key=lambda key: norm(np.array(key)-np.array(on)))]
# digit = DIGITS_LOOKUP[tuple(on)]
digits.append(digit)
# print(digits)
cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 1)
cv2.putText(img, str(digit), (x - 10, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.65, (0, 255, 0), 2)
# display the digits
print(digits)
cv2.imshow("Input", img)
cv2.waitKey(0)
cv2.destroyAllWindows()
else:
continue
Perhaps this will help you using division normalization in Python/OpenCV.
Input:
import cv2
import numpy as np
# load image
img = cv2.imread("digits.jpg")
# convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# blur
blur = cv2.GaussianBlur(gray, (0,0), sigmaX=51, sigmaY=51)
# divide
divide = cv2.divide(gray, blur, scale=255)
# threshold
thresh = cv2.threshold(divide, 235, 255, cv2.THRESH_BINARY)[1]
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11,11))
morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11,11))
morph = cv2.morphologyEx(morph, cv2.MORPH_CLOSE, kernel)
# write result to disk
cv2.imwrite("digits_division.jpg", divide)
cv2.imwrite("digits_threshold.jpg", thresh)
cv2.imwrite("digits_morph.jpg", morph)
# display it
cv2.imshow("divide", divide)
cv2.imshow("thresh", thresh)
cv2.imshow("morph", morph)
cv2.waitKey(0)
cv2.destroyAllWindows()
Division normalized image:
Thresholded image:
Morphology processed image:
You can then clean up further by getting contours and removing small contours and very long horizontal contours.
The key to getting this working is cleaning the image up which I have done to a good enough level to get it to work. I've done this using scikit image library.
I then look at certain squares on the image and take an average reading from that area.
On the right hand-side image I've marked some of the locations with red squares.
My script I used to get this result:
import numpy as np
from pathlib import Path
import imageio.v3 as iio
import skimage.filters as skif
from skimage.color import rgb2gray
from skimage.util import img_as_ubyte
from skimage.restoration import denoise_bilateral
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle
import matplotlib.cm as cm
threshold = 125
digit_loc = [1600, 1300, 1000, 730, 420, 155]
size = 20
x_mid = 80
x_right = 160
y_top = 130
y_mt = 250
y_mid = 380
y_bm = 520
y_bot = 630
def img_with_threshold(orig_img):
block_size = 255
local_thresh = skif.threshold_local(
orig_img,
block_size,
method="mean",
)
binary_local = orig_img > local_thresh
u8_val = img_as_ubyte(binary_local)
return u8_val
def image_denoise(orig_img):
return denoise_bilateral(orig_img, win_size=10, bins=10, )
def plot_imgs(orig_img, mod_img):
# Display the image
fig, axes = plt.subplots(1, 2, figsize=(8, 8), sharex=True, sharey=True)
ax = axes.ravel()
ax[0].imshow(orig_img, cmap=cm.Greys_r)
ax[1].imshow(mod_img, cmap=cm.Greys_r)
# Create a Rectangle patch
for x_loc in digit_loc:
rect1 = Rectangle((x_loc + x_mid, y_top), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect2 = Rectangle((x_loc, y_mt), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect3 = Rectangle((x_loc + x_right, y_mt), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect4 = Rectangle((x_loc + x_mid, y_mid), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect5 = Rectangle((x_loc, y_bm), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect6 = Rectangle((x_loc + x_right, y_bm), size, size, linewidth=1, edgecolor='r', facecolor='none')
rect7 = Rectangle((x_loc + x_mid, y_bot), size, size, linewidth=1, edgecolor='r', facecolor='none')
# Add the patch to the Axes
ax[1].add_patch(rect1)
ax[1].add_patch(rect2)
ax[1].add_patch(rect3)
ax[1].add_patch(rect4)
ax[1].add_patch(rect5)
ax[1].add_patch(rect6)
ax[1].add_patch(rect7)
plt.show()
def seg_to_digit(segments, location):
digit_values = {0b1110111: 0,
0b0010010: 1,
0b1011101: 2,
0b1011011: 3,
0b0111010: 4,
0b1101011: 5,
0b1101111: 6,
0b1110010: 7,
0b1111111: 8,
0b1111011: 9,
}
result = int("".join(["1" if i < threshold else "0" for i in segments]), 2)
# print("score:", result)
return digit_values.get(result, 0) * 10 ** location
def get_digit(location, mod_img):
"""
a
b c
d
e f
g
"""
x_loc = digit_loc[location]
m_loc = (x_loc + x_mid, x_loc + x_mid + size)
l_loc = (x_loc, x_loc + size)
r_loc = (x_loc + x_right, x_loc + x_right + size)
seg_a = np.average(mod_img[y_top:y_top + size, m_loc[0]:m_loc[1]])
seg_b = np.average(mod_img[y_mt:y_mt + size, l_loc[0]:l_loc[1]])
seg_c = np.average(mod_img[y_mt:y_mt + size, r_loc[0]:r_loc[1]])
seg_d = np.average(mod_img[y_mid:y_mid + size, m_loc[0]:m_loc[1]])
seg_e = np.average(mod_img[y_bm:y_bm + size, l_loc[0]:l_loc[1]])
seg_f = np.average(mod_img[y_bm:y_bm + size, r_loc[0]:r_loc[1]])
seg_g = np.average(mod_img[y_bot:y_bot + size, m_loc[0]:m_loc[1]])
segments = [seg_a, seg_b, seg_c, seg_d, seg_e, seg_f, seg_g]
# print(f"x loc: {x_loc}, digit index: {location}, segment values: {segments}")
# create an integer from the bits
# print('value:', result)
return seg_to_digit(segments, location)
def main():
data_dir = Path(__file__).parent.joinpath('data')
meter_img = data_dir.joinpath('meter_test.jpg')
img = iio.imread(meter_img)
gray_img = img_as_ubyte(rgb2gray(img))
img_result = image_denoise(gray_img)
img_result1 = img_with_threshold(img_result)
reading = 0
for dig_loc in range(6):
reading += get_digit(dig_loc, img_result1)
print(f"{reading:>21}")
print("Final reading:", reading)
plot_imgs(gray_img, img_result1)
if __name__ == '__main__':
main()
This gave the following output:
7
77
677
4677
24677
924677
Final reading: 924677
I'm trying to detects the numbers found in my sqares, and I thought I could use the libary pytesseract, but for some reason I read the wrong values.
This is the console output:
And here I have all my pictures (they are seperated, this is just to show them all)
import numpy as np
import cv2
import re
from PIL import Image
import pytesseract
pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract'
img = cv2.imread('gulRecNum.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# convert to HSV, since red and yellow are the lowest hue colors and come before green
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# create a binary thresholded image on hue between red and yellow
lower = (0,240,160)
upper = (30,255,255)
thresh = cv2.inRange(hsv, lower, upper)
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
clean = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15,15))
clean = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# get external contours
contours = cv2.findContours(clean, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
result1 = img.copy()
result2 = img.copy()
mask = np.zeros(result2.shape, dtype=np.uint8)
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
ROI_number = 0
for c in contours:
cv2.drawContours(result1,[c],0,(0,0,0),2)
# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(c)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
# draw rotated rectangle on copy of img
cv2.drawContours(result2,[box],0,(0,0,0),2)
# Gør noget hvis arealet er større end 1.
# Whats the area of the component?
areal = cv2.contourArea(c)
if(areal > 1):
# get the center of mass
M = cv2.moments(c)
cx = int(M['m10']/M['m00'])
cy = int(M['m01']/M['m00'])
center = (cx, cy)
print("\nx: ",cx,"\ny: ",cy)
color = (0, 0, 255)
cv2.circle(result2, center, 3, color, -1)
cv2.putText(result2, "center", (int(cx) - 10, int(cy) - 20),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, color, 2)
# LOOK AT THIS PART
x,y,w,h = cv2.boundingRect(c)
ROI = 255 - thresh[y:y+h, x:x+w]
cv2.drawContours(mask, [c], -1, (255,255,255), -1)
cv2.imwrite('ROI_{}.png'.format(ROI_number), ROI)
Number = pytesseract.image_to_string(ROI, config='--psm 13 --oem 3 -c tessedit_char_whitelist=0123456789')
print("Number ", Number)
ROI_number += 1
# save result
cv2.imwrite("4cubes_result2.png",result2)
# display result
imS = cv2.resize(result2, (600, 400))
cv2.imshow("result2", imS)
cv2.waitKey(0)
cv2.destroyAllWindows()
Thought I could write Number = pytesseract.image_to_string(ROI, config='--psm 13 --oem 3 -c tessedit_char_whitelist=0123456789') print(Number)
and then get the number from the image, but I don't, how can that be?
EDIT NEW ERROR
how do i solve it with this picture?
from PIL import Image
from operator import itemgetter
import numpy as np
import easyocr
import cv2
import re
import imutils
import pytesseract
pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract'
reader = easyocr.Reader(['ch_sim','en']) # need to run only once to load model into memory
#Define empty array
Cubes = []
def getNumber(ROI):
img = cv2.imread(ROI)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(gray,127,255,0)
#cv2.imshow(thresh)
#cv2.imshow('Thresholded original',thresh)
#cv2.waitKey(0)
## Get contours
contours,h = cv2.findContours(thresh,cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
## only draw contour that have big areas
imx = img.shape[0]
imy = img.shape[1]
lp_area = (imx * imy) / 10
tmp_img = img.copy()
for cnt in contours:
approx = cv2.approxPolyDP(cnt,0.01 * cv2.arcLength(cnt, True), True)
if cv2.contourArea(cnt) > lp_area:
# Draw box corners and minimum area rectangle
rect = cv2.minAreaRect(cnt)
box = cv2.boxPoints(rect)
box = np.int0(box)
#cv2.drawContours(tmp_img, [box], 0, (0, 50, 255), 3)
#cv2.circle(tmp_img, tuple(box[0]), 8, (0, 255, 0), -1)
#cv2.circle(tmp_img, tuple(box[1]), 8, (0, 255, 0), -1)
#cv2.circle(tmp_img, tuple(box[2]), 8, (0, 255, 0), -1)
#cv2.circle(tmp_img, tuple(box[3]), 8, (0, 255, 0), -1)
#cv2.imshow(tmp_img)
#cv2.imshow('Minimum Area Rectangle', tmp_img)
#cv2.waitKey(0)
## Correct orientation and crop
# Link, https://jdhao.github.io/2019/02/23/crop_rotated_rectangle_opencv/
width = int(rect[1][0])
height = int(rect[1][1])
src_pts = box.astype("float32")
dst_pts = np.array([[0, height-1],
[0, 0],
[width-1, 0],
[width-1, height-1]], dtype="float32")
M = cv2.getPerspectiveTransform(src_pts, dst_pts)
warped = cv2.warpPerspective(img, M, (width, height))
# Run OCR on cropped image
# If the predicted value is digit print else rotate first
result = reader.readtext(warped)
print(result)
predicted_digit = result[0][1]
if np.char.isdigit(predicted_digit) == True:
cv2.imshow("warped " + ROI,warped)
else:
rot_img = warped.copy()
for i in range(0, 3):
rotated_image = cv2.rotate(rot_img, cv2.cv2.ROTATE_90_CLOCKWISE)
result = reader.readtext(rotated_image)
#if np.array(result).size == 0:
# continue
if not result:
rot_img = rotated_image
continue
#if len(result) == 0:
# continue
predicted_digit = result[0][1]
#print(result)
#print(predicted_digit)
#cv2.imshow(rotated_image)
if np.char.isdigit(predicted_digit) == True:
cv2.imshow("Image " + ROI, rotated_image)
break
rot_img = rotated_image
return predicted_digit
def sortNumbers(Cubes):
Cubes = sorted(Cubes, key=lambda x: int(x[2]))
#Cubes.sort(key=itemgetter(2)) # In-place sorting
#Cubes = sorted(Cubes, key=itemgetter(2)) # Create a new list
return Cubes
#img = cv2.imread('gulRecNum.jpg')
img = cv2.imread('webcam7.png')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# convert to HSV, since red and yellow are the lowest hue colors and come before green
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# create a binary thresholded image on hue between red and yellow
#Change these if cube colours changes?
lower =(20, 100, 100)
upper = (30, 255, 255)
#lower = (0,240,160)
#upper = (30,255,255)
thresh = cv2.inRange(hsv, lower, upper)
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (9,9))
clean = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15,15))
clean = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# get external contours
contours = cv2.findContours(clean, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
result2 = img.copy()
mask = np.zeros(result2.shape, dtype=np.uint8)
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
ROI_number = 0
for c in contours:
cv2.drawContours(result2,[c],0,(0,0,0),2)
# get rotated rectangle from contour
rot_rect = cv2.minAreaRect(c)
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
# draw rotated rectangle on copy of img
cv2.drawContours(result2,[box],0,(0,0,0),2)
# Gør noget hvis arealet er større end 1.
# Whats the area of the component?
areal = cv2.contourArea(c)
if(areal > 1):
# get the center of mass
M = cv2.moments(c)
cx = int(M['m10']/M['m00'])
cy = int(M['m01']/M['m00'])
center = (cx, cy)
print("\nx: ",cx,"\ny: ",cy)
color = (0, 0, 255)
cv2.circle(result2, center, 3, color, -1)
cv2.putText(result2, "center", (int(cx) - 10, int(cy) - 20),
cv2.FONT_HERSHEY_SIMPLEX, 1.2, color, 2)
x,y,w,h = cv2.boundingRect(c)
ROI = 255 - thresh[y:y+h, x:x+w]
cv2.drawContours(mask, [c], -1, (255,255,255), -1)
cv2.imwrite('ROI_{}.png'.format(ROI_number), ROI)
#Read saved image (number)
result = getNumber('ROI_{}.png'.format(ROI_number))
print("ROI_number: ", result)
Cubes.append([cx, cy, result])
ROI_number += 1
# save result
cv2.imwrite("4cubes_result2.png",result2)
# display result
imS = cv2.resize(result2, (600, 400))
cv2.imshow("result2", imS)
#cv2.imshow('mask', mask)
#cv2.imshow('thresh', thresh)
SortedCubes = sortNumbers(Cubes)
print("\nFound array [x, y, Cube_num] = ", Cubes)
print("Sorted array [x, y, Cube_num] = ", SortedCubes)
cv2.waitKey(0)
cv2.destroyAllWindows()
I get the following error (it can't detect a number)
Traceback (most recent call last): File "c:/Users/Mads/OneDrive/Universitet/7. semester/ROB1/python/objectDetectiong.py", line 169, in <module> result = getNumber('ROI_{}.png'.format(ROI_number)) File "c:/Users/Mads/OneDrive/Universitet/7. semester/ROB1/python/objectDetectiong.py", line 70, in getNumber predicted_digit = result[0][1] IndexError: list index out of range
This is implementation of my comment. Since, I do not have individual images this code will work with given grid like processed image.
For OCR I used EasyOCR instead of Tesserect. You could also try pytesserect on each output cropped images. Instead of rotating 4 times by 90 degrees by confidence, I went with digit detection on OCR result. If a detection is not a number then only rotate and retry.
Tested on google colab. Replace cv2_imshow(...) with cv2.imshow(...) for working locally. Also remove from google.colab.patches import cv2_imshow import.
This is modified version of my answer on card orientation correction here, OpenCV: using Canny and Shi-Tomasi to detect round corners of a playing card. All previous code is left as comment.
Code
!pip install easyocr
import easyocr
reader = easyocr.Reader(['ch_sim','en']) # need to run only once to load model into memory
"""
Based on my answer of rotated card detection,
https://stackoverflow.com/questions/64860785/opencv-using-canny-and-shi-tomasi-to-detect-round-corners-of-a-playing-card/64862448#64862448
"""
import cv2
import numpy as np
from google.colab.patches import cv2_imshow
img = cv2.imread('1.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(gray,127,255,0)
#cv2_imshow(thresh)
#cv2.imshow('Thresholded original',thresh)
#cv2.waitKey(0)
## Get contours
contours,h = cv2.findContours(thresh,cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
## only draw contour that have big areas
imx = img.shape[0]
imy = img.shape[1]
lp_area = (imx * imy) / 10
#################################################################
# Four point perspective transform
# https://www.pyimagesearch.com/2014/08/25/4-point-opencv-getperspective-transform-example/
#################################################################
def order_points(pts):
# initialzie a list of coordinates that will be ordered
# such that the first entry in the list is the top-left,
# the second entry is the top-right, the third is the
# bottom-right, and the fourth is the bottom-left
rect = np.zeros((4, 2), dtype = "float32")
# the top-left point will have the smallest sum, whereas
# the bottom-right point will have the largest sum
s = pts.sum(axis = 1)
rect[0] = pts[np.argmin(s)]
rect[2] = pts[np.argmax(s)]
# now, compute the difference between the points, the
# top-right point will have the smallest difference,
# whereas the bottom-left will have the largest difference
diff = np.diff(pts, axis = 1)
rect[1] = pts[np.argmin(diff)]
rect[3] = pts[np.argmax(diff)]
# return the ordered coordinates
return rect
def four_point_transform(image, pts):
# obtain a consistent order of the points and unpack them
# individually
rect = order_points(pts)
(tl, tr, br, bl) = rect
# compute the width of the new image, which will be the
# maximum distance between bottom-right and bottom-left
# x-coordiates or the top-right and top-left x-coordinates
widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))
widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))
maxWidth = max(int(widthA), int(widthB))
# compute the height of the new image, which will be the
# maximum distance between the top-right and bottom-right
# y-coordinates or the top-left and bottom-left y-coordinates
heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))
heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))
maxHeight = max(int(heightA), int(heightB))
# now that we have the dimensions of the new image, construct
# the set of destination points to obtain a "birds eye view",
# (i.e. top-down view) of the image, again specifying points
# in the top-left, top-right, bottom-right, and bottom-left
# order
dst = np.array([
[0, 0],
[maxWidth - 1, 0],
[maxWidth - 1, maxHeight - 1],
[0, maxHeight - 1]], dtype = "float32")
# compute the perspective transform matrix and then apply it
M = cv2.getPerspectiveTransform(rect, dst)
warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))
# return the warped image
return warped
#################################################################
#print(len(contours))
tmp_img = img.copy()
for cnt in contours:
approx = cv2.approxPolyDP(cnt,0.01 * cv2.arcLength(cnt, True), True)
## calculate number of vertices
#print(len(approx))
## Get the largest contours only
## Side count cannot be used since contours are not all rectangular
if cv2.contourArea(cnt) > lp_area:
#if len(approx) == 4 and cv2.contourArea(cnt) > lp_area:
# print("\n\n")
# print("#################################################")
# print("rectangle")
# print("#################################################")
# print("\n\n")
#tmp_img = img.copy()
#cv2.drawContours(tmp_img, [cnt], 0, (0, 255, 0), 6)
#cv2_imshow(tmp_img)
#cv2.imshow('Contour Borders', tmp_img)
#cv2.waitKey(0)
# tmp_img = img.copy()
# cv2.drawContours(tmp_img, [cnt], 0, (255, 0, 255), -1)
# cv2_imshow(tmp_img)
# #cv2.imshow('Contour Filled', tmp_img)
# #cv2.waitKey(0)
# # Make a hull arround the contour and draw it on the original image
# tmp_img = img.copy()
# mask = np.zeros((img.shape[:2]), np.uint8)
# hull = cv2.convexHull(cnt)
# cv2.drawContours(mask, [hull], 0, (255, 255, 255), -1)
# cv2_imshow(mask)
# #cv2.imshow('Convex Hull Mask', mask)
# #cv2.waitKey(0)
# # Draw minimum area rectangle
# #tmp_img = img.copy()
# rect = cv2.minAreaRect(cnt)
# box = cv2.boxPoints(rect)
# box = np.int0(box)
# cv2.drawContours(tmp_img, [box], 0, (255, 0, 0), 2)
# #cv2_imshow(tmp_img)
# #cv2.imshow('Minimum Area Rectangle', tmp_img)
# #cv2.waitKey(0)
# Draw box corners and minimum area rectangle
#tmp_img = img.copy()
rect = cv2.minAreaRect(cnt)
box = cv2.boxPoints(rect)
box = np.int0(box)
#print(rect)
#print(box)
cv2.drawContours(tmp_img, [box], 0, (0, 50, 255), 3)
cv2.circle(tmp_img, tuple(box[0]), 8, (0, 255, 0), -1)
cv2.circle(tmp_img, tuple(box[1]), 8, (0, 255, 0), -1)
cv2.circle(tmp_img, tuple(box[2]), 8, (0, 255, 0), -1)
cv2.circle(tmp_img, tuple(box[3]), 8, (0, 255, 0), -1)
#cv2_imshow(tmp_img)
#cv2.imshow('Minimum Area Rectangle', tmp_img)
#cv2.waitKey(0)
## Correct orientation and crop
# Link, https://jdhao.github.io/2019/02/23/crop_rotated_rectangle_opencv/
width = int(rect[1][0])
height = int(rect[1][1])
src_pts = box.astype("float32")
dst_pts = np.array([[0, height-1],
[0, 0],
[width-1, 0],
[width-1, height-1]], dtype="float32")
M = cv2.getPerspectiveTransform(src_pts, dst_pts)
warped = cv2.warpPerspective(img, M, (width, height))
#cv2_imshow(warped)
# Run OCR on cropped image
# If the predicted value is digit print else rotate first
result = reader.readtext(warped)
predicted_digit = result[0][1]
print("Detected Text:")
if np.char.isdigit(predicted_digit) == True:
print(result)
print(predicted_digit)
cv2_imshow(warped)
else:
rot_img = warped.copy()
for i in range(0, 3):
rotated_image = cv2.rotate(rot_img, cv2.cv2.ROTATE_90_CLOCKWISE)
result = reader.readtext(rotated_image)
#if np.array(result).size == 0:
# continue
if not result:
rot_img = rotated_image
continue
#if len(result) == 0:
# continue
predicted_digit = result[0][1]
#print(result)
#print(predicted_digit)
#cv2_imshow(rotated_image)
if np.char.isdigit(predicted_digit) == True:
print(result)
print(predicted_digit)
cv2_imshow(rotated_image)
break
rot_img = rotated_image
# # Draw bounding rectangle
# #tmp_img = img.copy()
# x, y, w, h = cv2.boundingRect(cnt)
# cv2.rectangle(tmp_img, (x, y), (x + w, y + h), (255, 0, 0), 2)
# #cv2_imshow(tmp_img)
# #cv2.imshow('Bounding Rectangle', tmp_img)
# #cv2.waitKey(0)
# # Bounding Rectangle and Minimum Area Rectangle
# #tmp_img = img.copy()
# rect = cv2.minAreaRect(cnt)
# box = cv2.boxPoints(rect)
# box = np.int0(box)
# cv2.drawContours(tmp_img, [box], 0, (0, 0, 255), 2)
# x, y, w, h = cv2.boundingRect(cnt)
# cv2.rectangle(tmp_img, (x, y), (x + w, y + h), (0, 255, 0), 2)
# #cv2_imshow(tmp_img)
# #cv2.imshow('Bounding Rectangle', tmp_img)
# #cv2.waitKey(0)
# # determine the most extreme points along the contour
# # https://www.pyimagesearch.com/2016/04/11/finding-extreme-points-in-contours-with-opencv/
# tmp_img = img.copy()
# extLeft = tuple(cnt[cnt[:, :, 0].argmin()][0])
# extRight = tuple(cnt[cnt[:, :, 0].argmax()][0])
# extTop = tuple(cnt[cnt[:, :, 1].argmin()][0])
# extBot = tuple(cnt[cnt[:, :, 1].argmax()][0])
# cv2.drawContours(tmp_img, [cnt], -1, (0, 255, 255), 2)
# cv2.circle(tmp_img, extLeft, 8, (0, 0, 255), -1)
# cv2.circle(tmp_img, extRight, 8, (0, 255, 0), -1)
# cv2.circle(tmp_img, extTop, 8, (255, 0, 0), -1)
# cv2.circle(tmp_img, extBot, 8, (255, 255, 0), -1)
# print("Corner Points: ", extLeft, extRight, extTop, extBot)
# cv2_imshow(tmp_img)
# #cv2.imshow('img contour drawn', tmp_img)
# #cv2.waitKey(0)
# #cv2.destroyAllWindows()
# ## Perspective Transform
# tmp_img = img.copy()
# pts = np.array([extLeft, extRight, extTop, extBot])
# warped = four_point_transform(tmp_img, pts)
# cv2_imshow(tmp_img)
# #cv2.imshow("Warped", warped)
# #cv2.waitKey(0)
cv2_imshow(tmp_img)
#cv2.destroyAllWindows()
Output Prediction
Detected Text:
[([[85, 67], [131, 67], [131, 127], [85, 127]], '1', 0.9992043972015381)]
1
Detected Text:
[([[85, 65], [133, 65], [133, 125], [85, 125]], '2', 0.9991914629936218)]
2
Detected Text:
[([[96, 72], [144, 72], [144, 128], [96, 128]], '4', 0.9996564984321594)]
4
Detected Text:
[([[88, 76], [132, 76], [132, 132], [88, 132]], '3', 0.9973381161689758)]
3
White Region Detection With Corners
Alternate methods,
Try pretrained digit classification model trained from MNIST and others on each large contours exceeding certain area.
Use multitask object detection with rotation. One output of network will be detections another angle regression to predict orientation.
Use text detector like, East and run OCR on each detected text.
I have a pdf from which I want to extract text. I use tesseract for OCR which does a good job. But my problem is that it does not recognize the 2 column format of the document and hence it merges the 2 columns together.
I want to split the document on the vertical (in the middle of the page) and horizontal (on top of the page) lines and then feed it to tesseract. So I do the following
Preprocessing steps:
# color to gray
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# edge detection
edges = cv2.Canny(gray, 500, 1000, apertureSize=7)
# dialate
kernel = np.ones((5,5),np.float32)/25
edges = cv2.dilate(edges, kernel, iterations=1)
# blur
blur = cv2.GaussianBlur(edges, (7, 7), 0)
These steps produce:
Now, I do line detection:
minLineLength = 1000
maxLineGap = 500
lines = cv2.HoughLinesP(processed_img, 1, np.pi, 2, minLineLength, maxLineGap)
for line in lines:
x1, y1, x2, y2 = line[0]
cv2.line(img, (x1, y1), (x2, y2), (0, 0, 0), 1)
The final result (after stitching all the images back into a pdf) looks like this.
I have tried various combinations for theta, minLineLength and maxLineGap and this was the best result I could get. Any help/pointers would be greatly appreciated!
One of the possible solutions is described below:
1) Detect the horizontal line. Below is one way to do this:
import cv2
import numpy as np
def discard(image):
image = np.uint8(image)
_, im_label, stts, _ = cv2.connectedComponentsWithStats(image, connectivity=4)
msk1 = np.isin(im_label, np.where(stts[:, cv2.CC_STAT_WIDTH] > 500)[0])
msk2 = np.isin(im_label, np.where(stts[:, cv2.CC_STAT_HEIGHT] > 500)[0])
image[(msk1 | msk2)] = 0
return image
img = cv2.imread("page_1.jpg", 0)
img = cv2.resize(img, None, fx=0.35, fy=0.35, interpolation=cv2.INTER_LINEAR)
height, width = img.shape[:2]
# Binarization
thresh = 255 - img
ret, thresh = cv2.threshold(thresh, 5, 255, cv2.THRESH_BINARY)
# Discarding long connected components
without_lines = discard(thresh.copy())
just_lines = cv2.bitwise_xor(thresh, without_lines)
horizontal = just_lines.copy()
# separating horizontal line
h_kernel_large = np.array([[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[1, 1, 1, 1, 1],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]], np.uint8)
horizontal = cv2.morphologyEx(horizontal, cv2.MORPH_OPEN, h_kernel_large, iterations=2)
cv2.imshow("horizontal_line", horizontal)
This is what we get in the horizontal matrix:
2) Use findContours and boundingRect to get the coordinates of that horizontal line. Then use that coordinate to crop the image horizontally.
upper_portion = img
lower_portion = img
contours, hierarchy = cv2.findContours(horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
upper_portion = img[0:y, 0:width]
lower_portion = img[y+h:height, 0:width]
cv2.imshow("upper_portion", upper_portion)
cv2.imshow("lower_portion", lower_portion)
cv2.waitKey(0)
Below are images after cropping.
upper_portion:
lower_portion:
3) Detect the vertical line and crop lower_portion image using the same procedure described in step 1.
In step one, I basically used "Connected Component Analysis" followed by an "Opening operation". Read them here and here
i want to compare an image with specific pixel using
(score, diff) = compare_ssim(grayA[y:y+h, x:x+w], grayB[y:y+h, x:x+w], full=True)
But that function only support rectangle ROI. And my ROI is a contour.
To compare that i need largest rectangle inside the contour. How to find largest rectangle inside contour area ?
Sample image
According to your OP, I suggest to use warpAffine to rotate the ROI to a rectangle shape, because the ROI is already in rectangle shape but rotated. Here is a simple sample:
import cv2
import numpy as np
img = cv2.imread("1.png")
(H,W,c) = img.shape
print("shape = {},{}".format(H,W))
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
_,thresh = cv2.threshold(gray,128,255,cv2.THRESH_BINARY_INV)
_,contours,_ = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
res = np.zeros_like(img)
c = np.squeeze(contours[0])
# find rectangle's conner points
x = sorted(c, key=lambda a:a[0])
left = x[0]
right = x[-1]
y= sorted(c, key=lambda a:a[1])
top = y[0]
bottom = y[-1]
cv2.circle(img, (left[0],left[1]), 4, (0, 0, 255), -1)
cv2.circle(img, (right[0],right[1]), 4, (0, 0, 255), -1)
cv2.circle(img, (top[0],top[1]), 4, (0, 0, 255), -1)
cv2.circle(img, (bottom[0],bottom[1]), 4, (0, 0, 255), -1)
#calculate rectangle's shape
roi_w = int(np.sqrt((top[0]-right[0])*(top[0]-right[0])+(top[1]-right[1])*(top[1]-right[1])))
roi_h = int(np.sqrt((top[0]-left[0])*(top[0]-left[0])+(top[1]-left[1])*(top[1]-left[1])))
pts1 = np.float32([top,right,left])
# keep the top coords and calculate new coords for left and right
new_top = top
new_right = [top[0] + roi_w, top[1]]
new_left = [top[0], top[1] + roi_h]
pts2 = np.float32([new_top,new_right,new_left])
#rotate
matrix = cv2.getAffineTransform(pts1, pts2)
result = cv2.warpAffine(img, matrix, (W,H))
cv2.drawContours(res, [contours[0]], 0, (0,255,0), 3)
# extract roi
roi = result[new_top[1]:new_left[1],new_top[0]:new_right[0]]
cv2.imshow("img",img)
cv2.imshow("result",result)
cv2.waitKey(0)