We Keep Coding

How to detect and crop character from an image having some bounding lines in python?

I'm having an image and i want to crop only the number inside with out the lines bounding around image. Here is the sample image:
As you can see there is 2 lines on the right and bottom of the image, i want to crop only number 8, if nothing inside, i will return None for the result, but with my code, it also return the area which include these line. So is there anyway to fix this?
Result i got:
Or with empty image, i got:
Here is my code:
import cv2
def process(image, readFile=True, returnType='binary'):
out_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# Performing OTSU threshold
ret, thresh1 = cv2.threshold(out_gray, 220, 255, cv2.THRESH_OTSU + cv2.THRESH_BINARY_INV)
# Specify structure shape and kernel size.
# Kernel size increases or decreases the area
# of the rectangle to be detected.
# A smaller value like (10, 10) will detect
# each word instead of a sentence.
rect_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (8,8))
# Appplying dilation on the threshold image
dilation = cv2.dilate(thresh1, rect_kernel, iterations = 1)
# Finding contours
contours, hierarchy = cv2.findContours(dilation, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
if len(contours) <= 0:
return False
contours, boundingBoxes = sort_contours(contours, 'left-to-right')
# Creating a copy of image
im2 = img.copy()
idata = 0
# Looping through the identified contours
# Then rectangular part is cropped and passed on
# to pytesseract for extracting text from it
# Extracted text is then written into the text file
i = 1
minHeight = 10
minWidth = 10
result = []
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
# Drawing a rectangle on copied image
rect = cv2.rectangle(im2, (x, (y-1)), (x + w, y + h + 1), (0, 255, 0), 1)
cv2.imwrite(f'{constant.__RESULT_PATH}/cutted123withbounding.png', im2)
# Cropping the text block for giving input to OCR
cropped = im2[y:y + h, x:x + w]
if w >= minWidth and h >= minHeight:
result.append(thresh1)
i = i + 1
return result

OpenCV Find a middle line of a contour [Python]

In my image processing project, I have already obtained a masked image (black-and-white image) and its contours using the cv.findContours function. My goal now is to create an algorithm that can draw a middle line for this contour. The masked image and its contour are shown in the following images.
Masked image:
Contour:
In my imagination, for that contour, I would like to create a middle line which is near horizontal. I have manually marked my ideal middle line in red. Please check the following image for the red middle line that I have mentioned.
Contour with the middle line:
It is noticeable that my ultimate goal is to find the tip point that I have marked in yellow. If you have other ideas that can directly find the yellow tip point, please also let me know. For finding the yellow tip point, I have tried two approaches cv.convexHull and cv.minAreaRect, but the issue is the robustness. I made these two approaches worked for some images but for some other images in my dataset, they are not working very well. Therefore, I think to find the middle line might be a good approach that I can try.

I believe you're trying to determine the contour's center of gravity and orientation. We can easily do this using Central Moments. More info on that here.
The code below generates this plot. Is this the result you wanted?
# Determine contour
img = cv2.imread(img_file, cv2.IMREAD_GRAYSCALE)
img_bin = (img>128).astype(np.uint8)
contours, _ = cv2.findContours(img_bin, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_NONE)
# Determine center of gravity and orientation using Moments
M = cv2.moments(contours[0])
center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
theta = 0.5*np.arctan2(2*M["mu11"],M["mu20"]-M["mu02"])
endx = 600 * np.cos(theta) + center[0] # linelength 600
endy = 600 * np.sin(theta) + center[1]
# Display results
plt.imshow(img_bin, cmap='gray')
plt.scatter(center[0], center[1], marker="X")
plt.plot([center[0], endx], [center[1], endy])
plt.show()

My goal right now is to create an algorithm that can draw a middle line for this contour.
If you detect the upper and lower bounds of your horizontal-lines, then you can calculate the middle-line coordinates.
For instance:
Middle-line will be:
If you change the size to the width of the image:
Code:
import cv2
img = cv2.imread("contour.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
(h, w) = img.shape[:2]
x1_upper = h
x1_lower = 0
x2_upper = h
x2_lower = 0
y1_upper = h
y1_lower = 0
y2_upper = h
y2_lower = 0
lines = cv2.ximgproc.createFastLineDetector().detect(gray)
for cur in lines:
x1 = cur[0][0]
y1 = cur[0][1]
x2 = cur[0][2]
y2 = cur[0][3]
# upper-bound coords
if y1 < y1_upper and y2 < y2_upper:
y1_upper = y1
y2_upper = y2
x1_upper = x1
x2_upper = x2
elif y1 > y1_lower and y2 > y2_lower:
y1_lower = y1
y2_lower = y2
x1_lower = x1
x2_lower = x2
print("\n\n-lower-bound-\n")
print("({}, {}) - ({}, {})".format(x1_lower, y1_lower, x2_lower, y2_lower))
print("\n\n-upper-bound-\n")
print("({}, {}) - ({}, {})".format(x1_upper, y1_upper, x2_upper, y2_upper))
cv2.line(img, (x1_lower, y1_lower), (x2_lower, y2_lower), (0, 255, 0), 5)
cv2.line(img, (x1_upper, y1_upper), (x2_upper, y2_upper), (0, 0, 255), 5)
x1_avg = int((x1_lower + x1_upper) / 2)
y1_avg = int((y1_lower + y1_upper) / 2)
x2_avg = int((x2_lower + x2_upper) / 2)
y2_avg = int((y2_lower + y2_upper) / 2)
cv2.line(img, (0, y1_avg), (w, y2_avg), (255, 0, 0), 5)
cv2.imshow("result", img)
cv2.waitKey(0)
cv2.destroyAllWindows()

I beleive skeleton is what you are looking for.
import cv2
import timeit
img = cv2.imread('Ggh8d - Copy.jpg',0)
s = timeit.default_timer()
thinned = cv2.ximgproc.thinning(img, thinningType = cv2.ximgproc.THINNING_ZHANGSUEN)
e = timeit.default_timer()
print(e-s)
cv2.imwrite("thinned1.png", thinned)
if smooth the edge a little bit
Actually the line will not torch the yellow point, since the algorithm have to check distance from edges, yellow point is located on the edge.

Here is another way to do that by computing the centerline of the rotated bounding box about your object in Python/OpenCV.
Input:
import cv2
import numpy as np
# load image
img = cv2.imread("blob_mask.jpg")
# convert to gray
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# threshold the grayscale image
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY)[1]
# get coordinates of all non-zero pixels
# NOTE: must transpose since numpy coords are y,x and opencv uses x,y
coords = np.column_stack(np.where(thresh.transpose() > 0))
# get rotated rectangle from
rotrect = cv2.minAreaRect(coords)
box = cv2.boxPoints(rotrect)
box = np.int0(box)
print (box)
# get center line from box
# note points are clockwise from bottom right
x1 = (box[0][0] + box[3][0]) // 2
y1 = (box[0][1] + box[3][1]) // 2
x2 = (box[1][0] + box[2][0]) // 2
y2 = (box[1][1] + box[2][1]) // 2
# draw rotated rectangle on copy of img as result
result = img.copy()
cv2.drawContours(result, [box], 0, (0,0,255), 2)
cv2.line(result, (x1,y1), (x2,y2), (255,0,0), 2)
# write result to disk
cv2.imwrite("blob_mask_rotrect.png", result)
# display results
cv2.imshow("THRESH", thresh)
cv2.imshow("RESULT", result)
cv2.waitKey(0)
cv2.destroyAllWindows()
Result:

Is there any way to crop an image inside a box?

I want to crop the image only inside the box or rectangle. I tried so many approaches but nothing worked.
import cv2
import numpy as np
img = cv2.imread("C:/Users/hp/Desktop/segmentation/add.jpeg", 0);
h, w = img.shape[:2]
# print(img.shape)
kernel = np.ones((3,3),np.uint8)
img2 = img.copy()
img2 = cv2.medianBlur(img2,5)
img2 = cv2.adaptiveThreshold(img2,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
cv2.THRESH_BINARY,11,2)
img2 = 255 - img2
img2 = cv2.dilate(img2, kernel)
img2 = cv2.medianBlur(img2, 9)
img2 = cv2.medianBlur(img2, 9)
cv2.imshow('anything', img2)
cv2.waitKey(0)
cv2.destroyAllWindows()
position = np.where(img2 !=0)
x0 = position[0].min()
x1 = position[0].max()
y0 = position[1].min()
y1 = position[1].max()
print(x0,x1,y0,y1)
result = img[x0:x1,y0:y1]
cv2.imshow('anything', result)
cv2.waitKey(0)
cv2.destroyAllWindows()
Output should be the image inside the sqaure.

You can use contour detection for this. If your image has basically only a hand drawn rectangle in it, I think it's good enough to assume it's the largest closed contour in the image. From that contour, we can figure out a polygon/quadrilateral approximation and then finally get an approximate rectangle. I'll define some utilities at the beginning which I generally use to make my time easier when messing around with images:
def load_image(filename):
return cv2.imread(filename)
def bnw(image):
return cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
def col(image):
return cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
def fixrgb(image):
return cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
def show_image(image, figsize=(7,7), cmap=None):
cmap = cmap if len(image.shape)==3 else 'gray'
plt.figure(figsize=figsize)
plt.imshow(image, cmap=cmap)
plt.show()
def AdaptiveThresh(gray):
blur = cv2.medianBlur(gray, 5)
adapt_type = cv2.ADAPTIVE_THRESH_GAUSSIAN_C
thresh_type = cv2.THRESH_BINARY_INV
return cv2.adaptiveThreshold(blur, 255, adapt_type, thresh_type, 11, 2)
def get_rect(pts):
xmin = pts[:,0,1].min()
ymin = pts[:,0,0].min()
xmax = pts[:,0,1].max()
ymax = pts[:,0,0].max()
return (ymin,xmin), (ymax,xmax)
Let's load the image and convert it to grayscale:
image_name = 'test.jpg'
image_original = fixrgb(load_image(image_name))
image_gray = 255-bnw(image_original)
show_image(image_gray)
Use some morph ops to enhance the image:
kernel = np.ones((3,3),np.uint8)
d = 255-cv2.dilate(image_gray,kernel,iterations = 1)
show_image(d)
Find the edges and enhance/denoise:
e = AdaptiveThresh(d)
show_image(e)
m = cv2.dilate(e,kernel,iterations = 1)
m = cv2.medianBlur(m,11)
m = cv2.dilate(m,kernel,iterations = 1)
show_image(m)
Contour detection:
contours, hierarchy = cv2.findContours(m, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
total_area = np.prod(image_gray.shape)
max_area = 0
for cnt in contours:
# Simplify contour
perimeter = cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, 0.03 * perimeter, True)
area = cv2.contourArea(approx)
# Shape is recrangular, so 4 points approximately and it's convex
if (len(approx) == 4 and cv2.isContourConvex(approx) and max_area<area<total_area):
max_area = cv2.contourArea(approx)
quad_polygon = approx
img1 = image_original.copy()
img2 = image_original.copy()
cv2.polylines(img1,[quad_polygon],True,(0,255,0),10)
show_image(img1)
tl, br = get_rect(quad_polygon)
cv2.rectangle(img2, tl, br, (0,255,0), 10)
show_image(img2)
So you can see the approximate polygon and the corresponding rectangle, using which you can get your crop. I suggest you play around with median blur and morphological ops like erosion, dilation, opening, closing etc and see which set of operations suits your images the best; I can't really say what's good from just one image. You can crop using the top left and bottom right coordinates:
show_image(image_original[tl[1]:br[1],tl[0]:br[0],:])

Draw the square with a different color (e.g red) so it can be distinguishable from other writing and background. Then threshold it so you get a black and white image: the red line will be white in this image. Get the coordinates of white pixels: from this set, select only the two pairs (minX, minY)(maxX,maxY). They are the top-left and bottom-right points of the box (remember that in an image the 0,0 point is on the top left of the image) and you can use them to crop the image.

Image Differencing: Ignore translational movement

I'm looking for some insight into what the best approach might be to my problem.
I'm comparing two separate images for differences, but I'm running into a problem with small translational movements.
I have a "gospel" image which is the "gold standard" per se:
gospel image
Then I have multiple different taken images to compare against.
Here's an example: example image
Here's an example difference image showing my problem: difference image
As you can see, they are quite small. The way that I am differencing the images now is by first resizing the images to 32x32, manually decreasing the contrast by 100 and then applying a blur using OpenCV.
After, I am using skimage's 'structural_integrity' function to subtract and quantify the differences between the images. The rest is purely for viewing.
import cv2
import numpy as np
from PIL import Image
from skimage.metrics import structural_similarity
def change_contrast(img, level):
img = Image.fromarray(img)
factor = (259 * (level + 255)) / (255 * (259 - level))
def contrast(c):
return 128 + factor * (c - 128)
return np.asarray(img.point(contrast))
# Open and preprocess the images
image_orig = cv2.imread(IMAGE_PATH)
image = cv2.resize(image, (32, 32))
image = change_contrast(image_orig, -100)
image = cv2.blur(image, (5, 5))
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gospel_orig = cv2.imread(GOSPEL_PATH)
gospel = cv2.resize(gospel_orig, (32, 32))
gospel = change_contrast(gospel, -100)
gospel = cv2.blur(gospel, (5, 5))
gospel = cv2.cvtColor(gospel, cv2.COLOR_BGR2GRAY)
# Get image similarities and an output difference image
(score, diff) = structural_similarity(image, gospel, full=True)
print("Image similarity", score)
diff = (diff * 255).astype("uint8")
# Viewing stuff below
thresh = cv2.threshold(diff, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]
contours = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
filled_gospel = cv2.cvtColor(gospel, cv2.COLOR_GRAY2BGR)
for c in contours:
area = cv2.contourArea(c)
if area > 40:
x,y,w,h = cv2.boundingRect(c)
cv2.drawContours(filled_gospel, [c], 0, (0,255,0), -1)
cv2.imshow('image', image)
cv2.imshow('gospel', gospel)
cv2.imshow('diff',diff)
cv2.imshow('filled gospel',filled_gospel)
cv2.waitKey(0)
When I do the above steps, you can see some translational differences between the 'gospel' and the taken image. What would be the best way to combat this as I only want to get the differences in the black of the letter, not how well it is aligned?

Here is how I would do template matching and differencing in Python/OpenCV.
Read the reference and example images
Pad the example image to twice its dimensions with its background gray color.
Do template matching with the reference to find the best match location and match score.
Crop the padded example image with its top left corner at the match location, but the size of the reference image
Get the absolute difference image
Save results
Reference:
Example:
import cv2
import numpy as np
# read reference and convert to gray
ref = cv2.imread('reference.png')
ref_gray = cv2.cvtColor(ref, cv2.COLOR_BGR2GRAY)
hr, wr = ref_gray.shape
# read example and convert to gray
ex = cv2.imread('example.png')
ex_gray = cv2.cvtColor(ex, cv2.COLOR_BGR2GRAY)
he, we = ex_gray.shape
# pad the example to double its dimensions with gray=190
color=190
wp = we // 2
hp = he // 2
ex_gray = cv2.copyMakeBorder(ex_gray, hp,hp,wp,wp, cv2.BORDER_CONSTANT, value=color)
# do template matching
corrimg = cv2.matchTemplate(ref_gray,ex_gray,cv2.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(corrimg)
max_val_corr = '{:.3f}'.format(max_val)
print("correlation: " + max_val_corr)
xx = max_loc[0]
yy = max_loc[1]
print('x_match_loc =',xx,'y_match_loc =',yy)
# crop the padded example image at top left corner of xx,yy and size hr x wr
ex_gray_crop = ex_gray[yy:yy+hr, xx:xx+wr]
# get absolute difference image
ref_grayf = ref_gray.astype(np.float32)
ex_gray_cropf = ex_gray_crop.astype(np.float32)
diff = 255 - np.abs(cv2.add(ref_gray, -ex_gray_crop))
# compute mean of diff
mean = cv2.mean(diff)[0]
print("mean of diff in range 0 to 100 =",mean)
cv2.imshow('ref_gray', ref_gray)
cv2.imshow('ex_gray', ex_gray)
cv2.imshow('ex_gray_crop', ex_gray_crop)
cv2.imshow('correlation image', corrimg)
cv2.imshow('diff', diff)
cv2.waitKey(0)
cv2.destroyAllWindows()
# save results
cv2.imwrite('reference_gray.jpg', ref_gray)
cv2.imwrite('example_gray_padded.jpg', ex_gray)
cv2.imwrite('reference_example_correlation.jpg', (255*corrimg).clip(0,255).astype(np.uint8))
cv2.imwrite('example_gray_padded_cropped.jpg', ex_gray_crop)
cv2.imwrite('reference_example_diff.jpg', diff)
Example padded:
Correlation Image showing locations of best match:
Match Results:
correlation: 0.969
x_match_loc = 10 y_match_loc = 9
mean of diff in range 0 to 100 = 1.3956887102667155
Example cropped to align with reference:
Diff image (white is where they differ):

How to find the coordinates of the outside corners of these 4 squares? (morphological closing/opening does not conserve squares if image is rotated)

One of the first processing steps in a tool I'm coding is to find the coordinates of the outside corners of 4 big black squares. They will then be used to do a homographic transform, in order to deskew / unrotate the image (a.k.a perspective transform), to finally get a rectangular image. Here is an example of - rotated and noisy - input (download link here):
To keep the big squares only, I'm using morphological transformations like closing/opening:
import cv2, numpy as np
img = cv2.imread('rotatednoisy-cropped.png', cv2.IMREAD_GRAYSCALE)
kernel = np.ones((30, 30), np.uint8)
img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)
cv2.imwrite('output.png', img)
Input file (download link):
Output, after morphological transform:
Problem: the output squares are not square anymore, and therefore the coordinates of the top left corner of the square will be not precise at all!
I could reduce the kernel size, but then it would keep more unwanted small elements.
Question: how to get a better detection of the corners of the squares?
Note:
As a morphological closing is just a dilatation + an erosion, I found the culprit:
import cv2, numpy as np
img = cv2.imread('rotatednoisy-cropped.png', cv2.IMREAD_GRAYSCALE)
kernel = np.ones((30, 30), np.uint8)
img = cv2.dilate(img, kernel, iterations = 1)
After this step, it's still ok:
Then
img = cv2.erode(img, kernel, iterations = 1)
gives
and it's not ok anymore!

See this link for detailed explanation on how to de-skew an image.
import cv2
import numpy as np
def corners(box):
cx,cy,w,h,angle = box[0][0],box[0][1],box[1][0],box[1][1],box[2]
CV_PI = 22./7.
_angle = angle*CV_PI/180.;
b = np.cos(_angle)*0.5;
a = np.sin(_angle)*0.5;
pt = []
pt.append((int(cx - a*h - b*w),int(cy + b*h - a*w)));
pt.append((int(cx + a*h - b*w),int(cy - b*h - a*w)));
pt.append((int(2*cx - pt[0][0]),int(2*cy - pt[0][1])));
pt.append((int(2*cx - pt[1][0]),int(2*cy - pt[1][1])));
return pt
if __name__ == '__main__':
image = cv2.imread('image.jpg',cv2.IMREAD_UNCHANGED)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
n = 3
sigma = 0.3 * (n/2 - 1) + 0.8
gray = cv2.GaussianBlur(gray, ksize=(n,n), sigmaX=sigma)
ret,binary = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU+cv2.THRESH_BINARY)
_,contours,_ = cv2.findContours(binary, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
contours.sort(key=lambda x: len(x), reverse=True)
points = []
for i in range(0,4):
shape = cv2.approxPolyDP(contours[i], 0.05*cv2.arcLength(contours[i],True), True)
if len(shape) == 4:
points.append(shape)
points = np.array(points,dtype=np.int32)
points = np.reshape(points, (-1,2))
box = cv2.minAreaRect(points)
pt = corners(box)
for i in range(0,4):
image = cv2.line(image, (pt[i][0],pt[i][1]), (pt[(i+1)%4][0],pt[(i+1)%4][1]), (0,0,255))
(h,w) = image.shape[:2]
(center) = (w//2,h//2)
angle = box[2]
if angle < -45:
angle = (angle+90)
else:
angle = -angle
M = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(image, M, (w,h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_CONSTANT)
cv2.imshow('image', image)
cv2.imshow('rotated', rotated)
cv2.waitKey(0)
cv2.destroyAllWindows()

You could try by searching and filtering out your specific contours (black rectangles) and sorting them with a key. Then select the extreme point for each contour (left, right, top, bottom) and you will get the points. Note that this approach is ok for this picture only and if the picture was roteted in other direction, you would have to change the code accordingly. I am not an expert but I hope this helps a bit.
import numpy as np
import cv2
img = cv2.imread("rotate.jpg")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, threshold = cv2.threshold(gray,150,255,cv2.THRESH_BINARY)
im, contours, hierarchy = cv2.findContours(threshold,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
contours.sort(key=lambda c: np.min(c[:,:,1]))
j = 1
if len(contours) > 0:
for i in range(0, len(contours)):
size = cv2.contourArea(contours[i])
if 90 < size < 140:
if j == 1:
c1 = contours[i]
j += 1
elif j == 2:
c2 = contours[i]
j += 1
elif j == 3:
c3 = contours[i]
j += 1
elif j == 4:
c4 = contours[i]
break
Top = tuple(c1[c1[:, :, 1].argmin()][0])
Right = tuple(c2[c2[:, :, 0].argmax()][0])
Left = tuple(c3[c3[:, :, 0].argmin()][0])
Bottom = tuple(c4[c4[:, :, 1].argmax()][0])
cv2.circle(img, Top, 2, (0, 255, 0), -1)
cv2.circle(img, Right, 2, (0, 255, 0), -1)
cv2.circle(img, Left, 2, (0, 255, 0), -1)
cv2.circle(img, Bottom, 2, (0, 255, 0), -1)
cv2.imshow("Image", img)
cv2.waitKey(0)
Result:

You can extract the squares as single blobs after binarization with a suitable threshold, and select the appropriate ones based on size. You can also first denoise with a median filter if you want.
Then a tight rotated bounding rectangle will give you the corners (you can obtain it by running Rotating Calipers on the Convex Hull).