We Keep Coding

Detecting and ignoring rectangles that fall under another rectangle and efficient cropping

I am working on a project where I take a floor plan image as input that contains 2-3 floor plans and I need to detect each floorplan and crop them and save in as a different file.
Following are the sample input and output images.
Input:
Output:
So as you can see the second output is wrongly cropped. Also I get smaller rectangles (which are part of the output images) as byproducts.
Following is the code that I am using:
import cv2
import download_imgs_R1
import floorplan_threshold_R2
import pandas as pd
import os
filename = 'project_image.csv'
df = pd.read_csv(filename)
pids = df['id']
urls = df['complete_url']
for pid,url in zip(pids,urls):
name = url.split('/')[-1]
ext = name.split('.')[-1]
filepath = './xxxx/{}/original_images/'.format(pid)
savepath = './xxxx/{}/processed_images/'.format(pid)
savename = name.split('.')[0]
save = savepath+savename+'{}.png'
if ext == 'pdf':
image_name = download_imgs_R1.extract_from_pdf(filename=name, dest=filepath)
else:
image_name = filepath+name
print(image_name)
no_padding_image, crop_img_name = floorplan_threshold_R2.remove_white_space(image_name)
feature_dict = floorplan_threshold_R2.get_img_features(no_padding_image)
cont, hier = floorplan_threshold_R2.contour_method(no_padding_image)
area_dict = floorplan_threshold_R2.max_rect(cont)
roi_area = []
print(feature_dict)
img_area = feature_dict['area']
for area in area_dict:
if area >= img_area*0.1 and area < img_area:
roi_area.append(area)
plan_no = 1
for a in roi_area:
plan = area_dict[a]
# del area_dict[a]
x,y,w,h = plan
aspect_ratio = h/w
if x <=50 or y <= 25:
roi = no_padding_image[y:y+h, x:x+w]
else:
roi = no_padding_image[y-50:y+h+10, x-20:x+w+10]
print('PID: {}, No. {}'.format(pid,plan_no))
# cv2.rectangle(no_padding_image, (x-10,y-10), (x+w+10, y+h+10), (255,255,255), 2)
# roi = cv2.copyMakeBorder(roi, 10, 10, 10, 10, cv2.BORDER_CONSTANT, None, value = [255,255,255])
# cv2.imshow('ROI-{}'.format(image_name),roi)
cv2.imwrite(save.format(plan_no),roi)
cv2.waitKey(0)
plan_no += 1
floor_plan_threshold_R2.py:
import cv2
from cv2 import dilate
from cv2 import findContours
import imutils
import numpy as np
import download_imgs_R1
def remove_white_space(filename:str):
savename = filename
img = cv2.imread(filename=filename)
orignal = img.copy()
gray = cv2.cvtColor(orignal, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (25,25), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
# Perform morph operations, first open to remove noise, then close to combine
noise_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3,3))
opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, noise_kernel, iterations=2)
close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7,7))
close = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, close_kernel, iterations=3)
# Find enclosing boundingbox and crop ROI
coords = cv2.findNonZero(close)
x,y,w,h = cv2.boundingRect(coords)
# cv2.rectangle(orignal, (x, y), (x + w, y + h), (36,255,12), 2)
if x <= 50 or y <= 10:
crop = orignal[y:y+h, x:x+w]
else:
crop = orignal[y-10:y+h+10, x-60:x+w+10]
cv2.imwrite(savename,crop)
# cv2.imshow('Removed White space (Preprocess - 1)',crop)
cv2.waitKey(0)
return crop, savename
def get_img_features(image, filename:str=None,resize:bool=False):
if not filename:
res = image.copy()
else:
res = cv2.imread(filename)
img_height, img_width, img_channel = image.shape
if resize is True and image.shape[0] > 800:
res = imutils.resize(res, height=720)
img_height, img_width, img_channel = res.shape
img_area = img_width * img_height
img_aspect_ratio = img_width/img_height
img_features = {'height':img_height, 'width':img_width, 'area':img_area, 'aspect_ratio':img_aspect_ratio}
return img_features
def mask_method(image, filename:str=None):
if not filename:
res = image.copy()
else:
res = cv2.imread(filename)
hsv_plan = cv2.cvtColor(res, cv2.COLOR_BGR2HSV)
#define range for blue color (HSV Range)
blue_min = np.array([14,100,76])
blue_max = np.array([130,255,255])
bluemask = cv2.inRange(hsv_plan,blue_min,blue_max)
blue_output = cv2.bitwise_and(hsv_plan, hsv_plan, mask=bluemask)
grey_mask = cv2.cvtColor(blue_output, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(grey_mask, 100, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
kernel = np.ones((3,3), np.uint8)
dil = dilate(thresh, kernel, iterations=2)
cont,hier = findContours(dil, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
return cont, hier
def contour_method(image, filename:str=None):
if not filename:
res = image.copy()
else:
res = cv2.imread(filename)
grey_plan = cv2.cvtColor(res, cv2.COLOR_BGR2GRAY)
ret2, thresh2 = cv2.threshold(grey_plan, 160, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
kernel = np.ones((3,3), np.uint8)
dil_grey = dilate(thresh2, kernel, iterations=2)
cont,hier = findContours(dil_grey, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
return cont,hier
def max_rect(cntrs):
ar = {}
for cnt in cntrs:
x,y,w,h = cv2.boundingRect(cnt)
area = w*h
ar[area] = (x,y,w,h)
return ar
I need to find a generic solution for cropping the image as just providing a number for cropping will affect other images as well.

Pytesseract OCR doesn't recognize the digits

I am trying to read these images:
I have tried several options but I can't seem to read them correctly as 15/0, 30/0, 40/0.
frame = frame[900:1000, 450:500]
scale_percent = 200 # percent of original size
width = int(frame.shape[1] * scale_percent / 100)
height = int(frame.shape[0] * scale_percent / 100)
dim = (width, height)
frame = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
cv2.imshow("cropped", frame)
cv2.waitKey(0)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
cv2.imshow("cropped", frame)
cv2.waitKey(0)
pytesseract.pytesseract.tesseract_cmd = (
r"C:\Program Files\Tesseract-OCR\tesseract.exe"
)
results = pytesseract.image_to_data(
frame,
output_type=Output.DICT,
config="--psm 10 --oem 3 -c tessedit_char_whitelist=0123456789",
)
# results = replace_chars(results)
print(("-").join(results["text"]), "\n")

One way of solving is using inRange thresholding
The result will be:
If you set page-segmentation-mode 6
15
0
30
0
40
0
Code:
import cv2
import pytesseract
from numpy import array
image_list = ["LZxCs.png", "W06I0.png", "vvzE5.png"]
for image in image_list:
bgr_image = cv2.imread(image)
hsv_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv_image, array([0, 0, 0]), array([165, 10, 255]))
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 3))
dilate = cv2.dilate(mask, kernel, iterations=1)
thresh = cv2.bitwise_and(dilate, mask)
text = pytesseract.image_to_string(thresh, config='--psm 6')
print(text)
The second way is applying global-threshold
If you set page-segmentation-mode 6
15
0
30
0
40
0
Code:
import cv2
import pytesseract
image_list = ["LZxCs.png", "W06I0.png", "vvzE5.png"]
for image in image_list:
bgr_image = cv2.imread(image)
gray_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray_image, 127, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
text = pytesseract.image_to_string(thresh, config='--psm 6')
print(text)
cv2.imwrite(f"/Users/ahx/Desktop/{image}", thresh)
cv2.imshow('', thresh)
cv2.waitKey(0)
For more, you can check the documentation

How to crop multiple ROI in image using Python and OpenCV

I have an image that converted from PDF to PNG. The converted image contains several keywords that I wanted to extracted using OCR Tesseract.
Right now, I need to determine the ROI manually to crop the selected ROI. Since I have more than 5 ROI's to be applied, what would be the most efficient way to apply the ROI instead of doing it by try and error to find the exact location?
Below is the code:
def cropped(self, event):
#1st ROI
y = 20
x = 405
h = 230
w = 425
#2nd ROI
y1 = 30
x1 = 305
h1 = 330
w1 = 525
#open the converted image
image = cv2.imread("Output.png")
#perform image cropping
crop_image = image[x:w, y:h]
crop_image1 = image[x1:w1, y1:h1]
#save the cropped image
cv2.imwrite("Cropped.png", crop_image)
cv2.imwrite("Cropped1.png", crop_image1)
#open the cropped image and pass to the OCR engine
im = cv2.imread("Cropped.png")
im1 = cv2.imread("Cropped1.png")
## Do the text extraction here

you can use mouse event to select multiple ROI and crop based on the location
#!/usr/bin/env python3
import argparse
import cv2
import numpy as np
from PIL import Image
import os
drawing = False # true if mouse is pressed
ix,iy = -1,-1
refPt = []
img = ""
clone = ""
ROIRegion = []
# mouse callback function
def draw_rectangle(event,x,y,flags,param):
global ix,iy,drawing,img,clone,refPt, ROIRegion
if event == cv2.EVENT_LBUTTONDOWN:
drawing = True
ix,iy = x,y
refPt = [(x, y)]
ROIRegion.append(refPt)
#clone = img.copy()
elif event == cv2.EVENT_MOUSEMOVE:
if drawing == True:
img = clone.copy()
cv2.rectangle(img,(ix,iy),(x,y),(0,255,0),3)
a=x
b=y
if a != x | b != y:
cv2.rectangle(img,(ix,iy),(x,y),(0,0,0),-1)
elif event == cv2.EVENT_LBUTTONUP:
drawing = False
refPt.append((x,y))
img = clone.copy()
cv2.rectangle(img, (ix,iy),(x,y), (0, 255, 0), 2)
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="Path to the image")
args = vars(ap.parse_args())
# load the image, clone it, and setup the mouse callback function
img = cv2.imread(args["image"])
img = np.array(img)
clone = img.copy()
cv2.namedWindow('image')
cv2.setMouseCallback('image',draw_rectangle)
while(1):
cv2.imshow('image',img)
k = cv2.waitKey(1) & 0xFF
if k == ord("r"):
del ROIRegion[-1]
del refPt[-1]
img = clone.copy()
elif k == 27:
break
#Do your cropping here
for region in range(len(ROIRegion)):
cv2.rectangle(img, ROIRegion[region][0],ROIRegion[region][1], (0, 255, 0), 2)
roi = clone[ROIRegion[region][0][1]:ROIRegion[region][1][1], ROIRegion[region][0][0]:ROIRegion[region][1][0]]
roi = cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)

Here is one way in Python/OpenCV.
Read the input
Threshold on box outline color
Apply morphology to ensure closed
Get the external contours
Loop over each contour, get its bounding box, crop the region in the input and write the output
Input:
import cv2
import numpy as np
# read image
img = cv2.imread('text_boxes.jpg')
# threshold on box outline color
lowerBound = (80,120,100)
upperBound = (160,200,180)
thresh = cv2.inRange(img, lowerBound, upperBound)
# apply morphology to ensure regions are filled and remove extraneous noise
kernel = np.ones((3,3), np.uint8)
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# get contours
contours = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
# get bounding boxes
i = 1
for cntr in contours:
# get bounding boxes
x,y,w,h = cv2.boundingRect(cntr)
crop = img[y:y+h, x:x+w]
cv2.imwrite("text_boxes_crop_{0}.png".format(i), crop)
i = i + 1
# save threshold
cv2.imwrite("text_boxes_thresh.png",thresh)
# show thresh and result
cv2.imshow("thresh", thresh)
cv2.waitKey(0)
cv2.destroyAllWindows()
Threshold image:
Cropped Images:

Use grabcut algorithm to separate the saliency areas

enter image description hereI first detected the saliency of the image and then used the grabcut algorithm to segment the saliency targets. However, the result was a salient image but did not segment the saliency map. The error was as follows: error :-5 image mush have cv_8uc3 type in function grabcut, this is my source code, what should I do?`
import tensorflow as tf
import numpy as np
import os
from scipy import misc
import argparse
import sys,cv2
from skimage.io import imread, imsave
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
g_mean = np.array(([126.88,120.24,112.19])).reshape([1,1,3])
output_folder = "./test_output"
def rgba2rgb(img):
if img.ndim == 2:
img = gray2rgb(img)
elif img.shape[2] == 4:
img = img[:, :, :3]
upper_dim = max(img.shape[:2])
if upper_dim > args.max_dim:
img = rescale(img, args.max_dim/float(upper_dim), order=3)
return img
def largest_contours_rect(saliency):
contours, hierarchy = cv2.findContours(saliency * 3,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
contours = sorted(contours, key = cv2.contourArea)
return cv2.boundingRect(contours[-1])
def refine_saliency_with_grabcut(img, saliency):
rect = largest_contours_rect(saliency)
bgdmodel = np.zeros((1, 65),np.float64)
fgdmodel = np.zeros((1, 65),np.float64)
saliency[np.where(saliency > 0)] = cv2.GC_FGD
mask = saliency
cv2.grabCut(img, mask, rect, bgdmodel, fgdmodel, 1, cv2.GC_INIT_WITH_RECT)
mask = np.where((mask==2)|(mask==0),0,1).astype('uint8')
return mask
def backprojection_saliency(img,args):
saliency =main(args)
#cv2.imshow("original", saliency)
#saliency=mpimg.imread('alpha1.png')
img = cv2.resize(img, (320, 232))
mask = refine_saliency_with_grabcut(img, saliency)
#misc.imsave(os.path.join(output_folder,'flowers2.png'),result)
return mask
def main(args):
if not os.path.exists(output_folder):
os.mkdir(output_folder)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction = args.gpu_fraction)
with tf.Session(config=tf.ConfigProto(gpu_options = gpu_options)) as sess:
saver = tf.train.import_meta_graph('./meta_graph/my-model.meta')
saver.restore(sess,tf.train.latest_checkpoint('./salience_model'))
image_batch = tf.get_collection('image_batch')[0]
pred_mattes = tf.get_collection('mask')[0]
if args.rgb_folder:
rgb_pths = os.listdir(args.rgb_folder)
for rgb_pth in rgb_pths:
rgb = misc.imread(os.path.join(args.rgb_folder,rgb_pth))
if rgb.shape[2]==4:
rgb = rgba2rgb(rgb)
origin_shape = rgb.shape
rgb = np.expand_dims(misc.imresize(rgb.astype(np.uint8),[320,320,3],interp="nearest").astype(np.float32)-g_mean,0)
feed_dict = {image_batch:rgb}
pred_alpha = sess.run(pred_mattes,feed_dict = feed_dict)
final_alpha = misc.imresize(np.squeeze(pred_alpha),origin_shape)
misc.imsave(os.path.join(output_folder,rgb_pth),final_alpha)
else:
rgb = misc.imread(args.rgb)
if rgb.shape[2]==4:
rgb = rgba2rgb(rgb)
origin_shape = rgb.shape[:2]
rgb = np.expand_dims(misc.imresize(rgb.astype(np.uint8),[320,320,3],interp="nearest").astype(np.float32)-g_mean,0)
feed_dict = {image_batch:rgb}
pred_alpha = sess.run(pred_mattes,feed_dict = feed_dict)
final_alpha = misc.imresize(np.squeeze(pred_alpha),origin_shape)
misc.imsave(os.path.join(output_folder,'alpha.png'),final_alpha)
#rgbs = mpimg.imread('flower1.jpg')
result=refine_saliency_with_grabcut(rgb, final_alpha)
misc.imsave(os.path.join(output_folder,'segmentation.png'),result)
#cv2.imshow("original", final_alpha)
#plt.imshow(final_alpha)
return final_alpha;
def parse_arguments(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--rgb', type=str,
help='input rgb',default = None)
parser.add_argument('--rgb_folder', type=str,
help='input rgb',default = None)
parser.add_argument('--gpu_fraction', type=float,
help='how much gpu is needed, usually 4G is enough',default = 1.0)
return parser.parse_args(argv)
if __name__ == '__main__':
main(parse_arguments(sys.argv[1:]))``

I use the threshed binary image to find the max contour, then create a mask. Do grabcut using this mask.
The source:
And the result is like this:
#!/usr/bin/python3
# 2017.11.27 15:26:53 CST
# 2017.11.27 16:37:38 CST
import numpy as np
import cv2
## read the image(读取图像)
img = cv2.imread("tt04_flower.png")
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#displaySplit(img)
## threshed(阈值化)
th, threshed = cv2.threshold(gray, 100, 255, cv2.THRESH_BINARY|cv2.THRESH_OTSU)
## findContours(查找轮廓)
cnts = cv2.findContours(threshed, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)[-2]
## sorted by area(按照面积排序)
cnts = sorted(cnts, key=cv2.contourArea)
## get the maximum's boundinRect(获取最大边缘的外接矩形)
cnt = cnts[-1]
bbox = x,y,w,h = cv2.boundingRect(cnt)
## create mask(创建掩模)
mask = np.ones_like(gray, np.uint8)*cv2.GC_PR_BGD
cv2.drawContours(mask, [cnt], -1, cv2.GC_FGD, -1)
## 使用 grabcut 分割
bgdModel = np.zeros((1, 65), np.float64)
fgdModel = np.zeros((1, 65), np.float64)
rect = bbox
cv2.grabCut(img, mask, rect, bgdModel, fgdModel, 5, cv2.GC_INIT_WITH_MASK)
mask2 = np.where((mask==2)|(mask==0),0,1).astype("uint8")
grabcut = img*mask2[:,:,np.newaxis]
## save and display
cv2.imwrite("flower_res.png", grabcut)
cv2.imshow("(1) source", img)
cv2.imshow("(2) grabcut", grabcut)
cv2.waitKey()

Why doesn't my OCR opencv in python code work at al?

i am new to this forum so sorry if this is question is too long for this place. I just started coding and i never really had a lesson so i am searching wat i can find on the internet. I first looked at an youtube video where he tried this on python 2, so thats why it's sort of messy but i really want it to work.
import numpy as np
import cv2
import math
from PIL import Image
filepathinQ = "images/isomeren.jpg"
filepathinQPNG = 'images/imageinq.png'
filepathPNG = "images/atomen.png"
###First pictuce scan
class shapeRecognition():
def __init__(self, img):
self.img = img
def Voorbeeldenmaker(self):
im = Image.open('images/isomeren.png')
image = np.array(im)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
max_dimension = max(image.shape)
scale = 700/max_dimension
image = cv2.resize(image, None, fx=scale, fy=scale)
lower = np.array([0,0,0], dtype=np.uint8)
upper = np.array([15,15,15], dtype=np.uint8)
mask = cv2.inRange(self.img, lower, upper)
(flags, contours, h) = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(mask, contours, 0,(120,0,145), 3)
cv2.imshow("vorm", mask)
return contours
#second picture scan
class shape2Recognition():
def __init__(self2, img2):
self2.img2 = img2
def inquestion(self2):
im = Image.open(filepathinQPNG)
image = np.array(im)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
max_dimension = max(image.shape)
scale = 700/max_dimension
image = cv2.resize(image, None, fx=scale, fy=scale)
lower2 = np.array([0,0,0], dtype=np.uint8)
upper2 = np.array([15,15,15], dtype=np.uint8)
mask2 = cv2.inRange(image, lower2, upper2)
(flags, contours2, h) = cv2.findContours(mask2, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(mask2, contours2, 4,(120,0,145), 3)
cv2.imshow("vorm2", mask2)
return contours2
#threshold for the second picture
#to make it in black and white
def treshold(imageArray):
newAr = imageArray
img = Image.open(filepathinQ)
img = img.convert("RGBA")
datas = img.getdata()
newData = []
for eachRow in newAr:
for eachPix in eachRow:
if ((int(eachPix[0]) + int(eachPix[1]) + int(eachPix[2]))/3) > 220:
newData.append((255,255,255,255))
else:
newData.append((0,0,0,255))
img.putdata(newData)
img.save('images/imageinq.png', "PNG")
#comparing the two
def zijnzegelijk(image):
x = 0
while x < 21:
cnt = contours[x]
cnt2 = contours2[4]
ret = cv2.matchShapes(cnt, cnt2 ,1, 0.0)
print(ret)
print("\n")
print(x)
print("\n")
x+= 1
i = Image.open(filepathinQ)
iAr = np.array(i)
treshold(iAr)
img2 = cv2.imread(filepathinQPNG)
shape2Recognition = shape2Recognition(img2)
contours2 = shape2Recognition.inquestion()
i2 = Image.open(filepathinQPNG)
iAr2 = np.array(i2)
img = cv2.imread(filepathPNG)
shapeRecognition = shapeRecognition(img)
contours = shapeRecognition.Voorbeeldenmaker()
zijnzegelijk(iAr2)