Related
I'm trying to set up a fish-eye camera for object localisation with respect to a particular frame of reference.
I tried both the OpenCV fisheye module and the rational model from calibrateCamera() to calibrate. I obtained this result.
I collected 2 different datasets, one with calibration images taken mostly close to the camera (ds1) and a second one with images taken from afar (ds2). ds12 is a dataset obtained merging the two.
nok4 indicates the fisheye model with k4 fixed to 0.
rm is the rational model from cv2.calibrateCamera()
The camera has 178° horizontal FOV and 101° vertical FOV, the distortion is corrected mostly in the center of the image with disappointing results in the outermost parts of the image.
Am I doing something wrong? What could I do to improve the results?
Edit
Here's the code I'm using for both the calibration processes:
import cv2 as cv
import os
import numpy as np
cwd = os.path.dirname(os.path.realpath(__file__))
os.chdir(cwd)
number = None
folder_name = "merged_images"
if number is not None:
folder_name += "_" + str(number)
points_path = os.path.join(folder_name, "dataset", "good_detections")
npz = np.load(os.path.join(points_path, "points.npz"))
square_size = 0.02435
imgpoints = npz["imgpoints"]
objpoints = npz["objpoints"] * square_size
file_names = npz["file_names"]
shuffle = True
if shuffle:
if "indices.npz" in os.listdir(points_path):
p = np.load(os.path.join(points_path, "indices.npz"))["indices"]
else:
print("Random indices assigned")
p = np.random.permutation(len(imgpoints))
imgpoints = imgpoints[p]
objpoints = objpoints[p]
file_names = file_names[p]
img = cv.imread(os.path.join(points_path, file_names[2].replace("detected_", "")))
flag_list = [
cv.CALIB_RATIONAL_MODEL,
# cv.CALIB_ZERO_TANGENT_DIST,
]
calibration_flags = 0
for flag in flag_list:
calibration_flags += flag
gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
shape = gray.shape[::-1]
ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints,
imgpoints,
shape,
None,
None,
flags = calibration_flags
)
h, w = img.shape[:2]
newcameramtx, roi = cv.getOptimalNewCameraMatrix(mtx, dist, (w,h), 1, (w,h))
dst = cv.undistort(img, mtx, dist, None, newcameramtx)
# fisheye model
flag_list = [
cv.fisheye.CALIB_RECOMPUTE_EXTRINSIC,
cv.fisheye.CALIB_CHECK_COND,
cv.fisheye.CALIB_FIX_SKEW,
# cv.fisheye.CALIB_FIX_K4,
# cv.fisheye.CALIB_FIX_K3,
# cv.fisheye.CALIB_FIX_K2,
# cv.fisheye.CALIB_FIX_K1,
]
calibration_flags = 0
for flag in flag_list:
calibration_flags += flag
N_OK = len(objpoints)
K = np.zeros((3, 3))
D = np.zeros((4, 1))
rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
n_objpoints = [np.expand_dims(objp, 0) for objp in objpoints]
all_true_points = list(n_objpoints)
all_image_points = list(imgpoints)
all_frames = list(file_names)
rejected = []
counter = 0
while True:
try:
rms, mtx, dist, rvecs, tvecs = \
cv.fisheye.calibrate(
all_true_points,
all_image_points,
gray.shape[::-1],
K,
D,
rvecs,
tvecs,
calibration_flags,
(cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 1e-3)
)
print('Found a calibration based on {} well-conditioned images.'.format(len(all_true_points)))
break
except cv.error as err:
try:
idx = int(str(err).split('array ')[1][0]) # Parse index of invalid image from error message
all_true_points.pop(idx)
all_image_points.pop(idx)
rejected.append(all_frames.pop(idx))
print(f"{counter}. Removed ill-conditioned image {idx} from the data. Trying again...".format(idx))
counter += 1
except IndexError:
raise err
h,w = img.shape[:2]
DIM = (w, h)
dim1 = img.shape[:2][::-1] # dim1 is the dimension of input image to un-distort
dim2 = None
dim3 = None
balance = 1
assert dim1[0]/dim1[1] == DIM[0]/DIM[1], "Image to undistort needs to have same aspect ratio as the ones used in calibration"
if not dim2:
dim2 = dim1
if not dim3:
dim3 = dim1
scaled_K = K * dim1[0] / DIM[0] # The values of K is to scale with image dimension.
scaled_K[2][2] = 1.0 # Except that K[2][2] is always 1.0
# This is how scaled_K, dim2 and balance are used to determine the final K used to un-distort image. OpenCV document failed to make this clear!
new_K = cv.fisheye.estimateNewCameraMatrixForUndistortRectify(scaled_K, D, dim2, np.eye(3), balance=balance)
map1, map2 = cv.fisheye.initUndistortRectifyMap(scaled_K, D, np.eye(3), new_K, dim3, cv.CV_16SC2)
undistorted_img = cv.remap(img, map1, map2, interpolation=cv.INTER_LINEAR, borderMode=cv.BORDER_CONSTANT)
new_img = cv.hconcat([undistorted_img, img])
Corner extraction is performed with the following code:
objpoints = [] # 3d point in real world space
imgpoints = [] # 2d points in image plane.
file_names = [] # files analyzed
for file in os.listdir(detections_path):
if file.startswith("hd_frame"):
frame = cv.imread(os.path.join(detections_path, file))
gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
found, corners = cv.findChessboardCorners(gray, (9,6), None)
if found:
file_name = "detected_" + file
objpoints.append(objp)
imgpoints.append(corners)
# save file_name
file_names.append(file_name)
corners2 = cv.cornerSubPix(gray, corners, (11,11), (-1,-1), criteria)
# Draw and display the corners
cv.drawChessboardCorners(frame, (9,6), corners2, found)
# save image and points
cv.imwrite(os.path.join(detections_path, file_name), frame)
You can use a single image for calibration as described in details here: https://discorpy.readthedocs.io/en/latest/usage/demo_06.html . The correction model used in this package may give you better results than the model used by opencv: https://discorpy.readthedocs.io/en/latest/tutorials/methods.html
I am trying to extract blood network from this face image: Face image
For such task, i am using the P&M anisotropic diffusion found in this question: Anisotropic diffusion 2d images. Then i am using tophat transform followed by blackhat transform, afterwards i use a simple threshold to set to 255 all pixel that has an intensity value of 100.
The problem is that, after i use the threshold and try to open the image, whatever way i try, the image is displayed as fully black:
In short, my goal is to extract the blood vessels using P&M anisotropic diffusion with structuring element of flat disk of 5x5, then apply tophat and blackhat, respectively and a simple threshold and actually be able to view the image afterwards.
Here's my code on how i am trying it:
import cv2
import import cv2 numpy as np
import warnings
face_img=mpimg.imread('path')
def anisodiff(img, niter=1, kappa=50, gamma=0.1, step=(1., 1.), option=1):
if img.ndim == 3:
m = "Only grayscale images allowed, converting to 2D matrix"
warnings.warn(m)
img = img.mean(2)
img = img.astype('float32')
imgout = img.copy()
deltaS = np.zeros_like(imgout)
deltaE = deltaS.copy()
NS = deltaS.copy()
EW = deltaS.copy()
gS = np.ones_like(imgout)
gE = gS.copy()
for ii in range(niter):
deltaS[:-1, :] = np.diff(imgout, axis=0)
deltaE[:, :-1] = np.diff(imgout, axis=1)
if option == 1:
gS = np.exp(-(deltaS/kappa)**2.)/step[0]
gE = np.exp(-(deltaE/kappa)**2.)/step[1]
elif option == 2:
gS = 1./(1.+(deltaS/kappa)**2.)/step[0]
gE = 1./(1.+(deltaE/kappa)**2.)/step[1]
E = gE*deltaE
S = gS*deltaS
NS[:] = S
EW[:] = E
NS[1:, :] -= S[:-1, :]
EW[:, 1:] -= E[:, :-1]
imgout += gamma*(NS+EW)
return imgout
new_img = anisodiff(face_img, niter=1, kappa=20, gamma=0.1, step=(1., 1.), option=1)
filterSize =(3, 3)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,
filterSize)
input_image = new_img
first_tophat_img = cv2.morphologyEx(input_image,
cv2.MORPH_TOPHAT,
kernel)
filterSize =(3, 3)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,
filterSize)
second_tophat_img = cv2.morphologyEx(input_image,
cv2.MORPH_BLACKHAT,
kernel)
ret, thresh1 = cv2.threshold(second_tophat_img, 200, 255, cv2.THRESH_BINARY)
Even when i set the threshold to 254 for instance, the image goes black.
I executed a simple MATLAB implementation, and got a nice result.
MATLAB code:
I = imread('02_giorgos_1_f_M_30_830.tif');
I = im2double(uint8(I));
J = imdiffusefilt(I);
K = imtophat(J, ones(3));
figure;imshow(imadjust(K, stretchlim(K)));
Result:
I don't know if you know MATLAB, but I used the default arguments of imdiffusefilt (equivalent to anisodiff in your code).
Default MATLAB arguments are equivalent to:
Input image is in range [0, 1] and not [0, 255].
niter=5 (note: you used only 1 iteration and it's not enough).
kappa=0.1
gamma=0.125
MATLAB default is 8 neighbors connectivity (not 4 neighbors as used in anisodiff).
8 neighbors connectivity:
For getting same result as in MATLAB, I implemented an 8 neighbors connectivity Anisotropic diffusion (based on MATLAB source code).
Note: with 4 neighbors connectivity it's working, but result is not so nice as using 8 neighbors.
Displaying the output image:
In order to display the output image correctly, I used imadjust(K, stretchlim(K)).
The command stretches the range of the input image such that percentile 1 goes to 0, and percentile 99 goes to 1 (linear stretch).
One more thing:
Instead of using fixed threshold of 200, I used percentile 95 threshold:
t = np.percentile(first_tophat_img, 95)
ret, thresh1 = cv2.threshold(first_tophat_img, t, 255,
cv2.THRESH_BINARY)
Here is the code (uses cv2.imshow for testing):
import cv2
import numpy as np
import matplotlib.image as mpimg
import warnings
face_img = mpimg.imread('02_giorgos_1_f_M_30_830.tif')
def anisodiff8neighbors(img, niter=5, kappa=0.1, gamma=0.125):
""" See https://www.mathworks.com/help/images/ref/imdiffusefilt.html
Anisotropic diffusion filtering with 8 neighbors
Range of img is assumed to be [0, 1] (not [0, 255]).
"""
if img.ndim == 3:
m = "Only grayscale images allowed, converting to 2D matrix"
warnings.warn(m)
img = img.mean(2)
img = img.astype('float32')
imgout = img.copy()
for ii in range(niter):
# MATLAB source code is commented
#paddedImg = padarray(I, [1 1], 'replicate');
padded_img = np.pad(imgout, (1, 1), 'edge')
#diffImgNorth = paddedImg(1:end-1,2:end-1) - paddedImg(2:end,2:end-1);
#diffImgEast = paddedImg(2:end-1,2:end) - paddedImg(2:end-1,1:end-1);
#diffImgNorthWest = paddedImg(1:end-2,1:end-2) - I;
#diffImgNorthEast = paddedImg(1:end-2,3:end) - I;
#diffImgSouthWest = paddedImg(3:end,1:end-2) - I;
#diffImgSouthEast = paddedImg(3:end,3:end) - I;
diff_img_north = padded_img[0:-1, 1:-1] - padded_img[1:, 1:-1]
diff_img_east = padded_img[1:-1, 1:] - padded_img[1:-1, 0:-1]
diff_img_north_west = padded_img[0:-2, 0:-2] - imgout
diff_img_north_east = padded_img[0:-2, 2:] - imgout
diff_img_south_west = padded_img[2:, 0:-2] - imgout
diff_img_south_east = padded_img[2:, 2:] - imgout
#case 'exponential'
#conductCoeffNorth = exp(-(abs(diffImgNorth)/gradientThreshold).^2);
#conductCoeffEast = exp(-(abs(diffImgEast)/gradientThreshold).^2);
#conductCoeffNorthWest = exp(-(abs(diffImgNorthWest)/gradientThreshold).^2);
#conductCoeffNorthEast = exp(-(abs(diffImgNorthEast)/gradientThreshold).^2);
#conductCoeffSouthWest = exp(-(abs(diffImgSouthWest)/gradientThreshold).^2);
#conductCoeffSouthEast = exp(-(abs(diffImgSouthEast)/gradientThreshold).^2);
conduct_coeff_north = np.exp(-(np.abs(diff_img_north)/kappa)**2.0)
conduct_coeff_east = np.exp(-(np.abs(diff_img_east)/kappa)**2.0)
conduct_coeff_north_west = np.exp(-(np.abs(diff_img_north_west)/kappa)**2.0)
conduct_coeff_north_east = np.exp(-(np.abs(diff_img_north_east)/kappa)**2.0)
conduct_coeff_south_west = np.exp(-(np.abs(diff_img_south_west)/kappa)**2.0)
conduct_coeff_south_east = np.exp(-(np.abs(diff_img_south_east)/kappa)**2.0)
#fluxNorth = conductCoeffNorth .* diffImgNorth;
#fluxEast = conductCoeffEast .* diffImgEast;
#fluxNorthWest = conductCoeffNorthWest .* diffImgNorthWest;
#fluxNorthEast = conductCoeffNorthEast .* diffImgNorthEast;
#fluxSouthWest = conductCoeffSouthWest .* diffImgSouthWest;
#fluxSouthEast = conductCoeffSouthEast .* diffImgSouthEast;
flux_north = conduct_coeff_north * diff_img_north
flux_east = conduct_coeff_east * diff_img_east
flux_north_west = conduct_coeff_north_west * diff_img_north_west
flux_north_east = conduct_coeff_north_east * diff_img_north_east
flux_south_west = conduct_coeff_south_west * diff_img_south_west
flux_south_east = conduct_coeff_south_east * diff_img_south_east
#% Discrete PDE solution
#I = I + diffusionRate * (fluxNorth(1:end-1,:) - fluxNorth(2:end,:) + ...
# fluxEast(:,2:end) - fluxEast(:,1:end-1) + (1/(dd^2)).* fluxNorthWest + ...
# (1/(dd^2)).* fluxNorthEast + (1/(dd^2)).* fluxSouthWest + (1/(dd^2)).* fluxSouthEast);
imgout = imgout + gamma * (flux_north[0:-1,:] - flux_north[1:,:] +
flux_east[:,1:] - flux_east[:,0:-1] + 0.5*flux_north_west +
0.5*flux_north_east + 0.5*flux_south_west + 0.5*flux_south_east)
return imgout
#new_img = anisodiff(face_img, niter=1, kappa=20, gamma=0.1, step=(1., 1.), option=1)
face_img = face_img.astype(float) / 255;
#new_img = anisodiff(face_img, niter=5, kappa=0.1, gamma=0.125, step=(1., 1.), option=1)
new_img = anisodiff8neighbors(face_img, niter=5, kappa=0.1, gamma=0.125)
filterSize =(3, 3)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,
filterSize)
input_image = new_img
first_tophat_img = cv2.morphologyEx(input_image,
cv2.MORPH_TOPHAT,
kernel)
# Use percentile 95 (of image) as threshold instead of fixed threshold 200
t = np.percentile(first_tophat_img, 95)
ret, thresh1 = cv2.threshold(first_tophat_img, t, 255, cv2.THRESH_BINARY)
cv2.imshow('thresh1', thresh1)
filterSize =(3, 3)
kernel = cv2.getStructuringElement(cv2.MORPH_RECT,
filterSize)
second_tophat_img = cv2.morphologyEx(input_image,
cv2.MORPH_BLACKHAT,
kernel)
#ret, thresh1 = cv2.threshold(second_tophat_img, 200, 255, cv2.THRESH_BINARY)
# Use percentile 95 (of image) as threshold instead of fixed threshold 200
t = np.percentile(second_tophat_img, 95)
ret, thresh2 = cv2.threshold(second_tophat_img, t, 255, cv2.THRESH_BINARY)
cv2.imshow('thresh2', thresh2)
lo, hi = np.percentile(first_tophat_img, (1, 99))
first_tophat_img_stretched = (first_tophat_img.astype(float) - lo) / (hi-lo) # Apply linear "stretch" - lo goes to 0, and hi goes to 1
cv2.imshow('first_tophat_img_stretched', first_tophat_img_stretched)
cv2.waitKey()
cv2.destroyAllWindows()
Result:
thresh1:
thresh2:
first_tophat_img_stretched:
INFO:
I've calibrated my camera and have found the camera's intrinsics matrix (K) and its distortion coefficients (d) to be the following:
import numpy as np
K = np.asarray([[556.3834638575809,0,955.3259939726225],[0,556.2366649196925,547.3011305411478],[0,0,1]])
d = np.asarray([[-0.05165940570900624],[0.0031093602070252167],[-0.0034036648250202746],[0.0003390345044343793]])
From here, I can undistort my image using the following three lines:
final_K = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(K, d, (1920, 1080), np.eye(3), balance=1.0)
map_1, map_2 = cv2.fisheye.initUndistortRectifyMap(K, d, np.eye(3), final_K, (1920, 1080), cv2.CV_32FC1)
undistorted_image = cv2.remap(image, map_1, map_2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
The resulting undistored images appears to be correct Left image is distorted, right is undistorted, but when I try to undistort image points using cv2.remap() points aren't mapped to the same location as their corresponding pixel in the image. I detected the calibration board points in the left image using
ret, corners = cv2.findChessboardCorners(gray, (6,8),cv2.CALIB_CB_ADAPTIVE_THRESH+cv2.CALIB_CB_FAST_CHECK+cv2.CALIB_CB_NORMALIZE_IMAGE)
corners2 = cv2.cornerSubPix(gray, corners, (3,3), (-1,-1), (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1))
then remapped those points in the following way:
remapped_points = []
for corner in corners2:
remapped_points.append(
(map_1[int(corner[0][1]), int(corner[0][0])], map_2[int(corner[0][1]), int(corner[0][0])])
)
In these horizontally concatenated images, the left image shows the points detected in the distorted image, while the right image shows the remapped location of the points in the right image.
Also, I haven't been able to get correct results using cv2.fisheye.undistortPoints(). I have the following function to undistort points:
def undistort_list_of_points(point_list, in_K, in_d):
K = np.asarray(in_K)
d = np.asarray(in_d)
# Input can be list of bbox coords, poly coords, etc.
# TODO -- Check if point behind camera?
points_2d = np.asarray(point_list)
points_2d = points_2d[:, 0:2].astype('float32')
points2d_undist = np.empty_like(points_2d)
points_2d = np.expand_dims(points_2d, axis=1)
result = np.squeeze(cv2.fisheye.undistortPoints(points_2d, K, d))
fx = K[0, 0]
fy = K[1, 1]
cx = K[0, 2]
cy = K[1, 2]
for i, (px, py) in enumerate(result):
points2d_undist[i, 0] = px * fx + cx
points2d_undist[i, 1] = py * fy + cy
return points2d_undist
This image shows the results when undistorting using the above function.
(this is all running in OpenCV 4.2.0 on Ubuntu 18.04 in Python 3.6.8)
QUESTIONS
Why isn't this remapping of image coordinates working properly? Am I using map_1 and map_2 incorrectly?
Why are the results from using cv2.fisheye.undistortPoints() different from using map_1 and map_2?
Answer to Q1:
You are not using map_1 and map_2 correctly.
The map generate by the cv2.fisheye.initUndistortRectifyMap function should be the mapping of the pixel location of the destination image to the pixel location of the source image, i.e. dst(x,y)=src(mapx(x,y),mapy(x,y)). see remap in OpenCV.
In the code, map_1 is for the x-direction pixel mapping and map_2 is for the y-direction pixel mapping. For example,
(X_undistorted, Y_undistorted) is the pixel location in the undistorted image. map_1[Y_undistorted, X_undistorted] gives you where is this pixel should map to the x coordinate in the distorted image, and map_2 will give you the corresponding y coordinate.
So, map_1 and map_2 are useful for constructing an undistorted image from a distorted image, and not really suitable for the reversed process.
remapped_points = []
for corner in corners2:
remapped_points.append(
(map_1[int(corner[0][1]), int(corner[0][0])], map_2[int(corner[0][1]), int(corner[0][0])]))
This code to find the undistorted pixel location of the corners is not correct. You will need to use undistortPoints function.
Answer to Q2:
The mapping and undistortion are different.
You can think of mapping as constructing the undistorted image based on the pixel locations in the undistorted image with the pixel maps, while undistortion is to find undistorted pixel locations using the original pixel location using lens distortion model.
In order to find the correct pixel locations of the corners in the undistorted image. You need to convert the normalized coordinates of the undistorted points back to pixel coordinates using the newly estimated K, in your case, it's the final_K, because the undistorted image can be seen as taken by a camera with the final_K without distortion (there is a small zooming effect).
Here is the modified undistort function:
def undistort_list_of_points(point_list, in_K, in_d, in_K_new):
K = np.asarray(in_K)
d = np.asarray(in_d)
# Input can be list of bbox coords, poly coords, etc.
# TODO -- Check if point behind camera?
points_2d = np.asarray(point_list)
points_2d = points_2d[:, 0:2].astype('float32')
points2d_undist = np.empty_like(points_2d)
points_2d = np.expand_dims(points_2d, axis=1)
result = np.squeeze(cv2.fisheye.undistortPoints(points_2d, K, d))
K_new = np.asarray(in_K_new)
fx = K_new[0, 0]
fy = K_new[1, 1]
cx = K_new[0, 2]
cy = K_new[1, 2]
for i, (px, py) in enumerate(result):
points2d_undist[i, 0] = px * fx + cx
points2d_undist[i, 1] = py * fy + cy
return points2d_undist
Here is my code for doing the same thing.
import cv2
import numpy as np
import matplotlib.pyplot as plt
K = np.asarray([[556.3834638575809,0,955.3259939726225],[0,556.2366649196925,547.3011305411478],[0,0,1]])
D = np.asarray([[-0.05165940570900624],[0.0031093602070252167],[-0.0034036648250202746],[0.0003390345044343793]])
print("K:\n", K)
print("D:\n", D.ravel())
# read image and get the original image on the left
image_path = "sample.jpg"
image = cv2.imread(image_path)
image = image[:, :image.shape[1]//2, :]
image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
fig = plt.figure()
plt.imshow(image_gray, "gray")
H_in, W_in = image_gray.shape
print("Grayscale Image Dimension:\n", (W_in, H_in))
scale_factor = 1.0
balance = 1.0
img_dim_out =(int(W_in*scale_factor), int(H_in*scale_factor))
if scale_factor != 1.0:
K_out = K*scale_factor
K_out[2,2] = 1.0
K_new = cv2.fisheye.estimateNewCameraMatrixForUndistortRectify(K_out, D, img_dim_out, np.eye(3), balance=balance)
print("Newly estimated K:\n", K_new)
map1, map2 = cv2.fisheye.initUndistortRectifyMap(K, D, np.eye(3), K_new, img_dim_out, cv2.CV_32FC1)
print("Rectify Map1 Dimension:\n", map1.shape)
print("Rectify Map2 Dimension:\n", map2.shape)
undistorted_image_gray = cv2.remap(image_gray, map1, map2, interpolation=cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT)
fig = plt.figure()
plt.imshow(undistorted_image_gray, "gray")
ret, corners = cv2.findChessboardCorners(image_gray, (6,8),cv2.CALIB_CB_ADAPTIVE_THRESH+cv2.CALIB_CB_FAST_CHECK+cv2.CALIB_CB_NORMALIZE_IMAGE)
corners_subpix = cv2.cornerSubPix(image_gray, corners, (3,3), (-1,-1), (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1))
undistorted_corners = cv2.fisheye.undistortPoints(corners_subpix, K, D)
undistorted_corners = undistorted_corners.reshape(-1,2)
fx = K_new[0,0]
fy = K_new[1,1]
cx = K_new[0,2]
cy = K_new[1,2]
undistorted_corners_pixel = np.zeros_like(undistorted_corners)
for i, (x, y) in enumerate(undistorted_corners):
px = x*fx + cx
py = y*fy + cy
undistorted_corners_pixel[i,0] = px
undistorted_corners_pixel[i,1] = py
undistorted_image_show = cv2.cvtColor(undistorted_image_gray, cv2.COLOR_GRAY2BGR)
for corner in undistorted_corners_pixel:
image_corners = cv2.circle(np.zeros_like(undistorted_image_show), (int(corner[0]),int(corner[1])), 15, [0, 255, 0], -1)
undistorted_image_show = cv2.add(undistorted_image_show, image_corners)
fig = plt.figure()
plt.imshow(undistorted_image_show, "gray")
I am working on a stereo camera rig with 4 discrete cameras (of the same type) but at the moment only one pair (cam1 and cam2) are necessary.
The aim is to calibrate the stereo pair and get 3D information about the scene. I am using Python 3.6 with OpenCV 3.4.3 in Visual Studio 2017.
I took 28 images of a chessboard calibration pattern and calibrated the cameras individually as well as stereoscopic with the standard OpenCV procedure.
Since the calibration data seems good and the distortion correction is working fine, the next step is the rectification of the images.
This is where things become weird. I spent the last 3 weeks working on this and read a lot, tried a lot and always got crappy results. I used cv2.stereoCalibrate (also tried with different flags, as suggested in different topics), cv2.stereoRectify (also with different alpha values), cv2.initUndistortRectifyMap and cv2.remap for the actual remapping of the images (method 1). But the results are never as wanted.
I recently managed to get rectified images which look like they are actually rectified with cv2.uncalibratedRectification. Therefore I did not use matched points (since SURF and SIFT are unfree...) but a slightly different approach. The edges of the calibration pattern in my 28 calibration images are used as input points. This works good, but the rectified images don't look perfect.
Here are my images (these are no calibration images) so you can imagine what I'm talking about:
original left and right images
undistorted images
rectified with method 1, alpha=1
rectified with method 1, alpha=0
rectified uncalibrated, best result I got by now
Can anybody give me a hint whats wrong with my usage of method 1? I've seen a lot posts to similar problems but I couldn't find the solution in the comments there. Or is this a bug in OpenCV?
Or has anyone an idea how to improve the uncalibrated rectification?
Here is a code snippet with the relevant calls:
# imports
import numpy as np
import cv2
import glob
import argparse
import sys
import os
# size calib array
numEdgeX = 10
numEdgeY = 7
# preface
exitCode = 0
# get directories
pathDir = str(os.path.dirname(os.path.realpath(__file__)))
pathDir = pathDir[:-17]
pathCalib = pathDir + "\\CalibData" + "\\chess"
try:
# define pair
p = 1
cal_path = pathCalib + "\\pair" + str(p)
images_right = glob.glob(cal_path + '\RIGHT/*.bmp')
images_left = glob.glob(cal_path + '\LEFT/*.bmp')
images_left.sort()
images_right.sort()
# termination criteria
criteria = (cv2.TermCriteria_EPS +
cv2.TermCriteria_MAX_ITER, 30, 0.001)
criteria_cal = (cv2.TermCriteria_EPS +
cv2.TermCriteria_MAX_ITER, 30, 1e-5)
# prepare object points, like (0,0,0); (1,0,0); ...; (6,5,0)
objp = np.zeros((numEdgeX*numEdgeY, 3), np.float32)
objp[:, :2] = np.mgrid[0:numEdgeX, 0:numEdgeY].T.reshape(-1, 2)
objpoints = [] # 3d points in real world space
imgpoints_l = [] # 2d points in image plane for calibration
imgpoints_r = [] # 2d points in image plane for calibration
for i, fname in enumerate(images_right):
print(str(i+1) + " out of " + str(len(images_right)))
img_l = cv2.imread(images_left[i])
img_r = cv2.imread(images_right[i])
# convert to cv2
img_l = cv2.cvtColor(img_l, cv2.COLOR_BGR2GRAY)
img_r = cv2.cvtColor(img_r, cv2.COLOR_BGR2GRAY)
# find the chess board corners
ret_l, corners_l = cv2.findChessboardCorners(img_l, (numEdgeX, numEdgeY), None)
ret_r, corners_r = cv2.findChessboardCorners(img_r, (numEdgeX, numEdgeY), None)
objpoints.append(objp)
if ret_l is True:
print("image " + str(i+1) + "left - io")
rt = cv2.cornerSubPix(img_l, corners_l, (11, 11),
(-1, -1), criteria)
imgpoints_l.append(corners_l)
if ret_r is True:
print("image " + str(i+1) + "right - io")
rt = cv2.cornerSubPix(img_r, corners_r, (11, 11),
(-1, -1), criteria)
imgpoints_r.append(corners_r)
# get shape
img_shape = img_l.shape[::-1]
### CALIBRATION ###
# calibrate left camera
rt, M1, d1, r1, t1 = cv2.calibrateCamera(
objpoints, imgpoints_l, img_shape, None, None)
# calibrate right camera
rt, M2, d2, r2, t2 = cv2.calibrateCamera(
objpoints, imgpoints_r, img_shape, None, None)
# stereo calibration
flags = (cv2.CALIB_FIX_K5 + cv2.CALIB_FIX_K6)
stereocalib_criteria = (cv2.TERM_CRITERIA_MAX_ITER +
cv2.TERM_CRITERIA_EPS, 100, 1e-5)
#flags = 0
#flags = cv2.CALIB_USE_INTRINSIC_GUESS
#flags = cv2.CALIB_FIX_PRINCIPAL_POINT
#flags = cv2.CALIB_FIX_ASPECT_RATIO
#flags = cv2.CALIB_ZERO_TANGENT_DIST
#flags = cv2.CALIB_FIX_INTRINSIC
#flags = cv2.CALIB_FIX_FOCAL_LENGTH
#flags = cv2.CALIB_FIX_K1...6
#flags = cv2.CALIB_RATIONAL_MODEL
#flags = cv2.CALIB_THIN_PRISM_MODEL
#flags = cv2.CALIB_SAME_FOCAL_LENGTH
#flags = cv2.CALIB_FIX_S1_S2_S3_S4
flags = (cv2.CALIB_FIX_PRINCIPAL_POINT | cv2.CALIB_FIX_ASPECT_RATIO | cv2.CALIB_FIX_FOCAL_LENGTH |
cv2.CALIB_FIX_INTRINSIC | cv2.CALIB_FIX_K3 | cv2.CALIB_FIX_K4 | cv2.CALIB_FIX_K5 |
cv2.CALIB_FIX_K6)
T = np.zeros((3, 1), dtype=np.float64)
R = np.eye(3, dtype=np.float64)
ret, M1, d1, M2, d2, R, T, E, F = cv2.stereoCalibrate(
objpoints, imgpoints_l,
imgpoints_r, M1, d1, M2,
d2, img_shape,
criteria = stereocalib_criteria,
flags=flags)
# get new optimal camera matrix
newCamMtx1, roi1 = cv2.getOptimalNewCameraMatrix(M1, d1, img_shape, 0, img_shape)
newCamMtx2, roi2 = cv2.getOptimalNewCameraMatrix(M2, d2, img_shape, 0, img_shape)
# rectification and undistortion maps which can be used directly to correct the stereo pair
(rectification_l, rectification_r, projection_l,
projection_r, disparityToDepthMap, ROI_l, ROI_r) = cv2.stereoRectify(
M1, d1, M2, d2, img_shape, R, T,
None, None, None, None, None,
#cv2.CALIB_ZERO_DISPARITY, # principal points of each camera have the same pixel coordinates in rect views
alpha=0) # alpha=1 no pixels lost, alpha=0 pixels lost
leftMapX, leftMapY = cv2.initUndistortRectifyMap(
M1, d1, rectification_l, projection_l,
img_shape, cv2.CV_32FC1)
rightMapX, rightMapY = cv2.initUndistortRectifyMap(
M2, d2, rectification_r, projection_r,
img_shape, cv2.CV_32FC1)
### UNCALIBRATED RECTIFICATION ###
imgpoints_l_undis = []
imgpoints_r_undis = []
for i, fname in enumerate(images_right):
img_l = cv2.imread(images_left[i])
img_r = cv2.imread(images_right[i])
# convert to cv2
img_l = cv2.cvtColor(img_l, cv2.COLOR_BGR2GRAY)
img_r = cv2.cvtColor(img_r, cv2.COLOR_BGR2GRAY)
# undistort
img_l_undis = cv2.undistort(img_l, M1, d1, None, newCamMtx1)
img_r_undis = cv2.undistort(img_r, M2, d2, None, newCamMtx2)
# find the chess board corners in undistorted image
ret_l_undis, corners_l_undis = cv2.findChessboardCorners(img_l_undis, (numEdgeX, numEdgeY), None)
ret_r_undis, corners_r_undis = cv2.findChessboardCorners(img_r_undis, (numEdgeX, numEdgeY), None)
if ret_l_undis is True:
rt = cv2.cornerSubPix(img_l_undis, corners_l_undis, (11, 11), (-1, -1), criteria)
for j in range(0, len(rt)):
x = rt[j][0,:]
imgpoints_l_undis.append(x)
if ret_r_undis is True:
rt = cv2.cornerSubPix(img_r_undis, corners_r_undis, (11, 11), (-1, -1), criteria)
for j in range(0, len(rt)):
x = rt[j][0,:]
imgpoints_r_undis.append(x)
# convert to np array
imgpoints_l_undis = np.array(imgpoints_l_undis)
imgpoints_r_undis = np.array(imgpoints_r_undis)
# compute rectification uncalibrated
ret, uncRectMtx1, uncRectMtx2 = cv2.stereoRectifyUncalibrated(imgpoints_l_undis, imgpoints_r_undis, F, img_shape)
### REMAPPING ###
# load images and convert to cv2 format
img_l = cv2.imread(images_left[0])
img_l = cv2.cvtColor(img_l, cv2.COLOR_BGR2GRAY)
img_l_undis = cv2.undistort(img_l, M1, d1, None, newCamMtx1)
img_r = cv2.imread(images_right[0])
img_r = cv2.cvtColor(img_r, cv2.COLOR_BGR2GRAY)
img_r_undis = cv2.undistort(img_r, M2, d2, None, newCamMtx2)
# remap
imglCalRect = cv2.remap(img_l, leftMapX, leftMapY, cv2.INTER_LINEAR)
imgrCalRect = cv2.remap(img_r, rightMapX, rightMapY, cv2.INTER_LINEAR)
numpyHorizontalCalibRect = np.hstack((imglCalRect, imgrCalRect))
# warp for uncalibrated rectification
imglUncalRect = cv2.warpPerspective(img_l_undis, uncRectMtx1, img_shape)
imgrUncalRect = cv2.warpPerspective(img_r_undis, uncRectMtx2, img_shape)
numpyHorizontalUncalibRect = np.hstack((imglUncalRect, imgrUncalRect))
### SHOW RESULTS ###
# calculate point arrays for epipolar lines
lineThickness = 5
lineColor = [0, 255, 0]
numLines = 20
interv = round(img_shape[0] / numLines)
x1 = np.zeros((numLines, 1))
y1 = np.zeros((numLines, 1))
x2 = np.full((numLines, 1), (3*img_shape[1]))
y2 = np.zeros((numLines, 1))
for jj in range(0, numLines):
y1[jj] = jj * interv
y2 = y1
for jj in range(0, numLines):
cv2.line(numpyHorizontalCalibRect, (x1[jj], y1[jj]), (x2[jj], y2[jj]),
lineColor, lineThickness)
cv2.line(numpyHorizontalUncalibRect, (x1[jj], y1[jj]), (x2[jj], y2[jj]),
lineColor, lineThickness)
cv2.namedWindow("calibRect", cv2.WINDOW_NORMAL)
cv2.namedWindow("uncalibRect", cv2.WINDOW_NORMAL)
cv2.imshow("calibRect", numpyHorizontalCalibRect)
cv2.imshow("uncalibRect", numpyHorizontalUncalibRect)
cv2.waitKey()
except (IOError, ValueError):
print("An I/O error or a ValueError occurred")
except:
print("An unexpected error occurred")
raise
Thanks!
Done!
The issue was that OpenCV interpreted my images as a vertical stereo system, I just looked at it as it was horizontal.
Error: error: (-215) (unsigned)labels[i] < (unsigned)K in function kmeans
self.cluster[i] represents some calculated pixel position.
img = numpy.asarray(img)
Z = img.reshape((-1,3))
Z = numpy.float32(Z)
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
#prepare centers
labels = numpy.zeros(k)
for i in range(k):
labels[i] = self.cluster[i].j * img.shape[0] + self.cluster[i].i
print(Z)
ret, label, center = cv2.kmeans(Z, k, None, criteria, 10, cv2.KMEANS_USE_INITIAL_LABELS, labels)
I have read the other posts about the same error in c++, but I could not get it to work. Please help!
EDIT1:
import numpy as np
import sys
sys.path.append('/usr/local/lib/python3.5/site-packages')
import cv2
img = cv2.imread('image.jpg')
img = cv2.resize(img, (90,90))
Z = img.reshape((-1,3))
# convert to np.float32
Z = np.float32(Z)
# define criteria, number of clusters(K) and apply kmeans()
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
K = 4
labels = np.zeros(K)
labels[0] = 1
labels[1] = 100
labels[2] = 500
labels[3] = 1000
print(labels)
print(Z.shape)
ret,label,center=cv2.kmeans(Z,K,None,criteria,10,cv2.KMEANS_USE_INITIAL_LABELS, labels)
# Now convert back into uint8, and make original image
center = np.uint8(center)
res = center[label.flatten()]
res2 = res.reshape((img.shape))
cv2.imshow('res2',res2)
cv2.waitKey(0)
cv2.destroyAllWindows()
You can run this code alone to check what's wrong. By resizing the image to less than 90x90 causes it to crash. Anything bigger than 91x91 works. Can anyone explain why and maybe how to fix it?
My problem solved when using cv2.KMEANS_PP_CENTERS or cv2.KMEANS_RANDOM_CENTERS instead of cv2.KMEANS_USE_INITIAL_LABELS.