Related
I have an input image which contains a letter inside a box, what I want to do is to fix the perspective of the image so I can easily use it with my letter recognition model.
This is the code that I use for pre-processing:
def preprocess(image):
# Convert to grayscale
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# NOISE REMOVAL - START
# Apply CLAHE (makes the image lightness more uniform)
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
image = clahe.apply(image)
cv2.imwrite("preprocess_stages/0_clahe.png", image)
# Apply NlMeansDenoising (removes noise but loses some details)
image = cv2.fastNlMeansDenoising(image, None, 10, 7, 21)
cv2.imwrite("preprocess_stages/1_nl_means_denoising.png", image)
# Apply GaussianBlur with a medium kernel (removes noise but loses some edges)
image = cv2.GaussianBlur(image, (5, 5), 0)
cv2.imwrite("preprocess_stages/2_gaussian_blur.png", image)
# NOISE REMOVAL - END
# Apply OTSU thresholding (we need the letter to be white)
_, image = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
cv2.imwrite("preprocess_stages/3_otsu.png", image)
# Morphological erosion (removes noise and small objects)
kernel = np.ones((10, 10), np.uint8)
image = cv2.erode(image, kernel, iterations=1)
cv2.imwrite("preprocess_stages/4_morphological_erosion.png", image)
return image
I have seen many example on the internet and most of them require the 4 corner points to be provided as input too... there are few examples on how to calculate them and most of them don't work for my case, nonetheless I've tried writing something:
UPDATE: I successfully wrote a working code, even though I changed the Morphological Opening to a simple Erosion since it works better for me.
def fix_perspective_convex_hull(preprocessed):
# RETR_TREE retrieves all of the contours and reconstructs a full hierarchy of nested contours
# used for maximum accuracy (my deduction)
contours, _ = cv2.findContours(preprocessed, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# Draw all contours to a new image
image = cv2.cvtColor(preprocessed, cv2.COLOR_GRAY2BGR)
cv2.drawContours(image, contours, -1, (0, 255, 0), 2)
cv2.imwrite("perspective_stages/0_contours.png", image)
for i, contour in enumerate(contours):
hull = cv2.convexHull(contour)
# Reduce to quadrilateral
epsilon = 0.1 * cv2.arcLength(hull, True)
approximated = cv2.approxPolyDP(hull, epsilon, True)
# Draw approximated contour
image = cv2.cvtColor(preprocessed, cv2.COLOR_GRAY2BGR)
cv2.drawContours(image, [approximated], -1, (255, 0, 0), 1)
cv2.imwrite("perspective_stages/1_approximated_contour_{}.png".format(i), image)
if len(approximated) == 4:
image = cv2.cvtColor(preprocessed, cv2.COLOR_GRAY2BGR)
cv2.drawContours(image, [approximated], -1, (0, 0, 255), 2)
cv2.imwrite("perspective_stages/2_final_contour.png", image)
letter_hull = [a[0] for a in approximated] # Remove extra dimension [[x, y]] -> [x, y]
break
# Create placeholder for rectangle points
rectangle = np.zeros((4, 2), dtype="float32")
# Top-left point has smallest sum
# Bottom-right point has largest sum
# sum = x + y
s = np.sum(letter_hull, axis=1)
rectangle[0] = letter_hull[np.argmin(s)]
rectangle[2] = letter_hull[np.argmax(s)]
# Top-right point has smallest difference
# Bottom-left point has largest difference
# difference = x - y
d = np.diff(letter_hull, axis=1)
rectangle[1] = letter_hull[np.argmin(d)]
rectangle[3] = letter_hull[np.argmax(d)]
# Compute width and height of new image based on points
(top_left, top_right, bottom_right, bottom_left) = rectangle
# Width may be either top-right to top-left or bottom-right to bottom-left
# TL;DR: Pythagorean theorem
width_top = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
width_bottom = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
width = max(int(width_top), int(width_bottom))
# Height may be either top-right to bottom-right or top-left to bottom-left
# TL;DR: Pythagorean theorem
height_right = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
height_left = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
height = max(int(height_right), int(height_left))
# Create destination points
destination = np.array([
[0, 0],
[width - 1, 0],
[width - 1, height - 1],
[0, height - 1]
], dtype="float32")
# Compute perspective transform matrix
matrix = cv2.getPerspectiveTransform(rectangle, destination)
warped = cv2.warpPerspective(preprocessed, matrix, (width, height))
# Add padding
warped = cv2.copyMakeBorder(warped, 10, 10, 10, 10, cv2.BORDER_CONSTANT, value=(0, 0, 0))
# Return warped image
return warped
Results:
Original
Preprocessed
Finding a quadrilateral
Warped
Inverted and resized for the neural network
UPDATE 2: My code only works for letters without curves, such as the H. When I try to run the code on curve letters (such as the S) I get the following results:
How can I go about trying to order the items of a picture from top left to bottom right, such as in the image below? Currently receiving this error with the following code .
Error:
a = sorted(keypoints, key=lambda p: (p[0]) + (p1))[0] # find upper left point
ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all()
This question is modelled from this: Ordering coordinates from top left to bottom right
def preprocess(img):
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_blur = cv2.GaussianBlur(img_gray, (5, 5), 1)
img_canny = cv2.Canny(img_blur, 50, 50)
kernel = np.ones((3, 3))
img_dilate = cv2.dilate(img_canny, kernel, iterations=2)
img_erode = cv2.erode(img_dilate, kernel, iterations=1)
return img_erode
image_final = preprocess(picture_example.png)
keypoints, hierarchy = cv2.findContours(image_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
points = []
while len(keypoints) > 0:
a = sorted(keypoints, key=lambda p: (p[0]) + (p[1]))[0] # find upper left point
b = sorted(keypoints, key=lambda p: (p[0]) - (p[1]))[-1] # find upper right point
cv2.line(image_final, (int(a.pt[0]), int(a.pt[1])), (int(b.pt[0]), int(b.pt[1])), (255, 0, 0), 1)
# convert opencv keypoint to numpy 3d point
a = np.array([a.pt[0], a.pt[1], 0])
b = np.array([b.pt[0], b.pt[1], 0])
row_points = []
remaining_points = []
for k in keypoints:
p = np.array([k.pt[0], k.pt[1], 0])
d = k.size # diameter of the keypoint (might be a theshold)
dist = np.linalg.norm(np.cross(np.subtract(p, a), np.subtract(b, a))) / np.linalg.norm(b) # distance between keypoint and line a->b
if d/2 > dist:
row_points.append(k)
else:
remaining_points.append(k)
points.extend(sorted(row_points, key=lambda h: h.pt[0]))
keypoints= remaining_points
New Picture:
Reference Ordering Picture:
Will use center of mass to determine center point ordering.
The resulting numbering depends on how many rows you want there to be. With the program I will show you how to make, you can specify the number of rows before you run the program.
For example, here is the original image:
Here is the numbered image when you specify 4 rows:
Here is the numbered image when you specify 6 rows:
For the other image you provided (with its frame cropped so the frame won't be detected as a shape), you can see there will be 4 rows, so putting 4 into the program will give you:
Let's have a look at the workflow considering 4 rows. The concept I used is to divide the image into 4 segments along the y axis, forming 4 rows. For each segment of the image, find every shape that has its center in that segment. Finally, order the shapes in each segment by their x coordinate.
Import the necessary libraries:
import cv2
import numpy as np
Define a function that will take in an image input and return the image processed to something that will allow python to later retrieve their contours:
def process_img(img):
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_canny = cv2.Canny(img_gray, 100, 100)
kernel = np.ones((2, 3))
img_dilate = cv2.dilate(img_canny, kernel, iterations=1)
img_erode = cv2.erode(img_dilate, kernel, iterations=1)
return img_erode
Define a function that will return the center of a contour:
def get_centeroid(cnt):
length = len(cnt)
sum_x = np.sum(cnt[..., 0])
sum_y = np.sum(cnt[..., 1])
return int(sum_x / length), int(sum_y / length)
Define a function that will take in a processed image and return the center points of the shapes found in the image:
def get_centers(img):
contours, hierarchies = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
for cnt in contours:
if cv2.contourArea(cnt) > 100:
yield get_centeroid(cnt)
Define a function that will take in an image array, img, an array of coordinates, centers, the number of segments for the image, row_amt, and the height of each segment, row_h, as input. It will return row_amt arrays (sorted by their x coordinates), each containing every point in centers that lies in its corresponding row of the image:
def get_rows(img, centers, row_amt, row_h):
centers = np.array(centers)
d = row_h / row_amt
for i in range(row_amt):
f = centers[:, 1] - d * i
a = centers[(f < d) & (f > 0)]
yield a[a.argsort(0)[:, 0]]
Read in the image, get its processed form using the processed function defined, and get the center of each shape in the image using the centers function defined:
img = cv2.imread("shapes.png")
img_processed = process_img(img)
centers = list(get_centers(img_processed))
Get the height of the image to use for the get_rows function defined, and define a count variable, count, to keep track of the numbering:
h, w, c = img.shape
count = 0
Loop through the centers of the shape divided into 4 rows, drawing the line that connects the rows for visualization:
for row in get_rows(img, centers, 4, h):
cv2.polylines(img, [row], False, (255, 0, 255), 2)
for x, y in row:
Add to the count variable, and draw the count onto the specific location on the image from the row array:
count += 1
cv2.circle(img, (x, y), 10, (0, 0, 255), -1)
cv2.putText(img, str(count), (x - 10, y + 5), 1, cv2.FONT_HERSHEY_PLAIN, (0, 255, 255), 2)
Finally, show the image:
cv2.imshow("Ordered", img)
cv2.waitKey(0)
Altogether:
import cv2
import numpy as np
def process_img(img):
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img_canny = cv2.Canny(img_gray, 100, 100)
kernel = np.ones((2, 3))
img_dilate = cv2.dilate(img_canny, kernel, iterations=1)
img_erode = cv2.erode(img_dilate, kernel, iterations=1)
return img_erode
def get_centeroid(cnt):
length = len(cnt)
sum_x = np.sum(cnt[..., 0])
sum_y = np.sum(cnt[..., 1])
return int(sum_x / length), int(sum_y / length)
def get_centers(img):
contours, hierarchies = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
for cnt in contours:
if cv2.contourArea(cnt) > 100:
yield get_centeroid(cnt)
def get_rows(img, centers, row_amt, row_h):
centers = np.array(centers)
d = row_h / row_amt
for i in range(row_amt):
f = centers[:, 1] - d * i
a = centers[(f < d) & (f > 0)]
yield a[a.argsort(0)[:, 0]]
img = cv2.imread("shapes.png")
img_processed = process_img(img)
centers = list(get_centers(img_processed))
h, w, c = img.shape
count = 0
for row in get_rows(img, centers, 4, h):
cv2.polylines(img, [row], False, (255, 0, 255), 2)
for x, y in row:
count += 1
cv2.circle(img, (x, y), 10, (0, 0, 255), -1)
cv2.putText(img, str(count), (x - 10, y + 5), 1, cv2.FONT_HERSHEY_PLAIN, (0, 255, 255), 2)
cv2.imshow("Ordered", img)
cv2.waitKey(0)
This is not completly the same task as in your linked question you took the code from:
You have contours, while the other question has points.
You have to come up with a method to sort contours (they might overlap in one dimension and so on...). There are multiple ways to do that, depending on your use case. The easiest might be to use the center of mass of your contour. This can be done like here: Center of mass in contour (Python, OpenCV). Then you can make an array of objects out of it, that contain points and use the code that you found.
The code that you found assumes that the points are basically more or less on a grid. So all the points 1-5 on your reference image are roughly on a line. In the new picture you posted, this is not really the case. It might be better to go for a clustering approach here: Cluster the center points y coordinates with some aproach (maybe one from here). Then for each cluster: sort the elements by there centers x coordinate.
As I already said there are multiple ways in doing that and it depends hardly on your use case.
For this image, I tried to use hough cirlce to find the center of the "black hole".
After playing with the parameters of cv2.HoughCircles for a long time, the following is the best I can get.
raw image:
# reproducible code for stackoverflow
import cv2
import os
import sys
from matplotlib import pyplot as plt
import numpy as np
# read image can turn it gray
img = cv2.imread(FILE)
cimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img_gray = dst = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
plt.figure(figsize = (18,18))
plt.imshow(cimg, cmap = "gray")
# removing noises
element = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
closing = cv2.morphologyEx(y, cv2.MORPH_CLOSE, element, iterations = 7)
plt.figure(figsize = (12,12))
plt.imshow(closing, cmap = "gray")
# try to find the circles
circles = cv2.HoughCircles(closing,cv2.HOUGH_GRADIENT,3,50,
param1=50,param2=30,minRadius=20,maxRadius=50)
circles = np.uint16(np.around(circles))
for i in circles[0,:]:
# draw the outer circle
cv2.circle(cimg,(i[0],i[1]),i[2],(0,255,0),2)
# draw the center of the circle
cv2.circle(cimg,(i[0],i[1]),2,(0,0,255),3)
plt.figure(figsize = (12,12))
plt.imshow(cimg)
Update::
The one with Canny:
edges = cv2.Canny(closing, 100, 300)
plt.figure(figsize = (12,12))
plt.imshow(edges, cmap = "gray")
circles = cv2.HoughCircles(edges,cv2.HOUGH_GRADIENT,2,50,
param1=50,param2=30,minRadius=20,maxRadius=60)
circles = np.uint16(np.around(circles))
cimg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
for i in circles[0,:]:
# draw the outer circle
cv2.circle(cimg,(i[0],i[1]),i[2],(0,255,0),2)
# draw the center of the circle
cv2.circle(cimg,(i[0],i[1]),2,(0,0,255),3)
plt.figure(figsize = (12,12))
plt.imshow(cimg)
Still not the right circle that is wanted.
Update:
#crackanddie
Sometimes there is 6 or 9 in the identity number.
The circle in 6 or 9 is not very round.
Is there any way to filter that out?
This is an alternative method if you do not want to implement or fiddle with Hough's parameters. You must be sure there's at least one circle visible in your picture. The idea is to create a segmentation mask based on the CMYK color space and filter the blobs of interest by circularity and area. These are the steps:
Convert the image from BGR to CMYK
Threshold the K channel to get a binary mask
Filter blobs by circularity and area
Approximate the filtered blobs as circles
I'm choosing the CMYK color space because the circle is mostly black. The K (key) channel (in this case - black) should do a good job of representing the blob of interest, albeit, with some noise - as usual. Let's see the code:
# Imports:
import cv2
import numpy as np
# image path
path = "D://opencvImages//"
fileName = "dyj3O.jpg"
# load image
bgr = cv2.imread(path + fileName)
Alright, we need to convert the image from BGR to CMYK. OpenCV does not offer the conversion, so we need to do it manually. The formula is very straightforward. I'm just interested on the K channel, so I just calculate it like this:
# Make float and divide by 255:
bgrFloat = bgr.astype(np.float) / 255.
# Calculate K as (1 - whatever is biggest out of bgrFloat)
kChannel = 1 - np.max(bgrFloat, axis=2)
# Convert back to uint 8:
kChannel = 255 * kChannel
kChannel = kChannel.astype(np.uint8)
Gotta keep en eye on the data types, because there are float operations going on. This is the result:
As you see, the hole is almost 100% white, that's cool, we can threshold this image via Otsu like this:
# Compute binary mask of the hole via Otsu:
_, binaryImage = cv2.threshold(kChannel, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
Which gives you this nice binary mask:
Now, here comes the laborious part. Let's find contours on this image. For every contour/blob compute circularity and area. Use this info to filter noise and get the contour of interest, keep in mind that a perfect circle should have circularity close to 1.0. Once you get a contour of interest, approximate a circle to it. This is the process:
# Find the big contours/blobs on the filtered image:
contours, hierarchy = cv2.findContours(binaryImage, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
# Store the detected circles here:
detectedCircles = []
# Look for the potential contours of interest:
for _, c in enumerate(contours):
# Get the blob's area and perimeter:
contourArea = cv2.contourArea(c)
contourPerimeter = cv2.arcLength(c, True)
# Compute circularity:
if contourPerimeter > 0:
circularity = (4 * 3.1416 * contourArea) / (pow(contourPerimeter, 2))
else:
circularity = 0.0
# Set the min threshold values to identify the
# blob of interest:
minCircularity = 0.7
minArea = 2000
if circularity >= minCircularity and contourArea >= minArea:
# Approximate the contour to a circle:
(x, y), radius = cv2.minEnclosingCircle(c)
# Compute the center and radius:
center = (int(x), int(y))
# Cast radius to in:
radius = int(radius)
# Store the center and radius:
detectedCircles.append([center, radius])
# Draw the circles:
cv2.circle(bgr, center, radius, (0, 255, 0), 2)
cv2.imshow("Detected Circles", bgr)
print("Circles Found: " + str(len(detectedCircles)))
Additionally, I have stored the circle (center and radius) in the detectedCircles list. This is the final result:
Circles Found: 1
Here it is:
import numpy as np
import cv2
def threshold_gray_const(image_, rang: tuple):
return cv2.inRange(image_, rang[0], rang[1])
def binary_or(image_1, image_2):
return cv2.bitwise_or(image_1, image_2)
def negate_image(image_):
return cv2.bitwise_not(image_)
def particle_filter(image_, power):
# Abdrakov's particle filter
nb_components, output, stats, centroids = cv2.connectedComponentsWithStats(image_, connectivity=8)
sizes = stats[1:, -1]
nb_components = nb_components - 1
min_size = power
img2 = np.zeros(output.shape, dtype=np.uint8)
for i in range(0, nb_components):
if sizes[i] >= min_size:
img_to_compare = threshold_gray_const(output, (i + 1, i + 1))
img2 = binary_or(img2, img_to_compare)
img2 = img2.astype(np.uint8)
return img2
def reject_borders(image_):
# Abdrakov's border rejecter
out_image = image_.copy()
h, w = image_.shape[:2]
for row in range(h):
if out_image[row, 0] == 255:
cv2.floodFill(out_image, None, (0, row), 0)
if out_image[row, w - 1] == 255:
cv2.floodFill(out_image, None, (w - 1, row), 0)
for col in range(w):
if out_image[0, col] == 255:
cv2.floodFill(out_image, None, (col, 0), 0)
if out_image[h - 1, col] == 255:
cv2.floodFill(out_image, None, (col, h - 1), 0)
return out_image
src = cv2.imread("your_image")
img_gray = dst = cv2.cvtColor(src, cv2.COLOR_BGR2GRAY)
element = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
closing = cv2.morphologyEx(img_gray, cv2.MORPH_CLOSE, element, iterations=2)
tv, thresh = cv2.threshold(closing, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
neg = negate_image(thresh)
rej = reject_borders(neg)
filtered = particle_filter(rej, 300)
edges = cv2.Canny(filtered, 100, 200)
circles = cv2.HoughCircles(edges, cv2.HOUGH_GRADIENT, 3, 50, param1=50, param2=30, minRadius=20, maxRadius=50)
circles = np.uint16(np.around(circles))
for i in circles[0, :]:
# draw the outer circle
cv2.circle(src, (i[0], i[1]), i[2], (0, 255, 0), 2)
# draw the center of the circle
cv2.circle(src, (i[0], i[1]), 2, (0, 0, 255), 3)
cv2.imshow("closing", closing)
cv2.imshow("edges", edges)
cv2.imshow("out", src)
cv2.waitKey(0)
I changed cv2.morphologyEx parameters a bit, because they were too strong. And after this noise removing I made a binary image using cv2.THRESH_OTSU parameter, negated it, rejected borders and filtered a bit. Then I used cv2.Canny to find edges and this 'cannied' image I passed into cv2.HoughCircles. If any questions - ask me :)
If you want to use a "thinking out of the box" solution then check this solution out. Remember this might have a few false positives in some cases and would only work in cases where circle contour is complete or joined.
import numpy as np
import cv2
import matplotlib.pyplot as plt
from math import pi
pi_eps = 0.1
rgb = cv2.imread('/path/to/your/image/find_circle.jpg')
gray = cv2.cvtColor(rgb, cv2.COLOR_BGR2GRAY)
th = cv2.adaptiveThreshold(gray,255, cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY_INV,21,5)
contours, hier = cv2.findContours(th.copy(),cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
out_img = rgb.copy()
for i in range(len(contours)):
x,y,w,h = cv2.boundingRect(contours[i])
ar = min(w,h)/max(w,h)
# For a circle aspect ratio is close to 1.0
# In your use case circle diameter is between 40px-100px
if ar < 0.9 or \
w < 40 or w > 100:
continue
# P = 2 * PI * r
perimeter = cv2.arcLength(contours[i], True)
if perimeter == 0:
continue
# Second level confirmation could be done using PI = P * P / (4 * A)
# A = PI * r * r
area = cv2.contourArea(contours[i])
if area == 0:
continue
# d = (w+h) / 2 average diameter
# A contour is a circle if (P / d) = PI
ctr_pi = perimeter / ((w+h) / 2)
if abs(ctr_pi - pi) < pi_eps * pi:
cv2.circle(out_img, (int(x+w/2), int(y+h/2)), int(max(w,h)/2), (0, 255, 0), 1)
print("Center of the circle: ", x + w/2, y+h/2)
plt.imshow(out_img)
This is my input sudoku image : 0000.png and this is my desired output image : output.
I am trying to find the contours of the sudoku board so I can extract the board separately. This is the code I've tried so far.
import cv2
import operator
import numpy as np
import matplotlib.pyplot as plt
image = cv2.imread("0000.png")
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (5,5), 0)
thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)
contours,_ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
max_area = 0
c = 0
for i in contours:
area = cv2.contourArea(cv2.UMat(i))
if area > 1000:
if area > max_area:
max_area = area
best_cnt = i
image = cv2.drawContours(image, contours, c, (0, 255, 0), 3)
c+=1
mask = np.zeros((gray.shape),np.uint8)
cv2.drawContours(mask,[best_cnt],0,255,-1)
cv2.drawContours(mask,[best_cnt],0,0,2)
out = np.zeros_like(gray)
out[mask == 255] = gray[mask == 255]
blur = cv2.GaussianBlur(out, (11,11), 0)
thresh = cv2.adaptiveThreshold(blur, 255, 1, 1, 11, 2)
c = 0
for i in contours:
area = cv2.contourArea(i)
if area > 1000/2:
cv2.drawContours(image, contours, c, (0, 255, 0), 3)
c+=1
These are the images I'm able to extract after finding adaptive thresholds and using contours to draw masks.
plt.imshow(out, cmap='gray')
plt.imshow(thresh, cmap='gray')
plt.imshow(image)
How can I modify my code to generate the output image, where the sudoku board is cropped as in output ?
EDIT :
I tried extracting the coordinates of the biggest blog and cropped the largest rectangle, but it's still not accurate as the output image I desire.
contours, h = cv2.findContours(out.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = sorted(contours, key=cv2.contourArea, reverse=True)
polygon = contours[0]
bottom_right, _ = max(enumerate([pt[0][0] + pt[0][1] for pt in polygon]), key=operator.itemgetter(1))
top_left, _ = min(enumerate([pt[0][0] + pt[0][1] for pt in polygon]), key=operator.itemgetter(1))
bottom_left, _ = min(enumerate([pt[0][0] - pt[0][1] for pt in polygon]), key=operator.itemgetter(1))
top_right, _ = max(enumerate([pt[0][0] - pt[0][1] for pt in polygon]), key=operator.itemgetter(1))
box = [polygon[top_left][0], polygon[top_right][0], polygon[bottom_right][0], polygon[bottom_left][0]]
def distance_between(p1, p2):
"""Returns the scalar distance between two points"""
a = p2[0] - p1[0]
b = p2[1] - p1[1]
return np.sqrt((a ** 2) + (b ** 2))
def crop_and_warp(img, crop_rect):
"""Crops and warps a rectangular section from an image into a square of similar size."""
top_left, top_right, bottom_right, bottom_left = crop_rect[0], crop_rect[1], crop_rect[2], crop_rect[3]
src = np.array([top_left, top_right, bottom_right, bottom_left], dtype='float32')
side = max([
distance_between(bottom_right, top_right),
distance_between(top_left, bottom_left),
distance_between(bottom_right, bottom_left),
distance_between(top_left, top_right)
])
dst = np.array([[0, 0], [side - 1, 0], [side - 1, side - 1], [0, side - 1]], dtype='float32')
m = cv2.getPerspectiveTransform(src, dst)
return cv2.warpPerspective(img, m, (int(side), int(side)))
cropped = crop_and_warp(out, box)
plt.imshow(cropped, cmap='gray')
The border is not neatly extracted unlike my desired output image. The reason I want it in this format is because then then the image can be split into 81 images with the help of a 9x9 grid and I could detect all the numbers in each of the image. The black trace above the image is causing troubles there.
Problem
Using this answer to create a segmentation program, it is counting the objects incorrectly. I noticed that alone objects are being ignored or poor imaging acquisition.
I counted 123 objects and the program returns 117, as can be seen, bellow. The objects circled in red seem to be missing:
Using the following image from a 720p webcam:
Code
import cv2
import numpy as np
import matplotlib.pyplot as plt
from scipy.ndimage import label
import urllib.request
# https://stackoverflow.com/a/14617359/7690982
def segment_on_dt(a, img):
border = cv2.dilate(img, None, iterations=5)
border = border - cv2.erode(border, None)
dt = cv2.distanceTransform(img, cv2.DIST_L2, 3)
plt.imshow(dt)
plt.show()
dt = ((dt - dt.min()) / (dt.max() - dt.min()) * 255).astype(np.uint8)
_, dt = cv2.threshold(dt, 140, 255, cv2.THRESH_BINARY)
lbl, ncc = label(dt)
lbl = lbl * (255 / (ncc + 1))
# Completing the markers now.
lbl[border == 255] = 255
lbl = lbl.astype(np.int32)
cv2.watershed(a, lbl)
print("[INFO] {} unique segments found".format(len(np.unique(lbl)) - 1))
lbl[lbl == -1] = 0
lbl = lbl.astype(np.uint8)
return 255 - lbl
# Open Image
resp = urllib.request.urlopen("https://i.stack.imgur.com/YUgob.jpg")
img = np.asarray(bytearray(resp.read()), dtype="uint8")
img = cv2.imdecode(img, cv2.IMREAD_COLOR)
## Yellow slicer
mask = cv2.inRange(img, (0, 0, 0), (55, 255, 255))
imask = mask > 0
slicer = np.zeros_like(img, np.uint8)
slicer[imask] = img[imask]
# Image Binarization
img_gray = cv2.cvtColor(slicer, cv2.COLOR_BGR2GRAY)
_, img_bin = cv2.threshold(img_gray, 140, 255,
cv2.THRESH_BINARY)
# Morphological Gradient
img_bin = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN,
np.ones((3, 3), dtype=int))
# Segmentation
result = segment_on_dt(img, img_bin)
plt.imshow(np.hstack([result, img_gray]), cmap='Set3')
plt.show()
# Final Picture
result[result != 255] = 0
result = cv2.dilate(result, None)
img[result == 255] = (0, 0, 255)
plt.imshow(result)
plt.show()
Question
How to count the missing objects?
Answering your main question, watershed does not remove single objects. Watershed was functioning fine in your algorithm. It receives the predefined labels and perform segmentation accordingly.
The problem was the threshold you set for the distance transform was too high and it removed the weak signal from the single objects, thus preventing the objects from being labeled and sent to the watershed algorithm.
The reason for the weak distance transform signal was due to the improper segmentation during the color segmentation stage and the difficulty of setting a single threshold to remove noise and extract signal.
To remedy this, we need to perform proper color segmentation and use adaptive threshold instead of the single threshold when segmenting the distance transform signal.
Here is the code i modified. I have incorporated color segmentation method by #user1269942 in the code. Extra explanation is in the code.
import cv2
import numpy as np
import matplotlib.pyplot as plt
from scipy.ndimage import label
import urllib.request
# https://stackoverflow.com/a/14617359/7690982
def segment_on_dt(a, img, img_gray):
# Added several elliptical structuring element for better morphology process
struct_big = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(5,5))
struct_small = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3))
# increase border size
border = cv2.dilate(img, struct_big, iterations=5)
border = border - cv2.erode(img, struct_small)
dt = cv2.distanceTransform(img, cv2.DIST_L2, 3)
dt = ((dt - dt.min()) / (dt.max() - dt.min()) * 255).astype(np.uint8)
# blur the signal lighty to remove noise
dt = cv2.GaussianBlur(dt,(7,7),-1)
# Adaptive threshold to extract local maxima of distance trasnform signal
dt = cv2.adaptiveThreshold(dt, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 21, -9)
#_ , dt = cv2.threshold(dt, 2, 255, cv2.THRESH_BINARY)
# Morphology operation to clean the thresholded signal
dt = cv2.erode(dt,struct_small,iterations = 1)
dt = cv2.dilate(dt,struct_big,iterations = 10)
plt.imshow(dt)
plt.show()
# Labeling
lbl, ncc = label(dt)
lbl = lbl * (255 / (ncc + 1))
# Completing the markers now.
lbl[border == 255] = 255
plt.imshow(lbl)
plt.show()
lbl = lbl.astype(np.int32)
cv2.watershed(a, lbl)
print("[INFO] {} unique segments found".format(len(np.unique(lbl)) - 1))
lbl[lbl == -1] = 0
lbl = lbl.astype(np.uint8)
return 255 - lbl
# Open Image
resp = urllib.request.urlopen("https://i.stack.imgur.com/YUgob.jpg")
img = np.asarray(bytearray(resp.read()), dtype="uint8")
img = cv2.imdecode(img, cv2.IMREAD_COLOR)
## Yellow slicer
# blur to remove noise
img = cv2.blur(img, (9,9))
# proper color segmentation
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(hsv, (0, 140, 160), (35, 255, 255))
#mask = cv2.inRange(img, (0, 0, 0), (55, 255, 255))
imask = mask > 0
slicer = np.zeros_like(img, np.uint8)
slicer[imask] = img[imask]
# Image Binarization
img_gray = cv2.cvtColor(slicer, cv2.COLOR_BGR2GRAY)
_, img_bin = cv2.threshold(img_gray, 140, 255,
cv2.THRESH_BINARY)
plt.imshow(img_bin)
plt.show()
# Morphological Gradient
# added
cv2.morphologyEx(img_bin, cv2.MORPH_OPEN,cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3)),img_bin,(-1,-1),10)
cv2.morphologyEx(img_bin, cv2.MORPH_ERODE,cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3)),img_bin,(-1,-1),3)
plt.imshow(img_bin)
plt.show()
# Segmentation
result = segment_on_dt(img, img_bin, img_gray)
plt.imshow(np.hstack([result, img_gray]), cmap='Set3')
plt.show()
# Final Picture
result[result != 255] = 0
result = cv2.dilate(result, None)
img[result == 255] = (0, 0, 255)
plt.imshow(result)
plt.show()
Final results :
124 Unique items found.
An extra item was found because one of the object was divided to 2.
With proper parameter tuning, you might get the exact number you are looking. But i would suggest getting a better camera.
Looking at your code, it is completely reasonable so I'm just going to make one small suggestion and that is to do your "inRange" using HSV color space.
opencv docs on color spaces:
https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_imgproc/py_colorspaces/py_colorspaces.html
another SO example using inRange with HSV:
How to detect two different colors using `cv2.inRange` in Python-OpenCV?
and a small code edits for you:
img = cv2.blur(img, (5,5)) #new addition just before "##yellow slicer"
## Yellow slicer
#mask = cv2.inRange(img, (0, 0, 0), (55, 255, 255)) #your line: comment out.
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) #new addition...convert to hsv
mask = cv2.inRange(hsv, (0, 120, 120), (35, 255, 255)) #new addition use hsv for inRange and an adjustment to the values.
Improving Accuracy
Detecting missing objects
im_1, im_2, im_3
I've count 12 missing objects: 2, 7, 8, 11, 65, 77, 78, 84, 92, 95, 96. edit: 85 too
117 found, 12 missing, 6 wrong
1° Attempt: Decrease Mask Sensibility
#mask = cv2.inRange(img, (0, 0, 0), (55, 255, 255)) #Current
mask = cv2.inRange(img, (0, 0, 0), (80, 255, 255)) #1' Attempt
inRange documentaion
im_4, im_5, im_6, im_7
[INFO] 120 unique segments found
120 found, 9 missing, 6 wrong