Related
In order to maintain the aspect ratio of my image I'm making use of the following code to create a square block and applying my image over this which is fine but the problem is the leftover background is dark/black is there a way to keep that white/transparent?
my code:
def resize_image(img, size=(28,28)):
h, w = img.shape[:2]
c = img.shape[2] if len(img.shape)>2 else 1
if h == w:
return cv2.resize(img, size, cv2.INTER_AREA)
dif = h if h > w else w
interpolation = cv2.INTER_AREA if dif > (size[0]+size[1])//2 else
cv2.INTER_CUBIC
x_pos = (dif - w)//2
y_pos = (dif - h)//2
if len(img.shape) == 2:
mask = np.zeros((dif, dif), dtype=img.dtype)
mask[y_pos:y_pos+h, x_pos:x_pos+w] = img[:h, :w]
else:
mask = np.zeros((dif, dif, c), dtype=img.dtype)
mask[y_pos:y_pos+h, x_pos:x_pos+w, :] = img[:h, :w, :]
return cv2.resize(mask, size, interpolation)
The background color is the color you initialize it with. (mask = np.zeros...)
To have a white background use np.ones and for a specific color add a line after initialization mask[:, :] = (255, 0, 0).
You need to modify the mask you are using. It is initialized with zeros making the background black.
In the following snippet I have inverted the mask using cv2.bitwise_not() and used the result as background.
def resize_image1(img, size=(28,28)):
h, w = img.shape[:2]
c = img.shape[2] if len(img.shape)>2 else 1
if h == w:
return cv2.resize(img, size, cv2.INTER_AREA)
dif = h if h > w else w
interpolation = cv2.INTER_AREA if dif > (size[0]+size[1])//2 else cv2.INTER_CUBIC
x_pos = (dif - w)//2
y_pos = (dif - h)//2
if len(img.shape) == 2:
mask = np.ones((dif, dif), dtype=img.dtype)
mask = cv2.bitwise_not(mask) # Added mask inversion here
mask[y_pos:y_pos+h, x_pos:x_pos+w] = img[:h, :w]
else:
mask = np.ones((dif, dif, c), dtype=img.dtype)
mask = cv2.bitwise_not(mask) # Added mask inversion here
mask[y_pos:y_pos+h, x_pos:x_pos+w, :] = img[:h, :w, :]
return cv2.resize(mask, size, interpolation)
Sample output of image resized to 150 x 150 dimensions:
I have been trying to develop a program - written with python and using OpenCV -which counts the overall value of coins shown in the stream video in the real time. I should note that I am new to the opencv platform.
So, for now i have this code and it's working very well but with images, my question is how to change the input from image to a real time video stream.
Here is the code:
# import classifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
import math
import numpy as np
import argparse
import glob
import cv2
# construct argument parser and parse arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="path to image")
args = vars(ap.parse_args())
image = cv2.imread(args["image"])
# resize image while retaining aspect ratio
d = 1024 / image.shape[1]
dim = (1024, int(image.shape[0] * d))
image = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
# create a copy of the image to display results
output = image.copy()
# convert image to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# improve contrast accounting for differences in lighting conditions:
# create a CLAHE object to apply contrast limiting adaptive histogram equalization
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
gray = clahe.apply(gray)
def calcHistogram(img):
# create mask
m = np.zeros(img.shape[:2], dtype="uint8")
(w, h) = (int(img.shape[1] / 2), int(img.shape[0] / 2))
cv2.circle(m, (w, h), 60, 255, -1)
# calcHist expects a list of images, color channels, mask, bins, ranges
h = cv2.calcHist([img], [0, 1, 2], m, [8, 8, 8], [0, 256, 0, 256, 0, 256])
# return normalized "flattened" histogram
return cv2.normalize(h, h).flatten()
def calcHistFromFile(file):
img = cv2.imread(file)
return calcHistogram(img)
# define Enum class
class Enum(tuple): __getattr__ = tuple.index
# Enumerate material types for use in classifier
Material = Enum(('Copper', 'Brass', 'Euro1', 'Euro2'))
# locate sample image files
sample_images_copper = glob.glob("sample_images/copper/*")
sample_images_brass = glob.glob("sample_images/brass/*")
sample_images_euro1 = glob.glob("sample_images/euro1/*")
sample_images_euro2 = glob.glob("sample_images/euro2/*")
# define training data and labels
X = []
y = []
# compute and store training data and labels
for i in sample_images_copper:
X.append(calcHistFromFile(i))
y.append(Material.Copper)
for i in sample_images_brass:
X.append(calcHistFromFile(i))
y.append(Material.Brass)
for i in sample_images_euro1:
X.append(calcHistFromFile(i))
y.append(Material.Euro1)
for i in sample_images_euro2:
X.append(calcHistFromFile(i))
y.append(Material.Euro2)
# instantiate classifier
# Multi-layer Perceptron
# score: 0.974137931034
clf = MLPClassifier(solver="lbfgs")
# split samples into training and test data
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=.2)
# train and score classifier
clf.fit(X_train, y_train)
score = int(clf.score(X_test, y_test) * 100)
print("Classifier mean accuracy: ", score)
# blur the image using Gaussian blurring, where pixels closer to the center
# contribute more "weight" to the average, first argument is the source image,
# second argument is kernel size, third one is sigma (0 for autodetect)
# we use a 7x7 kernel and let OpenCV detect sigma
blurred = cv2.GaussianBlur(gray, (7, 7), 0)
# circles: A vector that stores x, y, r for each detected circle.
# src_gray: Input image (grayscale)
# CV_HOUGH_GRADIENT: Defines the detection method.
# dp = 2.2: The inverse ratio of resolution
# min_dist = 100: Minimum distance between detected centers
# param_1 = 200: Upper threshold for the internal Canny edge detector
# param_2 = 100*: Threshold for center detection.
# min_radius = 50: Minimum radius to be detected.
# max_radius = 120: Maximum radius to be detected.
circles = cv2.HoughCircles(blurred, cv2.HOUGH_GRADIENT, dp=2.2, minDist=100,
param1=200, param2=100, minRadius=50, maxRadius=120)
def predictMaterial(roi):
# calculate feature vector for region of interest
hist = calcHistogram(roi)
# predict material type
s = clf.predict([hist])
# return predicted material type
return Material[int(s)]
# todo: refactor
diameter = []
materials = []
coordinates = []
count = 0
if circles is not None:
# append radius to list of diameters (we don't bother to multiply by 2)
for (x, y, r) in circles[0, :]:
diameter.append(r)
# convert coordinates and radii to integers
circles = np.round(circles[0, :]).astype("int")
# loop over coordinates and radii of the circles
for (x, y, d) in circles:
count += 1
# add coordinates to list
coordinates.append((x, y))
# extract region of interest
roi = image[y - d:y + d, x - d:x + d]
# try recognition of material type and add result to list
material = predictMaterial(roi)
materials.append(material)
# write masked coin to file
if False:
m = np.zeros(roi.shape[:2], dtype="uint8")
w = int(roi.shape[1] / 2)
h = int(roi.shape[0] / 2)
cv2.circle(m, (w, h), d, (255), -1)
maskedCoin = cv2.bitwise_and(roi, roi, mask=m)
cv2.imwrite("extracted/01coin{}.png".format(count), maskedCoin)
# draw contour and results in the output image
cv2.circle(output, (x, y), d, (0, 255, 0), 2)
cv2.putText(output, material,
(x - 40, y), cv2.FONT_HERSHEY_PLAIN,
1.5, (0, 255, 0), thickness=2, lineType=cv2.LINE_AA)
# get biggest diameter
biggest = max(diameter)
i = diameter.index(biggest)
# scale everything according to maximum diameter
# todo: this should be chosen by the user
if materials[i] == "Euro2":
diameter = [x / biggest * 25.75 for x in diameter]
scaledTo = "Scaled to 2 Euro"
elif materials[i] == "Brass":
diameter = [x / biggest * 24.25 for x in diameter]
scaledTo = "Scaled to 50 Cent"
elif materials[i] == "Euro1":
diameter = [x / biggest * 23.25 for x in diameter]
scaledTo = "Scaled to 1 Euro"
elif materials[i] == "Copper":
diameter = [x / biggest * 21.25 for x in diameter]
scaledTo = "Scaled to 5 Cent"
else:
scaledTo = "unable to scale.."
i = 0
total = 0
while i < len(diameter):
d = diameter[i]
m = materials[i]
(x, y) = coordinates[i]
t = "Unknown"
# compare to known diameters with some margin for error
if math.isclose(d, 25.75, abs_tol=1.25) and m == "Euro2":
t = "2 Euro"
total += 200
elif math.isclose(d, 23.25, abs_tol=2.5) and m == "Euro1":
t = "1 Euro"
total += 100
elif math.isclose(d, 19.75, abs_tol=1.25) and m == "Brass":
t = "10 Cent"
total += 10
elif math.isclose(d, 22.25, abs_tol=1.0) and m == "Brass":
t = "20 Cent"
total += 20
elif math.isclose(d, 24.25, abs_tol=2.5) and m == "Brass":
t = "50 Cent"
total += 50
elif math.isclose(d, 16.25, abs_tol=1.25) and m == "Copper":
t = "1 Cent"
total += 1
elif math.isclose(d, 18.75, abs_tol=1.25) and m == "Copper":
t = "2 Cent"
total += 2
elif math.isclose(d, 21.25, abs_tol=2.5) and m == "Copper":
t = "5 Cent"
total += 5
# write result on output image
cv2.putText(output, t,
(x - 40, y + 22), cv2.FONT_HERSHEY_PLAIN,
1.5, (255, 255, 255), thickness=2, lineType=cv2.LINE_AA)
i += 1
# resize output image while retaining aspect ratio
d = 768 / output.shape[1]
dim = (768, int(output.shape[0] * d))
image = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
output = cv2.resize(output, dim, interpolation=cv2.INTER_AREA)
# write summary on output image
cv2.putText(output, scaledTo,
(5, output.shape[0] - 40), cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255), lineType=cv2.LINE_AA)
cv2.putText(output, "Coins detected: {}, EUR {:2}".format(count, total / 100),
(5, output.shape[0] - 24), cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255), lineType=cv2.LINE_AA)
cv2.putText(output, "Classifier mean accuracy: {}%".format(score),
(5, output.shape[0] - 8), cv2.FONT_HERSHEY_PLAIN,
1.0, (0, 0, 255), lineType=cv2.LINE_AA)
# show output and wait for key to terminate program
cv2.imshow("Output", np.hstack([image, output]))
cv2.waitKey(0)
Video stream is just a sequence of images. Here is an example of OpenCV:
import numpy as np
import cv2
cap = cv2.VideoCapture(0)
while(True):
# Capture frame-by-frame
ret, frame = cap.read()
# <Put your code for processing 1 image here>
# Display the resulting frame
cv2.imshow('frame',processed_frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
PS. You shouldn't post all your code, it's hard for others to follow. Try to make a minimal, reproducible example.
I've been trying for the last few days to get a sudoku grid from a picture, and I have been struggling on getting the smaller squares of the grid.
I am working on the picture below. I thought processing the image with a canny filter would work fine, but it didn't and I couldn't get every contour of each square. I then put adaptive threshold, otsu, and a classic thresholding to the test, but every time, it just could not seem to capture every small square.
The final goal is to get the cells containing a number, and recognize the numbers with pytorch, so I would really like to have some clean images of the numbers, so the recognition doesn't screw up :)
Would anyone have an idea on how to achieve this?
Thanks a lot in advance! :D
Here's a potential solution:
Obtain binary image. Convert image to grayscale
and adaptive threshold
Filter out all numbers and noise to isolate only boxes. We filter using contour area to remove the numbers since we only want each individual cell
Fix grid lines. Perform morphological closing
with a horizontal and vertical kernel
to repair grid lines.
Sort each cell in top-to-bottom and left-to-right order. We organize each cell into a sequential order using imutils.contours.sort_contours() with the top-to-bottom and left-to-right parameter
Here's the initial binary image (left) and filtered out numbers + repaired grid lines + inverted image (right)
Here's a visualization of the iteration of each cell
The detected numbers in each cell
Code
import cv2
from imutils import contours
import numpy as np
# Load image, grayscale, and adaptive threshold
image = cv2.imread('1.png')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
thresh = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV,57,5)
# Filter out all numbers and noise to isolate only boxes
cnts = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
area = cv2.contourArea(c)
if area < 1000:
cv2.drawContours(thresh, [c], -1, (0,0,0), -1)
# Fix horizontal and vertical lines
vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,5))
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, vertical_kernel, iterations=9)
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,1))
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, horizontal_kernel, iterations=4)
# Sort by top to bottom and each row by left to right
invert = 255 - thresh
cnts = cv2.findContours(invert, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
(cnts, _) = contours.sort_contours(cnts, method="top-to-bottom")
sudoku_rows = []
row = []
for (i, c) in enumerate(cnts, 1):
area = cv2.contourArea(c)
if area < 50000:
row.append(c)
if i % 9 == 0:
(cnts, _) = contours.sort_contours(row, method="left-to-right")
sudoku_rows.append(cnts)
row = []
# Iterate through each box
for row in sudoku_rows:
for c in row:
mask = np.zeros(image.shape, dtype=np.uint8)
cv2.drawContours(mask, [c], -1, (255,255,255), -1)
result = cv2.bitwise_and(image, mask)
result[mask==0] = 255
cv2.imshow('result', result)
cv2.waitKey(175)
cv2.imshow('thresh', thresh)
cv2.imshow('invert', invert)
cv2.waitKey()
Note: The sorting idea was adapted from an old previous answer in Rubrik cube solver color extraction.
Steps:
Image PreProcessing ( closing operation )
Finding Sudoku Square and Creating Mask Image
Finding Vertical lines
Finding Horizontal Lines
Finding Grid Points
Correcting the defects
Extracting the digits from each cell
Code:
# ==========import the necessary packages============
import imutils
import numpy as np
import cv2
from transform import four_point_transform
from PIL import Image
import pytesseract
import math
from skimage.filters import threshold_local
# =============== For Transformation ==============
def order_points(pts):
"""initialzie a list of coordinates that will be ordered
such that the first entry in the list is the top-left,
the second entry is the top-right, the third is the
bottom-right, and the fourth is the bottom-left"""
rect = np.zeros((4, 2), dtype = "float32")
# the top-left point will have the smallest sum, whereas
# the bottom-right point will have the largest sum
s = pts.sum(axis = 1)
rect[0] = pts[np.argmin(s)]
rect[2] = pts[np.argmax(s)]
# now, compute the difference between the points, the
# top-right point will have the smallest difference,
# whereas the bottom-left will have the largest difference
diff = np.diff(pts, axis = 1)
rect[1] = pts[np.argmin(diff)]
rect[3] = pts[np.argmax(diff)]
# return the ordered coordinates
return rect
def four_point_transform(image, pts):
# obtain a consistent order of the points and unpack them
# individually
rect = order_points(pts)
(tl, tr, br, bl) = rect
# compute the width of the new image, which will be the
# maximum distance between bottom-right and bottom-left
# x-coordiates or the top-right and top-left x-coordinates
widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))
widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))
maxWidth = max(int(widthA), int(widthB))
# compute the height of the new image, which will be the
# maximum distance between the top-right and bottom-right
# y-coordinates or the top-left and bottom-left y-coordinates
heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))
heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))
maxHeight = max(int(heightA), int(heightB))
# now that we have the dimensions of the new image, construct
# the set of destination points to obtain a "birds eye view",
# (i.e. top-down view) of the image, again specifying points
# in the top-left, top-right, bottom-right, and bottom-left
# order
dst = np.array([
[0, 0],
[maxWidth - 1, 0],
[maxWidth - 1, maxHeight - 1],
[0, maxHeight - 1]], dtype = "float32")
# compute the perspective transform matrix and then apply it
M = cv2.getPerspectiveTransform(rect, dst)
warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))
# return the warped image
return warped
############## To show image ##############
def show_image(img,title):
cv2.imshow(title, img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def find_largest_feature(inp_img, scan_tl=None, scan_br=None):
"""
Uses the fact the `floodFill` function returns a bounding box of the area it filled to find the biggest
connected pixel structure in the image. Fills this structure in white, reducing the rest to black.
"""
img = inp_img.copy() # Copy the image, leaving the original untouched
height, width = img.shape[:2]
max_area = 0
seed_point = (None, None)
if scan_tl is None:
scan_tl = [0, 0]
if scan_br is None:
scan_br = [width, height]
# Loop through the image
for x in range(scan_tl[0], scan_br[0]):
for y in range(scan_tl[1], scan_br[1]):
# Only operate on light or white squares
if img.item(y, x) == 255 and x < width and y < height: # Note that .item() appears to take input as y, x
area = cv2.floodFill(img, None, (x, y), 64)
if area[0] > max_area: # Gets the maximum bound area which should be the grid
max_area = area[0]
seed_point = (x, y)
# Colour everything grey (compensates for features outside of our middle scanning range
for x in range(width):
for y in range(height):
if img.item(y, x) == 255 and x < width and y < height:
cv2.floodFill(img, None, (x, y), 64)
mask = np.zeros((height + 2, width + 2), np.uint8) # Mask that is 2 pixels bigger than the image
# Highlight the main feature
if all([p is not None for p in seed_point]):
cv2.floodFill(img, mask, seed_point, 255)
for x in range(width):
for y in range(height):
if img.item(y, x) == 64: # Hide anything that isn't the main feature
cv2.floodFill(img, mask, (x, y), 0)
return img
################# Preprocessing of sudoku image ###############
def preprocess(image,case):
ratio = image.shape[0] / 500.0
orig = image.copy()
image = imutils.resize(image, height = 500)
if case == True:
gray = cv2.GaussianBlur(image,(5,5),0)
gray = cv2.cvtColor(gray,cv2.COLOR_BGR2GRAY)
mask = np.zeros((gray.shape),np.uint8)
kernel1 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(11,11))
close = cv2.morphologyEx(gray,cv2.MORPH_CLOSE,kernel1)
div = np.float32(gray)/(close)
res = np.uint8(cv2.normalize(div,div,0,255,cv2.NORM_MINMAX))
res2 = cv2.cvtColor(res,cv2.COLOR_GRAY2BGR)
edged = cv2.Canny(res, 75, 200)
cnts = cv2.findContours(edged.copy(), cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if imutils.is_cv2() else cnts[1]
cnts = sorted(cnts, key = cv2.contourArea, reverse = True)[:5]
# loop over the contours
for c in cnts:
# approximate the contour
rect = cv2.boundingRect(c)
area = cv2.contourArea(c)
cv2.rectangle(edged.copy(), (rect[0],rect[1]), (rect[2]+rect[0],rect[3]+rect[1]), (0,0,0), 2)
peri = cv2.arcLength(c, True)
approx = cv2.approxPolyDP(c, 0.02 * peri, True)
# if our approximated contour has four points, then we
# can assume that we have found our screen
if len(approx) == 4:
screenCnt = approx
#print(screenCnt)
break
# show the contour (outline) of the piece of paper
#print(screenCnt)
cv2.drawContours(image, [screenCnt], -1, (0, 255, 0), 2)
# apply the four point transform to obtain a top-down
# view of the original image
warped = four_point_transform(orig, screenCnt.reshape(4, 2) * ratio)
warped1 = cv2.resize(warped,(610,610))
warp = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
T = threshold_local(warp, 11, offset = 10, method = "gaussian")
warp = (warp > T).astype("uint8") * 255
th3 = cv2.adaptiveThreshold(warp,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
cv2.THRESH_BINARY_INV,11,2)
kernel = np.ones((5,5),np.uint8)
dilation =cv2.GaussianBlur(th3,(5,5),0)
else :
warped = image
warped1 = cv2.resize(warped,(610,610))
warp = cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY)
T = threshold_local(warp, 11, offset = 10, method = "gaussian")
warp = (warp > T).astype("uint8") * 255
th3 = cv2.adaptiveThreshold(warp,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
cv2.THRESH_BINARY_INV,11,2)
#show_image(warped1,"preprocessed")
return th3,warped1,warped
def grids(img,warped2):
#print("im:",img.shape)
img2 = img.copy()
img = np.zeros((500,500,3), np.uint8)
ratio2 = 3
kernel_size = 3
lowThreshold = 30
frame = img
img = cv2.resize(frame,(610,610))
for i in range(10):
cv2.line(img, (0,(img.shape[0]//9)*i),(img.shape[1],(img.shape[0]//9)*i), (255, 255, 255), 3, 1)
cv2.line(warped2, (0,(img.shape[0]//9)*i),(img.shape[1],(img.shape[0]//9)*i), (125, 0, 55), 3, 1)
for j in range(10):
cv2.line(img, ((img.shape[1]//9)*j, 0), ((img.shape[1]//9)*j, img.shape[0]), (255, 255, 255), 3, 1)
cv2.line(warped2, ((img.shape[1]//9)*j, 0), ((img.shape[1]//9)*j, img.shape[0]), (125, 0, 55), 3, 1)
#show_image(warped2,"grids")
return img
############### Finding out the intersection pts to get the grids #########
def grid_points(img,warped2):
img1 = img.copy()
kernelx = cv2.getStructuringElement(cv2.MORPH_RECT,(2,10))
dx = cv2.Sobel(img,cv2.CV_16S,1,0)
dx = cv2.convertScaleAbs(dx)
c=cv2.normalize(dx,dx,0,255,cv2.NORM_MINMAX)
c = cv2.morphologyEx(c,cv2.MORPH_DILATE,kernelx,iterations = 1)
cy = cv2.cvtColor(c,cv2.COLOR_BGR2GRAY)
closex = cv2.morphologyEx(cy,cv2.MORPH_DILATE,kernelx,iterations = 1)
kernely = cv2.getStructuringElement(cv2.MORPH_RECT,(10,2))
dy = cv2.Sobel(img,cv2.CV_16S,0,2)
dy = cv2.convertScaleAbs(dy)
c = cv2.normalize(dy,dy,0,255,cv2.NORM_MINMAX)
c = cv2.morphologyEx(c,cv2.MORPH_DILATE,kernely,iterations = 1)
cy = cv2.cvtColor(c,cv2.COLOR_BGR2GRAY)
closey = cv2.morphologyEx(cy,cv2.MORPH_DILATE,kernelx,iterations = 1)
res = cv2.bitwise_and(closex,closey)
#gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(res,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
kernel = np.ones((6,6),np.uint8)
# Perform morphology
se = np.ones((8,8), dtype='uint8')
image_close = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, se)
image_close = cv2.morphologyEx(image_close, cv2.MORPH_OPEN, kernel)
contour, hier = cv2.findContours (image_close,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(contour, key=cv2.contourArea, reverse=True)[:100]
centroids = []
for cnt in cnts:
mom = cv2.moments(cnt)
(x,y) = int(mom['m10']/mom['m00']), int(mom['m01']/mom['m00'])
cv2.circle(img1,(x,y),4,(0,255,0),-1)
cv2.circle(warped2,(x,y),4,(0,255,0),-1)
centroids.append((x,y))
#show_image(warped2,"grid_points")
Points = np.array(centroids,dtype = np.float32)
c = Points.reshape((100,2))
c2 = c[np.argsort(c[:,1])]
b = np.vstack([c2[i*10:(i+1)*10][np.argsort(c2[i*10:(i+1)*10,0])] for i in range(10)])
bm = b.reshape((10,10,2))
return c2,bm,cnts
############ Recognize digit images to number #############
def image_to_num(c2):
img = 255-c2
text = pytesseract.image_to_string(img, lang="eng",config='--psm 6 --oem 3') #builder=builder)
return list(text)[0]
###### To get the digit at the particular cell #############
def get_digit(c2,bm,warped1,cnts):
num = []
centroidx = np.empty((9, 9))
centroidy = np.empty((9, 9))
global list_images
list_images = []
for i in range(0,9):
for j in range(0,9):
x1,y1 = bm[i][j] # bm[0] row1
x2,y2 = bm[i+1][j+1]
coordx = ((x1+x2)//2)
coordy = ((y1+y2)//2)
centroidx[i][j] = coordx
centroidy[i][j] = coordy
crop = warped1[int(x1):int(x2),int(y1):int(y2)]
crop = imutils.resize(crop, height=69,width=67)
c2 = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
c2 = cv2.adaptiveThreshold(c2,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
cv2.THRESH_BINARY_INV,11,2)
kernel = np.ones((2,2),np.uint8)
#c2 = cv2.morphologyEx(c2, cv2.MORPH_OPEN, kernel)
c2= cv2.copyMakeBorder(c2,5,5,5,5,cv2.BORDER_CONSTANT,value=(0,0,0))
no = 0
shape=c2.shape
w=shape[1]
h=shape[0]
mom = cv2.moments(c2)
(x,y) = int(mom['m10']/mom['m00']), int(mom['m01']/mom['m00'])
c2 = c2[14:70,15:62]
contour, hier = cv2.findContours (c2,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
if cnts is not None:
cnts = sorted(contour, key=cv2.contourArea,reverse=True)[:1]
for cnt in cnts:
x,y,w,h = cv2.boundingRect(cnt)
aspect_ratio = w/h
# print(aspect_ratio)
area = cv2.contourArea(cnt)
#print(area)
if area>120 and cnt.shape[0]>15 and aspect_ratio>0.2 and aspect_ratio<=0.9 :
#print("area:",area)
c2 = find_largest_feature(c2)
#show_image(c2,"box2")
contour, hier = cv2.findContours (c2,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(contour, key=cv2.contourArea,reverse=True)[:1]
for cnt in cnts:
rect = cv2.boundingRect(cnt)
#cv2.rectangle(c2, (rect[0],rect[1]), (rect[2]+rect[0],rect[3]+rect[1]), (255,255,255), 2)
c2 = c2[rect[1]:rect[3]+rect[1],rect[0]:rect[2]+rect[0]]
c2= cv2.copyMakeBorder(c2,5,5,5,5,cv2.BORDER_CONSTANT,value=(0,0,0))
list_images.append(c2)
#show_image(c2,"box")
no = image_to_num(c2)
num.append(no)
centroidx = np.transpose(centroidx)
centroidy = np.transpose(centroidy)
return c2, num, centroidx, centroidy
######## creating matrix and filling numbers exist in the orig image #######
def sudoku_matrix(num):
c = 0
grid = np.empty((9, 9))
for i in range(9):
for j in range(9):
grid[i][j] = int(num[c])
c += 1
grid = np.transpose(grid)
return grid
######## Creating board to show the puzzle result in terminal##############
def board(arr):
for i in range(9):
if i%3==0 :
print("+",end="")
print("-------+"*3)
for j in range(9):
if j%3 ==0 :
print("",end="| ")
print(int(arr[i][j]),end=" ")
print("",end="|")
print()
print("+",end="")
print("-------+"*3)
return arr
def check_col(arr,num,col):
if all([num != arr[i][col] for i in range(9)]):
return True
return False
def check_row(arr,num,row):
if all([num != arr[row][i] for i in range(9)]):
return True
return False
def check_cell(arr,num,row,col):
sectopx = 3 * (row//3)
sectopy = 3 * (col//3)
for i in range(sectopx, sectopx+3):
for j in range(sectopy, sectopy+3):
if arr[i][j] == num:
return True
return False
def empty_loc(arr,l):
for i in range(9):
for j in range(9):
if arr[i][j] == 0:
l[0]=i
l[1]=j
return True
return False
#### Solving sudoku by back tracking############
def sudoku(arr):
l=[0,0]
if not empty_loc(arr,l):
return True
row = l[0]
col = l[1]
for num in range(1,10):
if check_row(arr,num,row) and check_col(arr,num,col) and not check_cell(arr,num,row,col):
arr[row][col] = int(num)
if(sudoku(arr)):
return True
# failure, unmake & try again
arr[row][col] = 0
return False
def overlay(arr,num,img,cx,cy):
no = -1
for i in range(9):
for j in range(9):
no += 1
#cv2.putText(img,str(no), (int(cx[i][j]),int(cy[i][j])),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2)
if num[no] == 0:
cv2.putText(img,str(int(arr[j][i])), (int(cx[i][j]-4),int(cy[i][j])+8),cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 4)
cv2.imshow("Sudoku",img)
cv2.waitKey(0)
case = "False" # If transformation is required set True
image = cv2.imread("QupKb.png")
th3,warped1,warped = preprocess(image,case)
warped2 = warped1.copy()
img = grids(warped,warped2)
c2,bm,cnts = grid_points(img,warped2)
c2,num,cx,cy = get_digit(c2,bm,warped1,cnts)
grid = sudoku_matrix(num)
if(sudoku(grid)):
arr = board(grid)
overlay(arr,num,warped1,cx,cy)
else:
print("There is no solution")
warped:
th3:
warped2:
sudoku result:
All the extracted digits:
########## To view all the extracted digits ###############
_, axs = plt.subplots(1, len(list_images), figsize=(24, 24))
axs = axs.flatten()
for img, ax in zip(list_images, axs):
ax.imshow(cv2.resize(img,(64,64)))
plt.show()
References:
grid
points
get features (of
digits)
Images
samples
If image contains just the tightly fitted sudoku grid, one crude way to achieve it would be to divide image into equal 9X9 grid and then try to extract number in each of that grid.
I'm in a struggle with a project that takes an image of a pretty clear font from say a label for example reads the "text region" and outputs it as a string using OCR tesseract for instance.
Now I've made quite some progress with the thing as I added varios global filters to get to a quite clear result but I'm struggling with finding method of filtering just the text out of there and then you have to think about rotating it to be as horizontal as possible and then after that the easy part should be to crop it.
May I have any leads to how to do that not using traning data and over complicating the system sins I only use a rasdpberry pi to do the computing?
Thanks for helping here's what I've came up with so far:
Original Image(Captured from PiCamera):
Adaptive thresh after shadow removal:
[
Glocad tresh after shadow removal:
Here's the code:
# import the necessary packages
from PIL import Image
import pytesseract
import argparse
import cv2
import os
import picamera
import time
import numpy as np
#preprocess = "tresh"
#Remaining textcorping and rotating:
import math
import json
from collections import defaultdict
from scipy.ndimage.filters import rank_filter
def dilate(ary, N, iterations):
"""Dilate using an NxN '+' sign shape. ary is np.uint8."""
kernel = np.zeros((N,N), dtype=np.uint8)
kernel[(N-1)/2,:] = 1
dilated_image = cv2.dilate(ary / 255, kernel, iterations=iterations)
kernel = np.zeros((N,N), dtype=np.uint8)
kernel[:,(N-1)/2] = 1
dilated_image = cv2.dilate(dilated_image, kernel, iterations=iterations)
return dilated_image
def props_for_contours(contours, ary):
"""Calculate bounding box & the number of set pixels for each contour."""
c_info = []
for c in contours:
x,y,w,h = cv2.boundingRect(c)
c_im = np.zeros(ary.shape)
cv2.drawContours(c_im, [c], 0, 255, -1)
c_info.append({
'x1': x,
'y1': y,
'x2': x + w - 1,
'y2': y + h - 1,
'sum': np.sum(ary * (c_im > 0))/255
})
return c_info
def union_crops(crop1, crop2):
"""Union two (x1, y1, x2, y2) rects."""
x11, y11, x21, y21 = crop1
x12, y12, x22, y22 = crop2
return min(x11, x12), min(y11, y12), max(x21, x22), max(y21, y22)
def intersect_crops(crop1, crop2):
x11, y11, x21, y21 = crop1
x12, y12, x22, y22 = crop2
return max(x11, x12), max(y11, y12), min(x21, x22), min(y21, y22)
def crop_area(crop):
x1, y1, x2, y2 = crop
return max(0, x2 - x1) * max(0, y2 - y1)
def find_border_components(contours, ary):
borders = []
area = ary.shape[0] * ary.shape[1]
for i, c in enumerate(contours):
x,y,w,h = cv2.boundingRect(c)
if w * h > 0.5 * area:
borders.append((i, x, y, x + w - 1, y + h - 1))
return borders
def angle_from_right(deg):
return min(deg % 90, 90 - (deg % 90))
def remove_border(contour, ary):
"""Remove everything outside a border contour."""
# Use a rotated rectangle (should be a good approximation of a border).
# If it's far from a right angle, it's probably two sides of a border and
# we should use the bounding box instead.
c_im = np.zeros(ary.shape)
r = cv2.minAreaRect(contour)
degs = r[2]
if angle_from_right(degs) <= 10.0:
box = cv2.cv.BoxPoints(r)
box = np.int0(box)
cv2.drawContours(c_im, [box], 0, 255, -1)
cv2.drawContours(c_im, [box], 0, 0, 4)
else:
x1, y1, x2, y2 = cv2.boundingRect(contour)
cv2.rectangle(c_im, (x1, y1), (x2, y2), 255, -1)
cv2.rectangle(c_im, (x1, y1), (x2, y2), 0, 4)
return np.minimum(c_im, ary)
def find_components(edges, max_components=16):
"""Dilate the image until there are just a few connected components.
Returns contours for these components."""
# Perform increasingly aggressive dilation until there are just a few
# connected components.
count = 21
dilation = 5
n = 1
while count > 16:
n += 1
dilated_image = dilate(edges, N=3, iterations=n)
contours, hierarchy = cv2.findContours(dilated_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
count = len(contours)
#print dilation
#Image.fromarray(edges).show()
#Image.fromarray(255 * dilated_image).show()
return contours
def find_optimal_components_subset(contours, edges):
"""Find a crop which strikes a good balance of coverage/compactness.
Returns an (x1, y1, x2, y2) tuple.
"""
c_info = props_for_contours(contours, edges)
c_info.sort(key=lambda x: -x['sum'])
total = np.sum(edges) / 255
area = edges.shape[0] * edges.shape[1]
c = c_info[0]
del c_info[0]
this_crop = c['x1'], c['y1'], c['x2'], c['y2']
crop = this_crop
covered_sum = c['sum']
while covered_sum < total:
changed = False
recall = 1.0 * covered_sum / total
prec = 1 - 1.0 * crop_area(crop) / area
f1 = 2 * (prec * recall / (prec + recall))
#print '----'
for i, c in enumerate(c_info):
this_crop = c['x1'], c['y1'], c['x2'], c['y2']
new_crop = union_crops(crop, this_crop)
new_sum = covered_sum + c['sum']
new_recall = 1.0 * new_sum / total
new_prec = 1 - 1.0 * crop_area(new_crop) / area
new_f1 = 2 * new_prec * new_recall / (new_prec + new_recall)
# Add this crop if it improves f1 score,
# _or_ it adds 25% of the remaining pixels for <15% crop expansion.
# ^^^ very ad-hoc! make this smoother
remaining_frac = c['sum'] / (total - covered_sum)
new_area_frac = 1.0 * crop_area(new_crop) / crop_area(crop) - 1
if new_f1 > f1 or (
remaining_frac > 0.25 and new_area_frac < 0.15):
print '%d %s -> %s / %s (%s), %s -> %s / %s (%s), %s -> %s' % (
i, covered_sum, new_sum, total, remaining_frac,
crop_area(crop), crop_area(new_crop), area, new_area_frac,
f1, new_f1)
crop = new_crop
covered_sum = new_sum
del c_info[i]
changed = True
break
if not changed:
break
return crop
def pad_crop(crop, contours, edges, border_contour, pad_px=15):
"""Slightly expand the crop to get full contours.
This will expand to include any contours it currently intersects, but will
not expand past a border.
"""
bx1, by1, bx2, by2 = 0, 0, edges.shape[0], edges.shape[1]
if border_contour is not None and len(border_contour) > 0:
c = props_for_contours([border_contour], edges)[0]
bx1, by1, bx2, by2 = c['x1'] + 5, c['y1'] + 5, c['x2'] - 5, c['y2'] - 5
def crop_in_border(crop):
x1, y1, x2, y2 = crop
x1 = max(x1 - pad_px, bx1)
y1 = max(y1 - pad_px, by1)
x2 = min(x2 + pad_px, bx2)
y2 = min(y2 + pad_px, by2)
return crop
crop = crop_in_border(crop)
c_info = props_for_contours(contours, edges)
changed = False
for c in c_info:
this_crop = c['x1'], c['y1'], c['x2'], c['y2']
this_area = crop_area(this_crop)
int_area = crop_area(intersect_crops(crop, this_crop))
new_crop = crop_in_border(union_crops(crop, this_crop))
if 0 < int_area < this_area and crop != new_crop:
print '%s -> %s' % (str(crop), str(new_crop))
changed = True
crop = new_crop
if changed:
return pad_crop(crop, contours, edges, border_contour, pad_px)
else:
return crop
def downscale_image(im, max_dim=2048):
"""Shrink im until its longest dimension is <= max_dim.
Returns new_image, scale (where scale <= 1).
"""
a, b = im.size
if max(a, b) <= max_dim:
return 1.0, im
scale = 1.0 * max_dim / max(a, b)
new_im = im.resize((int(a * scale), int(b * scale)), Image.ANTIALIAS)
return scale, new_im
def process_image(inputImg):
opnImg = Image.open(inputImg)
scale, im = downscale_image(opnImg)
edges = cv2.Canny(np.asarray(im), 100, 200)
# TODO: dilate image _before_ finding a border. This is crazy sensitive!
contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
borders = find_border_components(contours, edges)
borders.sort(key=lambda (i, x1, y1, x2, y2): (x2 - x1) * (y2 - y1))
border_contour = None
if len(borders):
border_contour = contours[borders[0][0]]
edges = remove_border(border_contour, edges)
edges = 255 * (edges > 0).astype(np.uint8)
# Remove ~1px borders using a rank filter.
maxed_rows = rank_filter(edges, -4, size=(1, 20))
maxed_cols = rank_filter(edges, -4, size=(20, 1))
debordered = np.minimum(np.minimum(edges, maxed_rows), maxed_cols)
edges = debordered
contours = find_components(edges)
if len(contours) == 0:
print '%s -> (no text!)' % path
return
crop = find_optimal_components_subset(contours, edges)
crop = pad_crop(crop, contours, edges, border_contour)
crop = [int(x / scale) for x in crop] # upscale to the original image size.
#draw = ImageDraw.Draw(im)
#c_info = props_for_contours(contours, edges)
#for c in c_info:
# this_crop = c['x1'], c['y1'], c['x2'], c['y2']
# draw.rectangle(this_crop, outline='blue')
#draw.rectangle(crop, outline='red')
#im.save(out_path)
#draw.text((50, 50), path, fill='red')
#orig_im.save(out_path)
#im.show()
text_im = opnImg.crop(crop)
text_im.save('Cropted_and_rotated_image.jpg')
return text_im
'''
text_im.save(out_path)
print '%s -> %s' % (path, out_path)
'''
#Camera capturing stuff:
myCamera = picamera.PiCamera()
myCamera.vflip = True
myCamera.hflip = True
'''
myCamera.start_preview()
time.sleep(6)
myCamera.stop_preview()
'''
myCamera.capture("Captured_Image.png")
#End capturing persidure
imgAddr = '/home/pi/My_examples/Mechanical_display_converter/Example1.jpg'
#imgAddr = "Captured_Image.png"
# construct the argument parse and parse the arguments
#ap = argparse.ArgumentParser()
'''
ap.add_argument("-i", "--image", required=True,
help="path to input image to be OCR'd")
ap.add_argument("-p", "--preprocess", type=str, default="thresh",
help="type of preprocessing to be done")
args = vars(ap.parse_args())
'''
# load the example image and convert it to grayscale
img = cv2.imread(imgAddr)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
cv2.imshow('Step1_gray_filter', gray)
'''
# check to see if we should apply thresholding to preprocess the
# image
if args["preprocess"] == "thresh":
gray = cv2.threshold(gray, 0, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
# make a check to see if median blurring should be done to remove
# noise
elif args["preprocess"] == "blur":
gray = cv2.medianBlur(gray, 3)
if preprocess == "thresh":
gray = cv2.threshold(gray, 150, 255,
cv2.THRESH_BINARY | cv2.THRESH_OTSU)[1]
# make a check to see if median blurring should be done to remove
# noise
elif preprocess == "blur":
gray = cv2.medianBlur(gray, 3)
'''
rgb_planes = cv2.split(img)
result_planes = []
result_norm_planes = []
for plane in rgb_planes:
dilated_img = cv2.dilate(plane, np.ones((7,7), np.uint8))
bg_img = cv2.medianBlur(dilated_img, 21)
diff_img = 255 - cv2.absdiff(plane, bg_img)
norm_img = cv2.normalize(diff_img, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_8UC1)
result_planes.append(diff_img)
result_norm_planes.append(norm_img)
result = cv2.merge(result_planes)
result_norm = cv2.merge(result_norm_planes)
cv2.imshow('shadows_out.png', result)
cv2.imshow('shadows_out_norm.png', result_norm)
grayUnShadowedImg = cv2.cvtColor(result, cv2.COLOR_BGR2GRAY)
cv2.imshow('Shadow_Gray_CVT', grayUnShadowedImg)
ret, threshUnShadowedImg = cv2.threshold(grayUnShadowedImg, 200, 255, cv2.THRESH_BINARY)
cv2.imshow('unShadowed_Thresh_filtering', threshUnShadowedImg)
#v2.imwrite('unShadowed_Thresh_filtering.jpg', threshUnShadowedImg)
#croptedunShadowedImg = process_image('unShadowed_Thresh_filtering.jpg')
adptThreshUnShadowedImg = cv2.adaptiveThreshold(grayUnShadowedImg, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 115, 1)
cv2.imshow('unShadowed_Adaptive_Thresh_filtering', adptThreshUnShadowedImg)
'''
blurFImg = cv2.GaussianBlur(adptThreshUnShadowedImg,(25,25), 0)
ret, f3Img = cv2.threshold(blurFImg,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)
cv2.imshow('f3Img', f3Img )
'''
#OCR Stage:
'''
# write the grayscale image to disk as a temporary file so we can
# apply OCR to it
filename = "{}.png".format(os.getpid())
cv2.imwrite(filename, threshImg)
# load the image as a PIL/Pillow image, apply OCR, and then delete
# the temporary file
text = pytesseract.image_to_string(Image.open(filename))
os.remove(filename)
print("\n" + text)
'''
cv2.waitKey(0)
cv2.destroyAllWindows()
Tryed this source out as well but this doesn't seem to work and is not that clear to understand:
https://www.danvk.org/2015/01/07/finding-blocks-of-text-in-an-image-using-python-opencv-and-numpy.html
I have made an example to maybe give you an idea on how to proceede. I made it without your transformations of the image but you could do it with them if you would like.
What I did was to first transform the image to binary with cv2.THRESH_BINARY. Next I made a mask and drew the contours by limiting them with size (cv2.contourArea()) and ratio (got it from cv2.boundingRect()) for threshold. Then I conected all the contours that are near each other using cv2.morphologyEx() and a big kernel size (50x50).
Then I selected the biggest contour (text) and drew a rotated rectangle with cv2.minAreaRect() which got me the rotational angle.
Then I could rotate the image using cv2.getRotationMatrix2D() and cv2.warpAffine() and get a slightly bigger bounding box using the highest X, Y and lowest X,Y values of the rotated rectangle which I used to crop the image.
Then I serched again for contours and removed the noise (little contours) from the image and the result is a text with high contrast.
Final result:
This code is meant only to give an idea or another point of view to the problem and it may not work with other images (if they differ from the original too much) or at least you would have to adjust some parameters of code. Hope it helps. Cheers!
Code:
import cv2
import numpy as np
# Read image and search for contours.
img = cv2.imread('rotatec.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
_, threshold = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
contours, hierarchy = cv2.findContours(threshold,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
# Create first mask used for rotation.
mask = np.ones(img.shape, np.uint8)*255
# Draw contours on the mask with size and ratio of borders for threshold.
for cnt in contours:
size = cv2.contourArea(cnt)
x,y,w,h = cv2.boundingRect(cnt)
if 10000 > size > 500 and w*2.5 > h:
cv2.drawContours(mask, [cnt], -1, (0,0,0), -1)
# Connect neighbour contours and select the biggest one (text).
kernel = np.ones((50,50),np.uint8)
opening = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
gray_op = cv2.cvtColor(opening, cv2.COLOR_BGR2GRAY)
_, threshold_op = cv2.threshold(gray_op, 150, 255, cv2.THRESH_BINARY_INV)
contours_op, hierarchy_op = cv2.findContours(threshold_op, cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
cnt = max(contours_op, key=cv2.contourArea)
# Create rotated rectangle to get the angle of rotation and the 4 points of the rectangle.
_, _, angle = rect = cv2.minAreaRect(cnt)
(h,w) = img.shape[:2]
(center) = (w//2,h//2)
# Rotate the image.
M = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(img, M, (int(w),int(h)), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_CONSTANT)
# Create bounding box for rotated text (use old points of rotated rectangle).
box = cv2.boxPoints(rect)
a, b, c, d = box = np.int0(box)
bound =[]
bound.append(a)
bound.append(b)
bound.append(c)
bound.append(d)
bound = np.array(bound)
(x1, y1) = (bound[:,0].min(), bound[:,1].min())
(x2, y2) = (bound[:,0].max(), bound[:,1].max())
cv2.drawContours(img,[box],0,(0,0,255),2)
# Crop the image and create new mask for the final image.
rotated = rotated[y1:y2, x1:x2]
mask_final = np.ones(rotated.shape, np.uint8)*255
# Remove noise from the final image.
gray_r = cv2.cvtColor(rotated, cv2.COLOR_BGR2GRAY)
_, threshold_r = cv2.threshold(gray_r, 150, 255, cv2.THRESH_BINARY_INV)
contours, hierarchy = cv2.findContours(threshold_r,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
for cnt in contours:
size = cv2.contourArea(cnt)
if size < 500:
cv2.drawContours(threshold_r, [cnt], -1, (0,0,0), -1)
# Invert black and white.
final_image = cv2.bitwise_not(threshold_r)
# Display results.
cv2.imshow('final', final_image)
cv2.imshow('rotated', rotated)
EDIT:
For text recognition I recomend you see this post from SO Simple Digit Recognition OCR in OpenCV-Python.
The result with the code from mentioned post:
EDIT:
This is my code implemented with the slightly modified code from the mentioned post. All steps are written in the comments. You should save the script and the training image to the same directory. This is my training image:
Code:
import cv2
import numpy as np
# Read image and search for contours.
img = cv2.imread('rotatec.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
_, threshold = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
contours, hierarchy = cv2.findContours(threshold,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
# Create first mask used for rotation.
mask = np.ones(img.shape, np.uint8)*255
# Draw contours on the mask with size and ratio of borders for threshold.
for cnt in contours:
size = cv2.contourArea(cnt)
x,y,w,h = cv2.boundingRect(cnt)
if 10000 > size > 500 and w*2.5 > h:
cv2.drawContours(mask, [cnt], -1, (0,0,0), -1)
# Connect neighbour contours and select the biggest one (text).
kernel = np.ones((50,50),np.uint8)
opening = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
gray_op = cv2.cvtColor(opening, cv2.COLOR_BGR2GRAY)
_, threshold_op = cv2.threshold(gray_op, 150, 255, cv2.THRESH_BINARY_INV)
contours_op, hierarchy_op = cv2.findContours(threshold_op, cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
cnt = max(contours_op, key=cv2.contourArea)
# Create rotated rectangle to get the angle of rotation and the 4 points of the rectangle.
_, _, angle = rect = cv2.minAreaRect(cnt)
(h,w) = img.shape[:2]
(center) = (w//2,h//2)
# Rotate the image.
M = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(img, M, (int(w),int(h)), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_CONSTANT)
# Create bounding box for rotated text (use old points of rotated rectangle).
box = cv2.boxPoints(rect)
a, b, c, d = box = np.int0(box)
bound =[]
bound.append(a)
bound.append(b)
bound.append(c)
bound.append(d)
bound = np.array(bound)
(x1, y1) = (bound[:,0].min(), bound[:,1].min())
(x2, y2) = (bound[:,0].max(), bound[:,1].max())
cv2.drawContours(img,[box],0,(0,0,255),2)
# Crop the image and create new mask for the final image.
rotated = rotated[y1:y2, x1-10:x2]
mask_final = np.ones(rotated.shape, np.uint8)*255
# Remove noise from the final image.
gray_r = cv2.cvtColor(rotated, cv2.COLOR_BGR2GRAY)
_, threshold_r = cv2.threshold(gray_r, 150, 255, cv2.THRESH_BINARY_INV)
contours, hierarchy = cv2.findContours(threshold_r,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
for cnt in contours:
size = cv2.contourArea(cnt)
if size < 500:
cv2.drawContours(threshold_r, [cnt], -1, (0,0,0), -1)
# Invert black and white.
final_image = cv2.bitwise_not(threshold_r)
# Display results.
cv2.imwrite('rotated12.png', final_image)
# Import module for finding path to database.
from pathlib import Path
# This code executes once amd writes two files.
# If file exists it skips this step, else it runs again.
file = Path("generalresponses.data")
if file.is_file() == False:
# Reading the training image
im = cv2.imread('pitrain1.png')
im3 = im.copy()
gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray,(5,5),0)
thresh = cv2.adaptiveThreshold(blur,255,1,1,11,2)
# Finding contour
_,contours,hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
# Creates array and list for appending data
samples = np.empty((0,100))
responses = []
# Value serving to increment the "automatic" learning
i = 0
# Iterating through contours and appending the array and list with "learned" values
for cnt in contours:
i+=1
[x,y,w,h] = cv2.boundingRect(cnt)
cv2.rectangle(im,(x,y),(x+w,y+h),(0,0,255),2)
roi = thresh[y:y+h,x:x+w] # Croping ROI to bounding rectangle
roismall = cv2.resize(roi,(10,10)) # Resizing ROI to smaller image
cv2.imshow('norm',im)
# Appending values based on the pitrain1.png image
if i < 36:
responses.append(int(45))
elif 35 < i < 80:
responses.append(int(48))
elif 79 < i < 125:
responses.append(int(57))
elif 124 < i < 160:
responses.append(int(56))
elif 159 < i < 205:
responses.append(int(55))
elif 204 < i < 250:
responses.append(int(54))
elif 249 < i < 295:
responses.append(int(53))
elif 294 < i < 340:
responses.append(int(52))
elif 339 < i < 385:
responses.append(int(51))
elif 384 < i < 430:
responses.append(int(50))
elif 429 < i < 485:
responses.append(int(49))
else:
break
sample = roismall.reshape((1,100))
samples = np.append(samples,sample,0)
# Reshaping and saving database
responses = np.array(responses)
responses = responses.reshape((responses.size,1))
print('end')
np.savetxt('generalsamples.data',samples)
np.savetxt('generalresponses.data',responses, fmt='%s')
################### Recognition ########################
# Dictionary for numbers and characters (in this sample code the only
# character is " - ")
number = {
48 : "0",
53 : "5",
52 : "4",
50 : "2",
45 : "-",
55 : "7",
51 : "3",
57 : "9",
56 : "8",
54 : "6",
49 : "1"
}
####### training part ###############
samples = np.loadtxt('generalsamples.data',np.float32)
responses = np.loadtxt('generalresponses.data',np.float32)
responses = responses.reshape((responses.size,1))
model = cv2.ml.KNearest_create()
model.train(samples,cv2.ml.ROW_SAMPLE,responses)
############################# testing part #########################
im = cv2.imread('rotated12.png')
out = np.zeros(im.shape,np.uint8)
gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
thresh = cv2.adaptiveThreshold(gray,255,1,1,11,2)
contours,hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
for cnt in contours:
[x,y,w,h] = cv2.boundingRect(cnt)
cv2.rectangle(im,(x,y),(x+w,y+h),(0,255,0),2)
roi = thresh[y:y+h,x:x+w]
roismall = cv2.resize(roi,(10,10))
roismall = roismall.reshape((1,100))
roismall = np.float32(roismall)
retval, results, neigh_resp, dists = model.findNearest(roismall,k=5)
string = int((results[0][0]))
string2 = number.get(string)
print(string2)
cv2.putText(out,str(string2),(x,y+h),0,1,(0,255,0))
cv2.imshow('im',im)
cv2.imshow('out',out)
cv2.waitKey(0)
cv2.destroyAllWindows()
Result:
Sorry for begin a complete moron in it,
I'm realy trying to learn as much as I can about coding,everything that goes around the computer and openCV with the very little time I have But here's the edited code I've managed to get partly working:
from PIL import Image
import pytesseract
import os
import picamera
import time
import cv2
import numpy as np
# Read image and search for contours.
img = cv2.imread('Example1.jpg')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
_, threshold = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
contours, hierarchy = cv2.findContours(threshold,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE) #EDITED
# Create first mask used for rotation.
mask = np.ones(img.shape, np.uint8)*255
# Draw contours on the mask with size and ratio of borders for threshold.
for cnt in contours:
size = cv2.contourArea(cnt)
x,y,w,h = cv2.boundingRect(cnt)
if 10000 > size > 500 and w*2.5 > h:
cv2.drawContours(mask, [cnt], -1, (0,0,0), -1)
# Connect neighbour contours and select the biggest one (text).
kernel = np.ones((50,50),np.uint8)
opening = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
gray_op = cv2.cvtColor(opening, cv2.COLOR_BGR2GRAY)
_, threshold_op = cv2.threshold(gray_op, 150, 255, cv2.THRESH_BINARY_INV)
contours_op, hierarchy_op = cv2.findContours(threshold_op, cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
cnt = max(contours_op, key=cv2.contourArea)
# Create rotated rectangle to get the angle of rotation and the 4 points of the rectangle.
_, _, angle = rect = cv2.minAreaRect(cnt)
(h,w) = img.shape[:2]
(center) = (w//2,h//2)
# Rotate the image.
M = cv2.getRotationMatrix2D(center, angle, 1.0)
rotated = cv2.warpAffine(img, M, (int(w),int(h)), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_CONSTANT)
# Create bounding box for rotated text (use old points of rotated rectangle).
box = cv2.cv.BoxPoints(rect) #edited
a, b, c, d = box = np.int0(box)
bound =[]
bound.append(a)
bound.append(b)
bound.append(c)
bound.append(d)
bound = np.array(bound)
(x1, y1) = (bound[:,0].min(), bound[:,1].min())
(x2, y2) = (bound[:,0].max(), bound[:,1].max())
cv2.drawContours(img,[box],0,(0,0,255),2)
# Crop the image and create new mask for the final image.
rotated = rotated[y1:y2, x1:x2]
mask_final = np.ones(rotated.shape, np.uint8)*255
# Remove noise from the final image.
gray_r = cv2.cvtColor(rotated, cv2.COLOR_BGR2GRAY)
_, threshold_r = cv2.threshold(gray_r, 150, 255, cv2.THRESH_BINARY_INV)
contours, hierarchy = cv2.findContours(threshold_r,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE)
for cnt in contours:
size = cv2.contourArea(cnt)
if size < 500:
cv2.drawContours(threshold_r, [cnt], -1, (0,0,0), -1)
# Invert black and white.
final_image = cv2.bitwise_not(threshold_r)
# Display results.
cv2.imshow('final', final_image)
cv2.imshow('rotated', rotated)
#OCR Stage:
# write the grayscale image to disk as a temporary file so we can
# apply OCR to it
filename = "{}.png".format(os.getpid())
cv2.imwrite('Final_proc.jpg', final_image)
# load the image as a PIL/Pillow image, apply OCR, and then delete
# the temporary file
text = pytesseract.image_to_string(Image.open('Final_proc.jpg'))
os.remove('Final_proc.jpg')
print("\n" + text)
cv2.waitKey(0)
cv2.destroyAllWindows()
When compiling it now it gives me this output:
[img]https://i.imgur.com/ImdKSCv.jpg[/img]
which is a little different from what you showed and compiled on the windows machine but still super close.
anyidea what happened? just after that this should be realy easy to dissect the code and learn it easily.
Again thank you very much for your time! :D
So for the python 3 and openCV 3 version of the code in order to make the img work with tesseract you'd need to add an around 20px white boarder to extend the image for somereason (I assume it's because the convolutional matrix scanning effort) according to my other post:
pytesseract struggling to recognize clean black and white pictures with font numbers and 7 seg digits(python)
and here's how you'd add the boarder:
how to add border around an image in opencv python
In one line of code:
outputImage = cv2.copyMakeBorder(
inputImage,
topBorderWidth,
bottomBorderWidth,
leftBorderWidth,
rightBorderWidth,
cv2.BORDER_CONSTANT,
value=color of border
)
I'm trying to extract a chessboard from an image. It has lot of other unwanted content which I want to remove. So I created a mask which will have all the slopes. Then bitwise_and it with the original grayscale image. I'm a newbie and this I'm finding OpenCV to be very interesting but I'm stuck with this problem. Please help!
import cv2
import numpy as np
from PIL import Image
kernel = np.ones((5,5),np.uint8)
img = cv2.imread('test1.jpg',0)
img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel)
ret,thresh1 = cv2.threshold(img,60,255,cv2.THRESH_BINARY)
edges = cv2.Canny(thresh1,100,200)
cv2.imwrite("canny.jpg", edges)
minL = 10000
maxL = 8
lines = cv2.HoughLinesP(edges,1,np.pi/180, 85)#, minL, maxL)
mask_l = np.zeros(img.shape[:2])
mask_r = np.zeros(img.shape[:2])
mask_t = np.zeros(img.shape[:2])
mask_b = np.zeros(img.shape[:2])
width, height = lines.shape[:2]
im_x, im_y = img.shape
for x1,y1,x2,y2 in lines[0]:
cv2.line(edges,(x1,y1),(x2,y2),(255,255,255),2)
if (x2-x1) == 0:
continue
else:
m = (y2 - y1)/(x2 - x1)
if(m > 0):
for im_count in range(im_x):
yp = round(m * (im_count - x1)) + y1
if yp >= 1 and yp <= im_y:
for temp1 in range(int(im_count),int(im_x)):
mask_r[int(yp)][int(temp1)] = 1
for temp2 in range(int(im_count)):
mask_l[int(yp)][int(temp2)] = 1
else:
for im_count in range(im_x):
yp = round(m * (im_count - x1)) + y1
if yp >= 1 and yp <= im_y:
for temp1 in range(int(yp), int(im_y)):
mask_b[int(im_count)][int(temp1)] = 1
for temp2 in range(int(yp)):
mask_t[int(im_count)][int(temp2)] = 1
cv2.imwrite('new.jpg', edges)
temp_mask1 = cv2.bitwise_and(mask_l, mask_r, mask=None)
temp_mask2 = cv2.bitwise_and(mask_t, mask_b, mask=None)
final_mask = cv2.bitwise_and(temp_mask1, temp_mask2)
final_mask = cv2.morphologyEx((final_mask * 1.0).astype(np.float32), cv2.MORPH_CLOSE, kernel=None)
cv2.imshow('final',final_mask)
cv2.waitKey(0)
cv2.destroyAllWindows()
x,y = img.shape
print x
print y
ret, orig_mask = cv2.threshold(final_mask, 10, 255, cv2.THRESH_BINARY)
a,b = final_mask.shape
print a
print b
imeg = cv2.imread('test1.jpg',cv2.CV_LOAD_IMAGE_GRAYSCALE)
ret, orig_mask1 = cv2.threshold(imeg, 10, 255, cv2.THRESH_BINARY)
(thresh, im_bw) = cv2.threshold(imeg, 128, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
images1 = cv2.bitwise_and(img,img, mask=orig_mask)
cv2.imwrite("123.jpg",images1)
This is the error I am getting:
OpenCV Error: Assertion failed ((mask.type() == CV_8UC1 || mask.type() == CV_8SC1)) in binary_op, file /build/buildd/opencv-2.3.1/modules/core/src/arithm.cpp, line 1033
The dimensions of both mask and image are same. But still I get this error!
mask needs to be formed of signed or unsigned 8-bit integers. That's what CV_8SC1 and CV_8C1 mean.
Try creating mask as np.zeros(shape, dtype=np.uint8) or np.int8 for signed version.
Also check that the calculations assigned to the mask fall into the proper 8-bit range: 0-255.