Develop Reference

Perspective Transform OMR Sheet - OpenCV Python

I am trying to make a perspective transformation with OpenCV in python. I want to align the image and find the coordinates of top left top right and bottom left and bottom right contours. here is my code so far where I can identify all the contours.
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray,(3,3),0)
edges = cv2.Canny(blur,50,100)
contours, hierarchy = cv2.findContours(edges,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) #find contours
cv2.drawContours(img,contours,-1,(0,255,0),2)
cv2.imshow('Contours',img)
cv2.waitKey(0)
Here is an image of what I get.
I'd be very thankful if you could try to help me solve this problem!
Original Image
Demo Inputs:
Demo Inputs
Demo Input 1:
Demo Input 1
Demo Input 2:
Demo Input 2
Demo Input 3:
Demo Input 3
Demo Input 4:
Demo Input 4
Desired Output:
Desired Output

Here is one way to do that in Python/OpenCV.
Read the input
Read the template (for its dimension)
Convert the input to gray and threshold
Pad the threshold (to preserve the corners when apply morphology)
Apply morphology close
Remove the padding
Get the largest external contour
Get its perimeter and approximate to 4 corners as the input corners for the warping
Get the output corners for the warping from the dimensions of the template
Get the perspective transformation matrix
Warp the input to match the template
Save the results
Input:
Template:
import cv2
import numpy as np
# read image
img = cv2.imread("omr_test.jpg")
hh, ww = img.shape[:2]
# read template
template = cv2.imread("omr_template.jpg")
ht, wd = template.shape[:2]
# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# do otsu threshold on gray image
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
# pad thresh with black to preserve corners when apply morphology
pad = cv2.copyMakeBorder(thresh, 20, 20, 20, 20, borderType=cv2.BORDER_CONSTANT, value=0)
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15,15))
morph = cv2.morphologyEx(pad, cv2.MORPH_CLOSE, kernel)
# remove padding
morph = morph[20:hh+20, 20:ww+20]
# get largest external contour
contours = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
big_contour = max(contours, key=cv2.contourArea)
# get perimeter and approximate a polygon
peri = cv2.arcLength(big_contour, True)
corners = cv2.approxPolyDP(big_contour, 0.04 * peri, True)
# draw polygon on input image from detected corners
polygon = img.copy()
cv2.polylines(polygon, [corners], True, (0,255,0), 2, cv2.LINE_AA)
# Alternate: cv2.drawContours(page,[corners],0,(0,0,255),1)
# print the number of found corners and the corner coordinates
# They seem to be listed counter-clockwise from the top most corner
print(len(corners))
print(corners)
# reformat input corners to x,y list
icorners = []
for corner in corners:
pt = [ corner[0][0],corner[0][1] ]
icorners.append(pt)
icorners = np.float32(icorners)
# get corresponding output corners form width and height
ocorners = [ [0,0], [0,ht], [wd,ht], [wd,0] ]
ocorners = np.float32(ocorners)
# get perspective tranformation matrix
M = cv2.getPerspectiveTransform(icorners, ocorners)
# do perspective
warped = cv2.warpPerspective(img, M, (wd, ht))
# write results
cv2.imwrite("omr_test_thresh.jpg", thresh)
cv2.imwrite("omr_test_morph.jpg", morph)
cv2.imwrite("omr_test_polygon.jpg", polygon)
cv2.imwrite("omr_test_warped.jpg", warped)
# display it
cv2.imshow("thresh", thresh)
cv2.imshow("pad", pad)
cv2.imshow("morph", morph)
cv2.imshow("polygon", polygon)
cv2.imshow("warped", warped)
cv2.waitKey(0)
Threshold Image:
Morphology Image:
Polygon Image:
Warped Input:

Here is a correction that solves whether the input image is rotated cw or ccw in Python/OpenCV.
Read the input
Read the template (for its dimensions)
Convert the input to gray and threshold
Pad the threshold (to preserve the corners when applying morphology)
Apply morphology close
Remove the padding
Get the largest external contour
Get its perimeter and approximate to 4 corners as the input corners for the warping
Sort the corners by Y and test the first two sorted corners for X. If diff=X2-X1 is negative, then the output corners need to be list in a slightly differently order.
Get the output corners for the warping from the dimensions of the template and the diff of the sorted corners.
Get the perspective transformation matrix
Warp the input to match the template
Save the results
Input 1 (cw rotated):
:
Input 2 (ccw rotated):
import cv2
import numpy as np
# read image
#img = cv2.imread("omr_test.jpg")
img = cv2.imread("omr_test2.jpg")
hh, ww = img.shape[:2]
# read template
template = cv2.imread("omr_template.jpg")
ht, wd = template.shape[:2]
# convert img to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# do otsu threshold on gray image
thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
# pad thresh with black to preserve corners when apply morphology
pad = cv2.copyMakeBorder(thresh, 20, 20, 20, 20, borderType=cv2.BORDER_CONSTANT, value=0)
# apply morphology
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15,15))
morph = cv2.morphologyEx(pad, cv2.MORPH_CLOSE, kernel)
# remove padding
morph = morph[20:hh+20, 20:ww+20]
# get largest external contour
contours = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
big_contour = max(contours, key=cv2.contourArea)
# get perimeter and approximate a polygon
peri = cv2.arcLength(big_contour, True)
corners = cv2.approxPolyDP(big_contour, 0.04 * peri, True)
# draw polygon on input image from detected corners
polygon = img.copy()
cv2.polylines(polygon, [corners], True, (0,255,0), 2, cv2.LINE_AA)
# print the number of found corners and the corner coordinates
# They seem to be listed counter-clockwise from the top most corner
print(len(corners))
print(corners)
# reformat input corners to x,y list
sortcorners = []
for corner in corners:
pt = [ corner[0][0],corner[0][1] ]
sortcorners.append(pt)
icorners = np.float32(sortcorners)
# sort corners on y
def takeSecond(elem):
return elem[1]
sortcorners.sort(key=takeSecond)
# check if second corner x is left or right of first corner x
x1 = sortcorners[0][0]
x2 = sortcorners[1][0]
diff = x2 - x1
print(x1, x2)
# get corresponding output corners from width and height
if diff >= 0:
ocorners = [ [0,0], [0,ht], [wd,ht], [wd,0] ]
else:
ocorners = [ [wd,0], [0,0], [0,ht], [wd,ht]]
ocorners = np.float32(ocorners)
# get perspective tranformation matrix
M = cv2.getPerspectiveTransform(icorners, ocorners)
# do perspective
warped = cv2.warpPerspective(img, M, (wd, ht))
# write results
cv2.imwrite("omr_test2_thresh.jpg", thresh)
cv2.imwrite("omr_test2_morph.jpg", morph)
cv2.imwrite("omr_test2_polygon.jpg", polygon)
cv2.imwrite("omr_test2_warped.jpg", warped)
# display it
cv2.imshow("thresh", thresh)
cv2.imshow("pad", pad)
cv2.imshow("morph", morph)
cv2.imshow("polygon", polygon)
cv2.imshow("warped", warped)
cv2.waitKey(0)
Result for first input (cw rotated):
Result for second input (ccw rotated):

How do I contour multiple largest objects in an image

I'm trying to extract the contours of multiple largest objects within an image. Currently I can only extract the one of the largest objects and the other objects are not contoured. This is the image after threshold, that i'm testing with.
cntrs = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cntrs = cntrs[0] if len(cntrs) == 2 else cntrs[1]
# create black background image
result = np.zeros_like(src)
area_thresh = 0
for c in cntrs:
area = cv2.contourArea(src)
if area > area_thresh:
area_thresh = area
big_contour = c
This is the code I'm currently using that only extracts one object.

Try this:
import cv2
# Read the image
img=cv2.imread('test.jpg')
# Convert to Gray
imgGray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Apply threshold and Dilate (to bring out the lines of the plane)
ImgThresh = cv2.threshold(imgGray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
ImgThreshDilation = cv2.dilate(ImgThresh,(3,3),iterations = 2)
# Find edges
imgEdges = cv2.Canny(ImgThreshDilation,100,200)
# Find contour
contours,hierarchy =cv2.findContours(imgEdges,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
# Loop through contours and find the two biggest area.
for cont in contours:
area=cv2.contourArea(cont)
if area>300:
#print(area)
cv2.drawContours(img,cont,-1,(0,0,255),2)
cv2.imshow('Image with planes in Red',img)
Here is an edit of the above code.
import cv2
# Read the image
img=cv2.imread('test.jpg')
imgCont=img.copy()
# Convert to Gray
imgGray =255- cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Find edges
imgEdges = cv2.Canny(imgGray,150,200)
# Find contour
contours,hierarchy =cv2.findContours(imgEdges,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
# Loop through contours and find the two biggest area.
for cont in contours:
area=cv2.contourArea(cont)
if area>150:
#print(area)
cv2.drawContours(imgCont,cont,-1,(0,0,255),5)
# Save your pictures with the contour in red
cv2.imwrite('Image with planes in Red.jpg',imgCont)
The result:

How to detect text on an X-Ray image with OpenCV

I want to detect text on x-ray images. The goal is to extract the oriented bounding boxes as a matrix where each row is a detected bounding box and each row contains the coordinates of all four edges i.e. [x1, x2, y1, y2]. I'm using python 3 and OpenCV 4.2.0.
Here is a sample image:
The string "test word", "a" and "b" should be detected.
I followed this OpenCV tutorial about creating rotated boxes for contours and this stackoverflow answer about detecting a text area in an image.
The resulting boundary boxes should look something like this:
I was able to detect the text, but the result included a lot of boxes without text.
Here is what I tried so far:
img = cv2.imread(file_name)
## Open the image, convert it into grayscale and blur it to get rid of the noise.
img2gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
ret, mask = cv2.threshold(img2gray, 180, 255, cv2.THRESH_BINARY)
image_final = cv2.bitwise_and(img2gray, img2gray, mask=mask)
ret, new_img = cv2.threshold(image_final, 180, 255, cv2.THRESH_BINARY) # for black text , cv.THRESH_BINARY_INV
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (3, 3))
dilated = cv2.dilate(new_img, kernel, iterations=6)
canny_output = cv2.Canny(dilated, 100, 100 * 2)
cv2.imshow('Canny', canny_output)
## Finds contours and saves them to the vectors contour and hierarchy.
contours, hierarchy = cv2.findContours(canny_output, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# Find the rotated rectangles and ellipses for each contour
minRect = [None] * len(contours)
for i, c in enumerate(contours):
minRect[i] = cv2.minAreaRect(c)
# Draw contours + rotated rects + ellipses
drawing = np.zeros((canny_output.shape[0], canny_output.shape[1], 3), dtype=np.uint8)
for i, c in enumerate(contours):
color = (255, 0, 255)
# contour
cv2.drawContours(drawing, contours, i, color)
# rotated rectangle
box = cv2.boxPoints(minRect[i])
box = np.intp(box) # np.intp: Integer used for indexing (same as C ssize_t; normally either int32 or int64)
cv2.drawContours(img, [box], 0, color)
cv2.imshow('Result', img)
cv2.waitKey()
Do I need to run the results through OCR to make sure whether it is text or not? What other approaches should I try?
PS: I'm quite new to computer vision and not familiar with most concepts yet.

Here's a simple approach:
Obtain binary image. Load image, create blank mask, convert to grayscale, Gaussian blur, then Otsu's threshold
Merge text into a single contour. Since we want to extract the text as one piece, we perform morphological operations to connect individual text contours into a single contour.
Extract text. We find contours then filter using contour area with cv2.contourArea and aspect ratio using cv2.arcLength + cv2.approxPolyDP. If a contour passes the filter, we find the rotated bounding box and draw this onto our mask.
Isolate text. We perform an cv2.bitwise_and operation to extract the text.
Here's a visualization of the process. Using this screenshotted input image (since your provided input image was connected as one image):
Input image -> Binary image
Morph close -> Detected text
Isolated text
Results with the other image
Input image -> Binary image + morph close
Detected text -> Isolated text
Code
import cv2
import numpy as np
# Load image, create mask, grayscale, Gaussian blur, Otsu's threshold
image = cv2.imread('1.png')
original = image.copy()
blank = np.zeros(image.shape[:2], dtype=np.uint8)
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (5,5), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]
# Merge text into a single contour
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
close = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel, iterations=3)
# Find contours
cnts = cv2.findContours(close, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]
for c in cnts:
# Filter using contour area and aspect ratio
x,y,w,h = cv2.boundingRect(c)
area = cv2.contourArea(c)
ar = w / float(h)
if (ar > 1.4 and ar < 4) or ar < .85 and area > 10 and area < 500:
# Find rotated bounding box
rect = cv2.minAreaRect(c)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(image,[box],0,(36,255,12),2)
cv2.drawContours(blank,[box],0,(255,255,255),-1)
# Bitwise operations to isolate text
extract = cv2.bitwise_and(thresh, blank)
extract = cv2.bitwise_and(original, original, mask=extract)
cv2.imshow('thresh', thresh)
cv2.imshow('image', image)
cv2.imshow('close', close)
cv2.imshow('extract', extract)
cv2.waitKey()

I removed the text using the following comand (after the code of above):
gray2 = cv2.cvtColor(extract, cv2.COLOR_BGR2GRAY)
blur2 = cv2.GaussianBlur(gray2, (5,5), 0)
thresh2 = cv2.threshold(blur2, 0, 255, cv2.THRESH_BINARY)[1]
test = cv2.inpaint(original, thresh2, 7, cv2.INPAINT_TELEA)

How to use OpenCV to crop an image based on a certain criteria?

I would like to crop the images like the one below using python's OpenCV library. The area of interest is inside the squiggly lines on the top and bottom, and the lines on the side. The problem is that every image is slightly different. This means that I need some automated way of cropping for the area of interest. I guess the top and the sides would be easy since you could just crop it by 10 pixels or so. But how can I crop out the bottom half of the image where the line is not straight? I have included this example image. The image that follows highlights in pink the area of the image that I am interested in keeping.

Here is one way using Python/OpenCV.
Read input
Get center point (assume it is inside the desired region)
Convert image to grayscale
Floodfill the gray image and set background to black
Get the largest contour and its bounding box
Draw the largest contour as filled on black background as mask
Apply the mask to the input image
Crop the masked input image
Input:
import cv2
import numpy as np
# load image and get dimensions
img = cv2.imread("odd_region.png")
hh, ww, cc = img.shape
# compute center of image (as integer)
wc = ww//2
hc = hh//2
# create grayscale copy of input as basis of mask
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
# create zeros mask 2 pixels larger in each dimension
zeros = np.zeros([hh + 2, ww + 2], np.uint8)
# do floodfill at center of image as seed point
ffimg = cv2.floodFill(gray, zeros, (wc,hc), (255), (0), (0), flags=8)[1]
# set rest of ffimg to black
ffimg[ffimg!=255] = 0
# get contours, find largest and its bounding box
contours = cv2.findContours(ffimg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
contours = contours[0] if len(contours) == 2 else contours[1]
area_thresh = 0
for cntr in contours:
area = cv2.contourArea(cntr)
if area > area_thresh:
area = area_thresh
outer_contour = cntr
x,y,w,h = cv2.boundingRect(outer_contour)
# draw the filled contour on a black image
mask = np.full([hh,ww,cc], (0,0,0), np.uint8)
cv2.drawContours(mask,[outer_contour],0,(255,255,255),thickness=cv2.FILLED)
# mask the input
masked_img = img.copy()
masked_img[mask == 0] = 0
#masked_img[mask != 0] = img[mask != 0]
# crop the bounding box region of the masked img
result = masked_img[y:y+h, x:x+w]
# draw the contour outline on a copy of result
result_outline = result.copy()
cv2.drawContours(result_outline,[outer_contour],0,(0,0,255),thickness=1,offset=(-x,-y))
# display it
cv2.imshow("img", img)
cv2.imshow("ffimg", ffimg)
cv2.imshow("mask", mask)
cv2.imshow("masked_img", masked_img)
cv2.imshow("result", result)
cv2.imshow("result_outline", result_outline)
cv2.waitKey(0)
cv2.destroyAllWindows()
# write result to disk
cv2.imwrite("odd_region_cropped.png", result)
cv2.imwrite("odd_region_cropped_outline.png", result_outline)
Result:
Result With Contour Drawn:

Insert an image onto a substrate

please help me, I need to insert an image on the substrate.
substrate:
It png, and in the area that is blank with cities, you must insert the image from edge to edge of the frame.
The problem is that I can't find an example of how to insert an image to the known coordinate points of the corners of a given substrate.
Pls help))
My test image
import cv2
import numpy as np
from skimage import io
frame = cv2.cvtColor(io.imread('as.png'), cv2.COLOR_RGB2BGR)
image = cv2.cvtColor(io.imread("Vw5Rc.jpg"), cv2.COLOR_RGB2BGR)
mask = 255 * np.uint8(np.all(frame == [0, 0, 0], axis=2))
contours, _ = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
cnt = min(contours, key=cv2.contourArea)
(x, y, w, h) = cv2.boundingRect(cnt)
# Copy appropriately resized image to frame
frame[y:y+h, x:x+w] = cv2.resize(image, (w, h))
cv2.imwrite('frame.png', frame)
I'm trying to find the area where to insert the image by color, the red color of the area I can find, and if there is no color?
The static frame has a constant size.

Here is one way to do it in Python/OpenCV, if I understand what you want.
Read the substrate and trees images
Extract the alpha channel from the substrate
Extract the substrate image without the alpha channel
Use the alpha channel to color the base substrate image white where the alpha channel is black to correct a flaw in the base image
Threshold the alpha channel and invert it
Use morphology to remove the grid lines so that there is only one "outer" contour.
Extract the contour and its bounding box
Resize the trees image to the size of the bounding box.
Use numpy indexing and slicing to multiply the region of the substrate with the resized trees image.
Save the results.
Optionally, display the various images.
Substrate Image:
Trees Image:
import cv2
import numpy as np
# load substrate with alpha channel
substrate = cv2.imread("substrate.png", cv2.IMREAD_UNCHANGED)
hh, ww, cc = substrate.shape
# load colored image
trees = cv2.imread("trees.jpg")
# make img white where alpha is black to merge the alpha channel with the image
alpha = substrate[:,:,3]
img = substrate[:,:,0-2]
img[alpha==0] = 255
img = cv2.merge((img,img,img))
# threshold the img
ret, thresh = cv2.threshold(alpha,0,255,0)
# invert thresh
thresh = 255 - thresh
# make grid lines white in thresh so will get only one contour
kernel = np.ones((9,9), np.uint8)
thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
# find one outer contour
cntrs = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cntrs = cntrs[0] if len(cntrs) == 2 else cntrs[1]
# get bounding box of contour of white rectangle in thresh
for c in cntrs:
x,y,w,h = cv2.boundingRect(c)
#cv2.rectangle(img, (x,y), (x+w,y+h),(0, 0, 255), 2)
# resize trees
trees = cv2.resize(trees,(w,h),0,0)
# generate result
result = img.copy()
result[y:y+h, x:x+w] = img[y:y+h, x:x+w]/255 * trees
# write result to disk
cv2.imwrite("substrate_over_trees.jpg", result)
cv2.imshow("ALPHA", alpha)
cv2.imshow("IMG", img)
cv2.imshow("THRESH", thresh)
cv2.imshow("TREES", trees)
cv2.imshow("RESULT", result)
cv2.waitKey(0)
cv2.destroyAllWindows()
Result:
Note that there is distortion of the trees image, because its aspect ratio does not match the region of the substrate image corresponding to the contour bounding box. This can be changed to maintain the aspect ratio, but then the image will need to be padded to white or some other color to fill the remaining area of the bounding box.