I am trying to implement a Python (3.7) OpenCV (3.4.3) ORB image alignment. I normally do most of my processing with ImageMagick. But I need to do some image alignment and am trying to use Python OpenCV ORB. My script is based upon one from Satya Mallick's Learn OpenCV tutorial at https://www.learnopencv.com/image-alignment-feature-based-using-opencv-c-python/.
However, I am trying to modify it to use a rigid alignment rather than a perspective homology and to filter the points using a mask to limit the difference in y values, since the images are nearly aligned already.
The mask approach was taken from a FLANN alignment code in the last example at https://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_feature2d/py_matcher/py_matcher.html.
My script works fine, if I remove the matchesMask, which should provide the point filtering. (I have two other working scripts. One is similar, but just filters the points and ignores the mask. The other is based upon the ECC algorithm.)
However, I would like to understand why my code below is not working.
Perhaps the structure of my mask is incorrect in current versions of Python Opencv?
The error that I get is:
Traceback (most recent call last):
File "warp_orb_rigid2_filter.py", line 92, in <module>
imReg, m = alignImages(im, imReference)
File "warp_orb_rigid2_filter.py", line 62, in alignImages
imMatches = cv2.drawMatches(im1, keypoints1, im2, keypoints2, matches, None, **draw_params)
SystemError: <built-in function drawMatches> returned NULL without setting an error
Here is my code. The first arrow shows where the mask is created. The second arrow shows the line I have to remove to get the script to work. But then it ignores my filtering of points.
#!/bin/python3.7
import cv2
import numpy as np
MAX_FEATURES = 500
GOOD_MATCH_PERCENT = 0.15
def alignImages(im1, im2):
# Convert images to grayscale
im1Gray = cv2.cvtColor(im1, cv2.COLOR_BGR2GRAY)
im2Gray = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY)
# Detect ORB features and compute descriptors.
orb = cv2.ORB_create(MAX_FEATURES)
keypoints1, descriptors1 = orb.detectAndCompute(im1Gray, None)
keypoints2, descriptors2 = orb.detectAndCompute(im2Gray, None)
# Match features.
matcher = cv2.DescriptorMatcher_create(cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING)
matches = matcher.match(descriptors1, descriptors2, None)
# Sort matches by score
matches.sort(key=lambda x: x.distance, reverse=False)
# Remove not so good matches
numGoodMatches = int(len(matches) * GOOD_MATCH_PERCENT)
matches = matches[:numGoodMatches]
# Extract location of good matches and filter by diffy
points1 = np.zeros((len(matches), 2), dtype=np.float32)
points2 = np.zeros((len(matches), 2), dtype=np.float32)
for i, match in enumerate(matches):
points1[i, :] = keypoints1[match.queryIdx].pt
points2[i, :] = keypoints2[match.trainIdx].pt
# initialize empty arrays for newpoints1 and newpoints2 and mask
newpoints1 = np.empty(shape=[0, 2])
newpoints2 = np.empty(shape=[0, 2])
matches_Mask = [0] * len(matches)
# filter points by using mask
for i in range(len(matches)):
pt1 = points1[i]
pt2 = points2[i]
pt1x, pt1y = zip(*[pt1])
pt2x, pt2y = zip(*[pt2])
diffy = np.float32( np.float32(pt2y) - np.float32(pt1y) )
print(diffy)
if abs(diffy) < 10.0:
newpoints1 = np.append(newpoints1, [pt1], axis=0)
newpoints2 = np.append(newpoints2, [pt2], axis=0)
matches_Mask[i]=[1,0] #<--- mask created
print(matches_Mask)
draw_params = dict(matchColor = (255,0,),
singlePointColor = (255,255,0),
matchesMask = matches_Mask, #<---- remove mask here
flags = 0)
# Draw top matches
imMatches = cv2.drawMatches(im1, keypoints1, im2, keypoints2, matches, None, **draw_params)
cv2.imwrite("/Users/fred/desktop/lena_matches.png", imMatches)
# Find Affine Transformation
# true means full affine, false means rigid (SRT)
m = cv2.estimateRigidTransform(newpoints1,newpoints2,False)
# Use affine transform to warp im1 to match im2
height, width, channels = im2.shape
im1Reg = cv2.warpAffine(im1, m, (width, height))
return im1Reg, m
if __name__ == '__main__':
# Read reference image
refFilename = "/Users/fred/desktop/lena.png"
print("Reading reference image : ", refFilename)
imReference = cv2.imread(refFilename, cv2.IMREAD_COLOR)
# Read image to be aligned
imFilename = "/Users/fred/desktop/lena_r1.png"
print("Reading image to align : ", imFilename);
im = cv2.imread(imFilename, cv2.IMREAD_COLOR)
print("Aligning images ...")
# Registered image will be stored in imReg.
# The estimated transform will be stored in m.
imReg, m = alignImages(im, imReference)
# Write aligned image to disk.
outFilename = "/Users/fred/desktop/lena_r1_aligned.jpg"
print("Saving aligned image : ", outFilename);
cv2.imwrite(outFilename, imReg)
# Print estimated homography
print("Estimated Affine Transform : \n", m)
Here are my two images: lena and lena rotated by 1 degree. Note that these are not my actual images. These image have no diffy values > 10, but my actual images do.
I am trying to align and warp the rotated image to match the original lena image.
The way you are creating the mask is incorrect. It only needs to be a list with single numbers, with each number telling you whether you want to use that particular feature match.
Therefore, replace this line:
matches_Mask = [[0,0] for i in range(len(matches))]
With this:
matches_Mask = [0] * len(matches)
... so:
# matches_Mask = [[0,0] for i in range(len(matches))]
matches_Mask = [0] * len(matches)
This creates a list of 0s that is as long as the number of matches. Finally, you need to change writing to the mask with a single value:
if abs(diffy) < 10.0:
#matches_Mask[i]=[1,0] #<--- mask created
matches_Mask[i] = 1
I finally get this:
Estimated Affine Transform :
[[ 1.00001187 0.01598318 -5.05963793]
[-0.01598318 1.00001187 -0.86121051]]
Take note that the format of the mask is different depending on what matcher you use. In this case, you use brute force matching so the mask needs to be in the format that I just described.
If you used FLANN's knnMatch for example, then it will be a nested list of lists, with each element being a list that is k long. For example, if you had k=3 and five keypoints, it will be a list of five elements long, with each element being a three element list. Each element in the sub-list delineates what match you want to use for drawing.
Related
Here is my code, and i am getting an AttributeError: 'tuple' object has no attribute 'sort. I am trying to do an image alignment and found this standard image alignment code in an article. I am learning openCV and python which i am really new too, I am able to do basic stuff with openCV right now i am trying to learn image alignment and i am stuck on this part.
Traceback (most recent call last):
File "test9.py", line 31, in <module>
matches.sort(key = lambda x: x.distance)
AttributeError: 'tuple' object has no attribute 'sort'
------------------
(program exited with code: 1)
Press return to continue
import cv2
import numpy as np
# Open the image files.
img1_color = cv2.imread("/home/pi/Desktop/Project AOI/testboard1/image_2.jpg") # Image to be aligned.
img2_color = cv2.imread("/home/pi/Desktop/Project AOI/testboard1/image_0.jpg") # Reference image.
# Convert to grayscale.
img1 = cv2.cvtColor(img1_color, cv2.COLOR_BGR2GRAY)
img2 = cv2.cvtColor(img2_color, cv2.COLOR_BGR2GRAY)
height, width = img2.shape
# Create ORB detector with 5000 features.
orb_detector = cv2.ORB_create(5000)
# Find keypoints and descriptors.
# The first arg is the image, second arg is the mask
# (which is not required in this case).
kp1, d1 = orb_detector.detectAndCompute(img1, None)
kp2, d2 = orb_detector.detectAndCompute(img2, None)
# Match features between the two images.
# We create a Brute Force matcher with
# Hamming distance as measurement mode.
matcher = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck = True)
# Match the two sets of descriptors.
matches = matcher.match(d1, d2)
# Sort matches on the basis of their Hamming distance.
matches.sort(key = lambda x: x.distance)
# Take the top 90 % matches forward.
matches = matches[:int(len(matches)*0.9)]
no_of_matches = len(matches)
# Define empty matrices of shape no_of_matches * 2.
p1 = np.zeros((no_of_matches, 2))
p2 = np.zeros((no_of_matches, 2))
for i in range(len(matches)):
p1[i, :] = kp1[matches[i].queryIdx].pt
p2[i, :] = kp2[matches[i].trainIdx].pt
# Find the homography matrix.
homography, mask = cv2.findHomography(p1, p2, cv2.RANSAC)
# Use this matrix to transform the
# colored image wrt the reference image.
transformed_img = cv2.warpPerspective(img1_color,
homography, (width, height))
# Save the output.
cv2.imwrite('output.jpg', transformed_img)
You're getting a tuple returned, not a list. You can't just matches.sort(...) that.
OpenCV, since v4.5.4, exhibits this behavior in its Python bindings generation.
You have to use this instead:
matches = sorted(matches, ...)
This creates a new list, which contains the sorted elements of the original tuple.
Related issues:
https://github.com/opencv/opencv/issues/21266
https://github.com/opencv/opencv/issues/21284
Honestly, I am pretty lost. I was finally able to stitch these two pictures together but am unsure on how to update my code to incorporate more than two photos. How would I change my code in order to allow for multiple picture stitchings? Below is what I have so far, and I should mention that the pictures I am using are low quality, so other simpler examples I found either did not work or could not use all of the pictures I needed. If someone could just give me a general direction on how I would begin to alter this code for up to five pictures, I would appreciate it.
import cv2
import numpy as np
import matplotlib.pyplot as plt
import imageio
cv2.ocl.setUseOpenCL(False)
import warnings
warnings.filterwarnings('ignore')
#sift is a feature descriptor that helps locate pixel coordinates i.e. corner detector
feature_extraction_algo = 'sift'
feature_to_match = 'bf'
#train image needs to be the one transformed
train_photo = cv2.imread('Stitching/Images/Log771/Log2.bmp')
#converting from BGR to RGB for Matplotlib
train_photo = cv2.cvtColor(train_photo, cv2.COLOR_BGR2RGB)
train_photo_crop = train_photo[0:10000, 425:750]
#converting to gray scale
train_photo_gray = cv2.cvtColor(train_photo_crop, cv2.COLOR_RGB2GRAY)
#Do the same for the query image
query_photo = cv2.imread('Stitching/Images/Log771/Log3.bmp')
query_photo = cv2.cvtColor(query_photo, cv2.COLOR_BGR2RGB)
query_photo_crop = query_photo[0:10000, 425:750]
query_photo_gray = cv2.cvtColor(query_photo_crop, cv2.COLOR_RGB2GRAY)
#crop both images
#view/plot images
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, constrained_layout=False, figsize=(16,9))
ax1.imshow(query_photo_crop, cmap="gray")
ax1.set_xlabel("Query Image", fontsize=14)
ax2.imshow(train_photo_crop, cmap="gray")
ax2.set_xlabel("Train Image", fontsize=14)
plt.savefig("./"+'.jpg', bbox_inches='tight', dpi=300, optimize=True, format='jpeg')
plt.show()
#sift.detectAndCompute() gets keypoints and descriptors--helps to determine how similar or different keypoints are-- ie. one picture is
#huge and one is small. Keypoints match but are not similar enough, which is where descriptors come in.
#to compare the keypoints in vector format
def select_descriptor_methods(image, method=None):
assert method is not None, "Please define a feature descriptor method. accepted Values are: 'sift, 'surf'"
if method == 'sift':
descriptor = cv2.SIFT_create()
elif method == 'surf':
descriptor = cv2.SURF_create()
elif method == 'brisk':
descriptor = cv2.BRISK_create()
elif method =='orb':
descriptor = cv2.ORB_create()
(keypoints, features) = descriptor.detectAndCompute (image, None)
return (keypoints, features)
keypoints_train_img, features_train_img = select_descriptor_methods(train_photo_gray, method=feature_extraction_algo)
keypoints_query_img, features_query_img = select_descriptor_methods(query_photo_gray, method=feature_extraction_algo)
for keypoint in keypoints_query_img:
x,y = keypoint.pt
size = keypoint.size
orientation = keypoint.angle
response = keypoint.response
octave = keypoint.octave
class_id = keypoint.class_id
print (x,y)
print(size)
print(orientation)
print(response)
print(octave)
print(class_id)
print(len(keypoints_query_img))
features_query_img.shape
#Noting a basic fact that - SIFT descriptor is computed for every key-point detected in the image.
#Before computing descriptor, you probably used a detector (as Harris, Sift or Surf Detector) to detect points of interest. Detecting key-points and computing descriptors are two independent steps!
#drawing keypoints using drawKeypoints(input image,
# keypoints, output image, color, flag) -- keypoints based off input picture
#Displaying keypoints and features on both detected images
fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2, figsize=(20,8), constrained_layout=False)
ax1.imshow(cv2.drawKeypoints(train_photo_gray, keypoints_query_img, None, color=(0,255,0)))
ax1.set_xlabel("(a)", fontsize=14)
ax2.imshow(cv2.drawKeypoints(query_photo_gray, keypoints_query_img,None,color=(0,255,0)))
ax2.set_xlabel("(b)", fontsize=14)
plt.savefig("./Stitching/" + feature_extraction_algo + "Images" + '.jpg', bbox_inches='tight', dpi=300, optimize=True, format='jpg')
plt.show()
def create_matching_object(method,crossCheck):
"Create and return a Matcher Object"
# For BF matcher, first we have to create the BFMatcher object using cv2.BFMatcher().
# It takes two optional params.
# normType - It specifies the distance measurement
# crossCheck - which is false by default. If it is true, Matcher returns only those matches
# with value (i,j) such that i-th descriptor in set A has j-th descriptor in set B as the best match
# and vice-versa.
if method == 'sift' or method == 'surf':
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=crossCheck)
elif method == 'orb' or method == 'brisk':
bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=crossCheck)
return bf
def key_points_matching(features_train_img, features_query_img, method):
bf = create_matching_object(method, crossCheck=True)
# Match descriptors.
best_matches = bf.match(features_train_img,features_query_img)
# Sort the features in order of distance.
# The points with small distance (more similarity) are ordered first in the vector
rawMatches = sorted(best_matches, key = lambda x:x.distance)
print("Raw matches with Brute force):", len(rawMatches))
return rawMatches
def key_points_matching_KNN(features_train_img, features_query_img, ratio, method):
bf = create_matching_object(method, crossCheck=False)
# compute the raw matches and initialize the list of actual matches
rawMatches = bf.knnMatch(features_train_img, features_query_img, k=2)
print("Raw matches (knn):", len(rawMatches))
matches = []
#loop over raw matches
for m,n in rawMatches:
# ensure the distance is within a certain ratio of each
# other (i.e. Lowe's ratio test)
if m.distance < n.distance * ratio:
matches.append(m)
return matches
print("Drawing: {} matched features Lines".format(feature_to_match))
fig = plt.figure(figsize=(20,8))
if feature_to_match == 'bf':
matches = key_points_matching(features_train_img, features_query_img, method=feature_extraction_algo)
mapped_features_image = cv2.drawMatches(train_photo_crop,keypoints_train_img,query_photo_crop,keypoints_query_img,matches[:100],None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
# Now for cross checking draw the feature-mapping lines also with KNN
elif feature_to_match == 'knn':
matches = key_points_matching_KNN(features_train_img, features_query_img, ratio=0.75, method=feature_extraction_algo)
mapped_features_image_knn = cv2.drawMatches(train_photo_crop, keypoints_train_img, query_photo_crop, keypoints_query_img, np.random.choice(matches,50),None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
plt.imshow(mapped_features_image)
plt.axis('off')
plt.savefig("./Stitching/" + feature_to_match + "_matching_img_log_"+'.jpeg', bbox_inches='tight', dpi=300, optimize=True, format='jpeg')
plt.show()
feature_to_match = 'knn'
print("Drawing: {} matched features Lines".format(feature_to_match))
fig = plt.figure(figsize=(20,8))
if feature_to_match == 'bf':
matches = key_points_matching(features_train_img, features_query_img, method=feature_extraction_algo)
mapped_features_image = cv2.drawMatches(train_photo_crop,keypoints_train_img,query_photo_crop,keypoints_query_img,matches[:100],
None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
# Now for cross checking draw the feature-mapping lines also with KNN
elif feature_to_match == 'knn':
matches = key_points_matching_KNN(features_train_img, features_query_img, ratio=0.75, method=feature_extraction_algo)
mapped_features_image_knn = cv2.drawMatches(train_photo_crop, keypoints_train_img, query_photo_crop, keypoints_query_img, np.random.choice(matches,100),None,flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS)
plt.imshow(mapped_features_image_knn)
plt.axis('off')
plt.savefig("./Stitching/" + feature_to_match + "_Images"+'.jpg', bbox_inches='tight', dpi=300, optimize=True, format='jpg')
plt.show()
def homography_stitching(keypoints_train_img, keypoints_query_img, matches, reprojThresh):
keypoints_train_img = np.float32([keypoint.pt for keypoint in keypoints_train_img])
keypoints_query_img = np.float32([keypoint.pt for keypoint in keypoints_query_img])
''' For findHomography() - I need to have an assumption of a minimum of correspondence points that are present between the 2 images. Here, I am assuming that Minimum Match Count to be 4 '''
if len(matches) > 4:
# construct the two sets of points
points_train = np.float32([keypoints_train_img[m.queryIdx] for m in matches])
points_query = np.float32([keypoints_query_img[m.trainIdx] for m in matches])
# Calculate the homography between the sets of points
(H, status) = cv2.findHomography(points_train, points_query, cv2.RANSAC, reprojThresh)
return (matches, H, status)
else:
return None
M = homography_stitching(keypoints_train_img, keypoints_query_img, matches, reprojThresh=4)
if M is None:
print("Error!")
(matches, Homography_Matrix, status) = M
print(Homography_Matrix)
#Finally, we can apply our transformation by calling the cv2.warpPerspective function. The first parameter is our
# original image that we want to warp,
#the second is our transformation matrix M (which will be obtained from homography_stitching),
#and the final parameter is a tuple, used to indicate the width and height of the output image.
# For the calculation of the width and height of the final horizontal panoramic images
# I can just add the widths of the individual images and for the height
# I can take the max from the 2 individual images.
width = query_photo_crop.shape[1] + train_photo_crop.shape[1]
print("width ", width)
# 2922 - Which is exactly the sum value of the width of
# my train.jpg and query.jpg
height = max(query_photo_crop.shape[0], train_photo_crop.shape[0])
# otherwise, apply a perspective warp to stitch the images together
# Now just plug that "Homography_Matrix" into cv::warpedPerspective and I shall have a warped image1 into image2 frame
result = cv2.warpPerspective(train_photo_crop, Homography_Matrix, (width, height))
# The warpPerspective() function returns an image or video whose size is the same as the size of the original image or video. Hence set the pixels as per my query_photo
result[0:query_photo_crop.shape[0], 0:query_photo_crop.shape[1]] = query_photo_crop
plt.figure(figsize=(20,10))
plt.axis('off')
plt.imshow(result)
imageio.imwrite("./Stitching/Images/Log771/finishedLog"+'.jpg', result)
plt.show()
Simple method:
OpenCV makes it easy for you with the help of cv2.Stitcher_create module. Everything is handled internally, right from identifying key feature points to matching them appropriately and finally warping the images. You can pass in more than 2 images to be stitched. But I must warn you the greater the number of images and/or their dimensions; the greater the time needed for computation.
How to use cv2.Stitcher_create module?
First, we need to create an instance of the class.
imageStitcher = cv2.Stitcher_create()
To get a list of all the functions associated with this class just type help(imageStitcher). It will return a list of all the functions along with its required input parameters and expected output.
The created instance contains a function stitch() which is used to create the panorama. stitch can be used in either of the following two ways:
ret, panorama_image = stitch(images):
Pass the list of all the images and the module identifies key features, matches them and yields the warped image
ret, panorama_image = stitch(images, masks):
Optionally, we can also pass a list of masks, with one mask corresponding to every image. A mask is a binary image comprising of black and white. The module looks for keypoints/features only in the white regions of every mask, proceeds to match them and yields the warped image.
Both the above ways also returns a variable ret, if the value is 0 it means stitching was performed without any issues.
The following code sample (which I borrowed) shows the first method:
Sample images to be stitched:
Code:
import os
import cv2
# path containing images to be stitched
path = 'stitching_images'
# append all the images within the path to a list
images = []
for image_file in os.listdir(path):
if image_file.endswith ('.jpg'):
img = cv2.imread(os.path.join(path, image_file))
images.append(img)
# creating instance of stitcher class
imageStitcher = cv2.Stitcher_create()
# call the 'stitch' function and pass in the list of images
status, stitched_img = imageStitcher.stitch(images)
# display the panorama if stitching is successful
if status = 0:
cv2.imshow('Panorama', stitched_img)
Result:
(Code and images borrowed from https://github.com/niconielsen32/ComputerVision/tree/master/imageStitching)
Hard method:
If you want to create a panorama using your code, I would suggest performing it sequentially:
Iterate through all the images in your collection (say A, B, C, D, E)
For each pair of images, find keypoints and match them. (AB, AC, AD, AE, BC, BD, etc..)
Select the pair where keypoint matches are the highest and stitch them using homography (say images A and C are warped to P1)
Next find keypoint matches between the stitched image and every other image in your collection (P1B, P1D, P1E)
Find the pair with most number of matches and stitch them
Repeat likewise
I was using the code given on this site to align a photograph of my certificate to a 'template' i created by converting my pdf certificate to png and deleting my name, candidate id and certification date. Sadly it key point matches the wrong areas (click here for image), specifically matching my name, canditate ID and the date to areas in the certificate when it shouldn't. Oddly enough it works fine when I key point match the photograph with itself (i.e.: without deleting my name, date etc.)
I have tried using different matchers including sift and the brute force matcher but still all result in the same problem. Does anyone know why this may be happening and is there anything I can try to overcome this?
Thanks :)
Here is the code that I am using:
# import the necessary packages
#import numpy as np
import imutils
import cv2
def align_images(image, template, maxFeatures=500, keepPercent=0.2,
debug=True):
# convert both the input image and template to grayscale
imageGray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
templateGray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
# use ORB to detect keypoints and extract (binary) local
# invariant features
orb = cv2.ORB_create(maxFeatures)
(kpsA, descsA) = orb.detectAndCompute(imageGray, None)
(kpsB, descsB) = orb.detectAndCompute(templateGray, None)
# match the features
method = cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING
matcher = cv2.DescriptorMatcher_create(method)
matches = matcher.match(descsA, descsB, None)
# sort the matches by their distance (the smaller the distance,
# the "more similar" the features are)
matches = sorted(matches, key=lambda x:x.distance)
# keep only the top matches
keep = int(len(matches) * keepPercent)
matches = matches[:keep]
# Extract location of good matches
points1 = np.zeros((len(matches), 2), dtype=np.float32)
points2 = np.zeros((len(matches), 2), dtype=np.float32)
matches = cv2.drawMatches(image, kpsA,template,kpsB,matches,None)
matches = imutils.resize(matches, width = 1000)
cv2.imshow("Matched Keypoints",matches)
cv2.waitKey(0)
for i, match in enumerate(matches):
points1[i, :] = kpsA[match.queryIdx].pt
points2[i, :] = kpsB[match.trainIdx].pt
# Find homography
H, mask = cv2.findHomography(points1, points2, cv2.RANSAC)
# use the homography matrix to align the images
(h, w) = template.shape[:2]
aligned = cv2.warpPerspective(image, H, (w, h))
# return the aligned image
return aligned
image = cv2.imread(r"C:\Users\Soffie\Documents\GrayceAssignments\certificate_scanner\certif_image2.jpg")
template = cv2.imread(r"C:\Users\Soffie\Documents\GrayceAssignments\certificate_scanner\certificate_template.png")
aligned = align_images(image, template)
aligned = imutils.resize(aligned,width =700)
template = imutils.resize(template,width = 700)
stacked = np.hstack([aligned,template])
overlay = template.copy()
output = aligned.copy()
cv2.addWeighted(overlay,0.5,output,0.5,0,output)
cv2.imshow("Image Alignment Stacked",stacked)
cv2.imshow("Image Alignment Overlay", output)
cv2.waitKey(0)
Image stitching not work properly. The warped image is cropped and interpolation cannot be done because images do not intersect.
Hi,
I was assigned an homework in which I have to stitch togheter two images, shot by different cameras.
I should find the homography matrix and then warp the second image using this matrix. At the end I must interpolate the two images.
Unfortunately, the code I wrote seems not to work properly. During second image warp I lost most of the image information; a lot of pixels are black and not the whole transformed image is transformed.
I track in the two images four pixels each, in the same order. Below you can find the piece of code I wrote.
# Globals
points = []
def show_and_fetch(image, title):
cv2.namedWindow(title, cv2.WINDOW_NORMAL)
cv2.setMouseCallback(title, mouse_callback)
# Show the image
cv2.imshow(title, image)
# Wait for user input to continue
cv2.waitKey(0)
cv2.destroyAllWindows()
# mouse callback function
def mouse_callback(event,x,y,flags,param):
if event == cv2.EVENT_LBUTTONDOWN:
points.append([x, y])
def stitching():
"""
This procedure stiches two images
:return:
"""
print "Stitching starts..."
###########################################################################
# Get input information
in_file_1 = utils.get_input(
"Insert 0 to exit, the path to the first image to stitch "
"or empty input to use default image: ", "string",
constants.default_stitching1)
in_file_2 = utils.get_input(
"Insert 0 to exit, the path to the second image to stitch "
"or empty input to use default image: ", "string",
constants.default_stitching2)
image_1 = utils.read_image(in_file_1)
image_2 = utils.read_image(in_file_2)
global points
show_and_fetch(image_1, "Image 1 to Stitch")
image_1_points = np.asarray(points, dtype=np.float32)
points = []
show_and_fetch(image_2, "Image 2 to Stitch")
image_2_points = np.asarray(points, dtype=np.float32)
matrix, mask = cv2.findHomography(image_1_points, image_2_points, cv2.RANSAC, 5)
image_1_warped = cv2.warpPerspective(image_1, matrix, dsize=image_1.shape[0:2])
utils.show_image_and_wait(image_1_warped, 'Image 1 warped', wait=False)
utils.show_image_and_wait(image_1, 'Image 1', wait=False)
utils.show_image_and_wait(image_2, 'Image 2')
if __name__ == "__main__":
stitching()
I expect the warped image to be transformed, preserving the most of the information, in terms of pixels. Then interpolation should apply the intersection of the two images that overlap in a certain area.
For instance I want to interpolete these two images:
I've managed to stitch images based on this solution. Here is the stitching result:
Here is the full code:
import cv2
import imutils
import numpy as np
class Stitcher(object):
def __init__(self):
self.isv3 = imutils.is_cv3()
def stitch(self, images, ratio=0.75, reprojThresh=4.0, showMatches=False):
(imageB, imageA) = images
(kpsA, featuresA) = self.detectAndDescribe(imageA)
(kpsB, featuresB) = self.detectAndDescribe(imageB)
# match features between the two images
m = self.matchKeypoints(kpsA, kpsB, featuresA, featuresB, ratio, reprojThresh)
if not m:
return None
# otherwise, apply a perspective warp to stitch the images
# together
(matches, H, status) = m
result = cv2.warpPerspective(imageA, H,
(imageA.shape[1] + imageB.shape[1], imageA.shape[0]))
result[0:imageB.shape[0], 0:imageB.shape[1]] = imageB
# check to see if the keypoint matches should be visualized
if showMatches:
vis = self.drawMatches(imageA, imageB, kpsA, kpsB, matches,
status)
# return a tuple of the stitched image and the
# visualization
return result, vis
# return the stitched image
return result
def detectAndDescribe(self, image):
# convert the image to grayscale
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
# check to see if we are using OpenCV 3.X
if self.isv3:
# detect and extract features from the image
descriptor = cv2.xfeatures2d.SIFT_create()
(kps, features) = descriptor.detectAndCompute(image, None)
# otherwise, we are using OpenCV 2.4.X
else:
# detect keypoints in the image
detector = cv2.xfeatures2d.SIFT_create()
kps = detector.detect(gray)
# extract features from the image
extractor = cv2.xfeatures2d.SIFT_create()
(kps, features) = extractor.compute(gray, kps)
# convert the keypoints from KeyPoint objects to NumPy
# arrays
kps = np.float32([kp.pt for kp in kps])
# return a tuple of keypoints and features
return kps, features
def matchKeypoints(self, kpsA, kpsB, featuresA, featuresB,
ratio, reprojThresh):
# compute the raw matches and initialize the list of actual
# matches
matcher = cv2.DescriptorMatcher_create("BruteForce")
rawMatches = matcher.knnMatch(featuresA, featuresB, 2)
matches = []
# loop over the raw matches
for m in rawMatches:
# ensure the distance is within a certain ratio of each
# other (i.e. Lowe's ratio test)
if len(m) == 2 and m[0].distance < m[1].distance * ratio:
matches.append((m[0].trainIdx, m[0].queryIdx))
# computing a homography requires at least 4 matches
if len(matches) > 4:
# construct the two sets of points
ptsA = np.float32([kpsA[i] for (_, i) in matches])
ptsB = np.float32([kpsB[i] for (i, _) in matches])
# compute the homography between the two sets of points
(H, status) = cv2.findHomography(ptsA, ptsB, cv2.RANSAC,
reprojThresh)
# return the matches along with the homograpy matrix
# and status of each matched point
return (matches, H, status)
# otherwise, no homograpy could be computed
return None
def drawMatches(self, imageA, imageB, kpsA, kpsB, matches, status):
# initialize the output visualization image
(hA, wA) = imageA.shape[:2]
(hB, wB) = imageB.shape[:2]
vis = np.zeros((max(hA, hB), wA + wB, 3), dtype="uint8")
vis[0:hA, 0:wA] = imageA
vis[0:hB, wA:] = imageB
# loop over the matches
for ((trainIdx, queryIdx), s) in zip(matches, status):
# only process the match if the keypoint was successfully
# matched
if s == 1:
# draw the match
ptA = (int(kpsA[queryIdx][0]), int(kpsA[queryIdx][1]))
ptB = (int(kpsB[trainIdx][0]) + wA, int(kpsB[trainIdx][1]))
cv2.line(vis, ptA, ptB, (0, 255, 0), 1)
# return the visualization
return vis
image1 = cv2.imread('image1.jpg')
image2 = cv2.imread('image2.jpg')
stitcher = Stitcher()
(result, vis) = stitcher.stitch([image1, image2], showMatches=True)
cv2.imwrite('result.jpg', result)
I faced with the same problem. It turns out that the order of my images was wrong.
I had two images for stitching. One needs to stitch from left to another. However, I was computing the transform as I want it to stitch from right.
I am analyzing multiple images and need to be able to tell if they are shifted compared to a reference image. The purpose is to tell if the camera moved at all in between capturing images. I would ideally like to be able to correct the shift in order to still do the analysis, but at a minimum I need to be able to determine if an image is shifted and discard it if it's beyond a certain threshold.
Here are some examples of the shifts in an image I would like to detect:
I will use the first image as a reference and then compare all of the following images to it to figure out if they are shifted. The images are gray-scale (they are just displayed in color using a heat-map) and are stored in a 2-D numpy array. Any ideas how I can do this? I would prefer to use the packages I already have installed (scipy, numpy, PIL, matplotlib).
As Lukas Graf hints, you are looking for cross-correlation. It works well, if:
The scale of your images does not change considerably.
There is no rotation change in the images.
There is no significant illumination change in the images.
For plain translations cross-correlation is very good.
The simplest cross-correlation tool is scipy.signal.correlate. However, it uses the trivial method for cross-correlation, which is O(n^4) for a two-dimensional image with side length n. In practice, with your images it'll take very long.
The better too is scipy.signal.fftconvolve as convolution and correlation are closely related.
Something like this:
import numpy as np
import scipy.signal
def cross_image(im1, im2):
# get rid of the color channels by performing a grayscale transform
# the type cast into 'float' is to avoid overflows
im1_gray = np.sum(im1.astype('float'), axis=2)
im2_gray = np.sum(im2.astype('float'), axis=2)
# get rid of the averages, otherwise the results are not good
im1_gray -= np.mean(im1_gray)
im2_gray -= np.mean(im2_gray)
# calculate the correlation image; note the flipping of onw of the images
return scipy.signal.fftconvolve(im1_gray, im2_gray[::-1,::-1], mode='same')
The funny-looking indexing of im2_gray[::-1,::-1] rotates it by 180° (mirrors both horizontally and vertically). This is the difference between convolution and correlation, correlation is a convolution with the second signal mirrored.
Now if we just correlate the first (topmost) image with itself, we get:
This gives a measure of self-similarity of the image. The brightest spot is at (201, 200), which is in the center for the (402, 400) image.
The brightest spot coordinates can be found:
np.unravel_index(np.argmax(corr_img), corr_img.shape)
The linear position of the brightest pixel is returned by argmax, but it has to be converted back into the 2D coordinates with unravel_index.
Next, we try the same by correlating the first image with the second image:
The correlation image looks similar, but the best correlation has moved to (149,200), i.e. 52 pixels upwards in the image. This is the offset between the two images.
This seems to work with these simple images. However, there may be false correlation peaks, as well, and any of the problems outlined in the beginning of this answer may ruin the results.
In any case you should consider using a windowing function. The choice of the function is not that important, as long as something is used. Also, if you have problems with small rotation or scale changes, try correlating several small areas agains the surrounding image. That will give you different displacements at different positions of the image.
Another way to solve it is to compute sift points in both images, use RANSAC to get rid of outliers and then solve for translation using a least squares estimator.
as Bharat said as well another is using sift features and Ransac:
import numpy as np
import cv2
from matplotlib import pyplot as plt
def crop_region(path, c_p):
"""
This function crop the match region in the input image
c_p: corner points
"""
# 3 or 4 channel as the original
img = cv2.imread(path, -1)
# mask
mask = np.zeros(img.shape, dtype=np.uint8)
# fill the the match region
channel_count = img.shape[2]
ignore_mask_color = (255,)*channel_count
cv2.fillPoly(mask, c_p, ignore_mask_color)
# apply the mask
matched_region = cv2.bitwise_and(img, mask)
return matched_region
def features_matching(path_temp,path_train):
"""
Function for Feature Matching + Perspective Transformation
"""
img1 = cv2.imread(path_temp, 0) # template
img2 = cv2.imread(path_train, 0) # input image
min_match=10
# SIFT detector
sift = cv2.xfeatures2d.SIFT_create()
# extract the keypoints and descriptors with SIFT
kps1, des1 = sift.detectAndCompute(img1,None)
kps2, des2 = sift.detectAndCompute(img2,None)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
search_params = dict(checks = 50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)
# store all the good matches (g_matches) as per Lowe's ratio
g_match = []
for m,n in matches:
if m.distance < 0.7 * n.distance:
g_match.append(m)
if len(g_match)>min_match:
src_pts = np.float32([ kps1[m.queryIdx].pt for m in g_match ]).reshape(-1,1,2)
dst_pts = np.float32([ kps2[m.trainIdx].pt for m in g_match ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)
img2 = cv2.polylines(img2, [np.int32(dst)], True, (0,255,255) , 3, cv2.LINE_AA)
else:
print "Not enough matches have been found! - %d/%d" % (len(g_match), min_match)
matchesMask = None
draw_params = dict(matchColor = (0,255,255),
singlePointColor = (0,255,0),
matchesMask = matchesMask, # only inliers
flags = 2)
# region corners
cpoints=np.int32(dst)
a, b,c = cpoints.shape
# reshape to standard format
c_p=cpoints.reshape((b,a,c))
# crop matching region
matching_region = crop_region(path_train, c_p)
img3 = cv2.drawMatches(img1, kps1, img2, kps2, g_match, None, **draw_params)
return (img3,matching_region)