I am trying to create my own face filtering augmented reality program in open-cv. The idea is it will map a beaver to the users face. Currently, I am unable to get proper transperency when loading this image with 'cv2.imread(...)'. It looks black in the background, and often shows partially in certain areas that are white. When I open this image in photoshop, I am fully capable of moving this image on top of a background with expected transparency results. I am wondering if the alpha is not getting rendered properly. Here is the relevent code where I am loading the image in.
import numpy
import cv2
def augment_stream(face: numpy.array, augment: numpy.array) -> numpy.array:
face_h, face_w, _ = face.shape
augment_h, augment_w, _ = augment.shape
scalar = min(face_h / augment_h, face_w / augment_w)
delta_augment_h = int(scalar * augment_h)
delta_augment_w = int(scalar * augment_w)
delta_augment_shape = (delta_augment_w, delta_augment_h)
resized_augment = cv2.resize(augment, delta_augment_shape)
augmented_face = face.copy()
dark_pixels = (resized_augment < 250).all(axis=2)
offset_x = int((face_w - delta_augment_w) / 2)
offset_y = int((face_h - delta_augment_h) / 2)
augmented_face[offset_y: offset_y+delta_augment_h, offset_x: offset_x+delta_augment_w][dark_pixels] = resized_augment[dark_pixels]
return augmented_face
def main():
stream = cv2.VideoCapture(0)
cascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
augment = cv2.imread('assets/normal.png')
# tmp = cv2.cvtColor(augment, cv2.COLOR_BGR2GRAY)
# _,alpha = cv2.threshold(tmp,0,255,cv2.THRESH_BINARY)
# b, g, r = cv2.split(augment)
# rgba = [b,g,r, alpha]
# dst = cv2.merge(rgba,4)
# cv2.imwrite("assets/normal.png", dst)
while True:
ret, border = stream.read()
border_h, border_w, _ = border.shape
bw = cv2.equalizeHist(cv2.cvtColor(border, cv2.COLOR_BGR2GRAY))
rects = cascade.detectMultiScale(bw, scaleFactor=1.3, minNeighbors=4, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
for x, y, w, h in rects:
y0 = int(y - 0.25*h)
y1 = int(y + 0.75*h)
x0 = x
x1 = x + w
if x0 < 0 or x1 > border_w or y0 < 0 or y1 > border_h:
continue
border[y0: y1, x0: x1] = augment_stream(border[y0: y1, x0: x1], augment)
cv2.imshow('border', border)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
stream.release()
cv2.destroyAllWindows()
if __name__ == '__main__':
main()
Using example from question overlay a smaller image on a larger image python OpenCv
I reduced it to show only how to put image
use cv2.IMREAD_UNCHANGED to load it as RGBA.
split it to RGB and A
use A to create masks for image and border
use loop to add channels
import cv2
stream = cv2.VideoCapture(0)
# load RGBA
augment = cv2.imread('image.png', cv2.IMREAD_UNCHANGED) # load RGBA
# make it smaller then frame - only for test
W = 320
H = 240
augment = cv2.resize(augment, (W, H))
# split image and alpha
image = augment[:,:,0:3]
alpha = augment[:,:,3]
mask_image = alpha / 255.0
mask_border = 1.0 - mask_image
# ROI - region of interest
x1 = 200
y1 = 100
x2 = x1 + W
y2 = y1 + H
while True:
ret, border = stream.read()
# copy only in some region (ROI) (don't assign to variable) but gives worse result
#cv2.copyTo(image, alpha, border[y1:y2, x1:x2])
for c in range(0, 3): # channels RGB
border[y1:y2, x1:x2, c] = (image[:, :, c]*mask_image + border[y1:y2, x1:x2, c]*mask_border)
cv2.imshow('border', border)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
stream.release()
cv2.destroyAllWindows()
BTW: I tried to use cv2.copyTo(image, mask_image, border) but it gives worse result - maybe mask/alpha needs 3 channels.
It seems it can be done in C/C++ - how to insert a small size image on to a big image
Related
I have the next Image:
I want to find all the occurrences of an specific character, let's say "a", and then replacing it with another character, let's say "e"
This can be done in Python3 using OpenCV and Numpy, to do that I'm using the next two images in order to do template matching:
my code is the next:
import cv2 as cv
import numpy as np
img = cv.imread('source_image.jpg') # Importing Source Image
imgray = cv.cvtColor(img, cv.COLOR_BGR2GRAY) # Converting Source Image to Gray Scale
template_a = cv.imread('template_a.jpg') # Reading Template "a" Character
template_a_gray = cv.cvtColor(template_a, cv.COLOR_BGR2GRAY) # Converting Template "a" Character to Gray Scale
template_e = cv.imread('template_e.jpg') # Reading Template "e" Character
template_e_gray = cv.cvtColor(template_e, cv.COLOR_BGR2GRAY) # Converting Template "e" Character to Gray Scale
# Template Matching
w, h = template_a_gray.shape[::-1] # Obtaining Width and Height of Template "a" Character
res = cv.matchTemplate(imgray, template_a_gray, cv.TM_CCOEFF_NORMED) # Template Matching
threshold = 0.6
loc = np.where( res >= threshold)
for pt in zip(*loc[::-1]):
cv.rectangle(imgray, pt, (pt[0] + w, pt[1] + h), color=(255,255,255), thickness=cv.FILLED) # Removing Matched Template "a" Characters
y_offset = pt[1]
x_offset = pt[0]
x_end = x_offset + template_e_gray.shape[1]
y_end = y_offset + template_e_gray.shape[0]
imgray[y_offset:y_end,x_offset:x_end] = template_e_gray # Pasting Template "e" Characters
filename = 'savedImage.jpg'
cv.imwrite(filename, imgray) # Saving Image
cv.waitKey(0)
cv.destroyAllWindows()
When executed the result is:
The problem is that when zoom in, the pasted image isn't exactly the same as the template character for e
Blobs appear above and in the left of the original "e" template character. Why is this happening? How do I get rid of them?
Edit: in Response of Christoph Rackwitz's Comment:
Edit: Code fixed upon Christoph Rackwitz's Suggestions
This is because template_e_gray is pasted multiple times. Blobs are the edges of another e pasted just before. Print pt and see where e is pasted.
There are several ways to deal with this problem. The easiest way is to set the threshold to 0.8.
threshold = 0.8
If this is sufficient, use this. But may not be sufficient for some applications because the threshold needs to be tuned per image.
Another way is to fill white with one pixel larger.
imgray[y_offset - 1:y_end + 1, x_offset - 1:x_end + 1] = 255
imgray[y_offset:y_end, x_offset:x_end] = template_e_gray # Pasting Template "e" Characters
This is easy too, but inefficient, misaligned, and may not work with large images.
My recommandation is to use non maximum suppression (NMS). NMS is a technique commonly used in object detection that delete all overlapping rectangles except the one with the highest score.
Borrowing the implementation from here, this is the complete code.
import cv2 as cv
import numpy as np
# https://github.com/rbgirshick/fast-rcnn/blob/master/lib/utils/nms.py
def nms(dets, thresh):
x1 = dets[:, 0]
y1 = dets[:, 1]
x2 = dets[:, 2]
y2 = dets[:, 3]
scores = dets[:, 4]
areas = (x2 - x1 + 1) * (y2 - y1 + 1)
order = scores.argsort()[::-1]
keep = []
while order.size > 0:
i = order[0]
keep.append(i)
xx1 = np.maximum(x1[i], x1[order[1:]])
yy1 = np.maximum(y1[i], y1[order[1:]])
xx2 = np.minimum(x2[i], x2[order[1:]])
yy2 = np.minimum(y2[i], y2[order[1:]])
w = np.maximum(0.0, xx2 - xx1 + 1)
h = np.maximum(0.0, yy2 - yy1 + 1)
inter = w * h
ovr = inter / (areas[i] + areas[order[1:]] - inter)
inds = np.where(ovr <= thresh)[0]
order = order[inds + 1]
return keep
img = cv.imread('source_image.jpg') # Importing Source Image
imgray = cv.cvtColor(img, cv.COLOR_BGR2GRAY) # Converting Source Image to Gray Scale
template_a = cv.imread('temp/template_a.jpg') # Reading Template "a" Character
template_a_gray = cv.cvtColor(template_a, cv.COLOR_BGR2GRAY) # Converting Template "a" Character to Gray Scale
template_e = cv.imread('temp/template_e.jpg') # Reading Template "e" Character
template_e_gray = cv.cvtColor(template_e, cv.COLOR_BGR2GRAY) # Converting Template "e" Character to Gray Scale
# Template Matching
w, h = template_a_gray.shape[::-1] # Obtaining Width and Height of Template "a" Character
res = cv.matchTemplate(imgray, template_a_gray, cv.TM_CCOEFF_NORMED) # Template Matching
threshold = 0.6
loc = np.where(res >= threshold)
top, left = loc
score = res[loc]
iou_threshold = 0.5
matched_boxes = np.array([left, top, left + w, top + h, score]).T
valid_indices = nms(matched_boxes, iou_threshold)
boxes = matched_boxes[valid_indices]
boxes = boxes[..., :-1].astype(int)
for pt in boxes:
print(pt)
cv.rectangle(imgray, pt[:2], pt[2:], color=(255, 255, 255),
thickness=cv.FILLED) # Removing Matched Template "a" Characters
y_offset = pt[1]
x_offset = pt[0]
x_end = x_offset + template_e_gray.shape[1]
y_end = y_offset + template_e_gray.shape[0]
imgray[y_offset:y_end, x_offset:x_end] = template_e_gray # Pasting Template "e" Characters
filename = 'temp/savedImage.jpg'
cv.imwrite(filename, imgray) # Saving Image
cv.waitKey(0)
cv.destroyAllWindows()
This code works perfectly as expected, but you will see the following results. See the area circled in red.
This is a trace of a, not e.
This is caused by the difference in size between a in the image and a in the template, and is unrelated to the original issue. I mentioned this because I thought you would suspect that the same issue remains.
I originally attempted to use the script located here to darken the highlights of a captured 8mm film frame without affecting the shadows.
I never call the script with parameters that returned an acceptable improvement in the image. I have since attempted to try this code:
import numpy as np
import cv2
def ResizeWithAspectRatio(image, width=None, height=None, inter=cv2.INTER_AREA):
dim = None
(h, w) = image.shape[:2]
if width is None and height is None:
return image
if width is None:
r = height / float(h)
dim = (int(w * r), height)
else:
r = width / float(w)
dim = (width, int(h * r))
return cv2.resize(image, dim, interpolation=inter)
def gammaCorrection(src, gamma):
invGamma = 1 / gamma
table = [((i / 255) ** invGamma) * 255 for i in range(256)]
table = np.array(table, np.uint8)
return cv2.LUT(src, table)
image = cv2.imread(r'X:\temp2\Test00001-02.jpg')
resize = ResizeWithAspectRatio(image, width=1280) # Resize by width OR
# resize = ResizeWithAspectRatio(image, height=1280) # Resize by height
cv2.imshow('Image', resize)
#gammaImg = gammaCorrection(resize, 0.3)
#cv2.imshow('Gamma corrected image', gammaImg)
# Convert to HSV and take V channel
V = cv2.cvtColor(resize,cv2.COLOR_BGR2HSV)[...,2]
# Threshold V channel at 100 to make alpha channel (A)
_, A = cv2.threshold(V,150,255,cv2.THRESH_BINARY)
# Threshold V channel at 100 to make alpha channel (A)
_, A2 = cv2.threshold(V,0,255,cv2.THRESH_BINARY)
# Stack A channel onto RGB channels
result = np.dstack((resize,A))
# Stack A channel onto RGB channels
fullchannel = np.dstack((resize,A2))
gammaImg = gammaCorrection(result, 0.3)
final_im = cv2.addWeighted(fullchannel, 1, gammaImg, 0.5, 0)
# Save result
cv2.imwrite(r'X:\temp2\original_im.png',resize)
cv2.imwrite(r'X:\temp2\final_im.png',final_im)
cv2.imwrite(r'X:\temp2\gamma_adjusted.png',gammaImg)
cv2.imwrite(r'X:\temp2\fullchannel.png',fullchannel)
The following images show the output of this script.
Original Image:
Gamma Adjusted Image:
Full Channel Image:
Final Image:
The final image actually makes the highlights in the image even brighter, so there is something I am doing wrong when I merge the gamma adjusted image with the full channel image.
So, any thoughts on getting just the highlights above a set threshold to darken and not affect the darker regions of the image?
Hello I want to reflect an object in the image as in this image[enter image description here][1]
[1]: https://i.stack.imgur.com/N9J3I.jpg How can I get this kind of result?
It is possible that OpenCV does not have good solutions for this, take a closer look at Pillow.
from PIL import Image, ImageFilter
def drop_shadow(image, iterations=3, border=8, offset=(5,5), background_colour=0xffffff, shadow_colour=0x444444):
shadow_width = image.size[0] + abs(offset[0]) + 2 * border
shadow_height = image.size[1] + abs(offset[1]) + 2 * border
shadow = Image.new(image.mode, (shadow_width, shadow_height), background_colour)
shadow_left = border + max(offset[0], 0)
shadow_top = border + max(offset[1], 0)
shadow.paste(shadow_colour, [shadow_left, shadow_top, shadow_left + image.size[0], shadow_top + image.size[1]])
for i in range(iterations):
shadow = shadow.filter(ImageFilter.BLUR)
img_left = border - min(offset[0], 0)
img_top = border - min(offset[1], 0)
shadow.paste(image, (img_left, img_top))
return shadow
drop_shadow(Image.open('boobs.jpg')).save('shadowed_boobs.png')
Here is one way to do the reflection in Python/OpenCV.
One flips the image. Then makes a vertical ramp (gradient) image and puts that into the alpha channel of the flipped image. Then one concatenates the original and the flipped images.
Input:
import cv2
import numpy as np
# set top and bottom opacity percentages
top = 85
btm = 15
# load image
img = cv2.imread('bear2.png')
hh, ww = img.shape[:2]
# flip the input
flip = np.flip(img, axis=0)
# add opaque alpha channel to input
img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)
# make vertical gradient that is bright at top and dark at bottom as alpha channel for the flipped image
gtop = 255*top//100
gbtm = 255*btm//100
grady = np.linspace(gbtm, gtop, hh, dtype=np.uint8)
gradx = np.linspace(1, 1, ww, dtype=np.uint8)
grad = np.outer(grady, gradx)
grad = np.flip(grad, axis=0)
# alternate method
#grad = np.linspace(0, 255, hh, dtype=np.uint8)
#grad = np.tile(grad, (ww,1))
#grad = np.transpose(grad)
#grad = np.flip(grad, axis=0)
# put the gradient into the alpha channel of the flipped image
flip = cv2.cvtColor(flip, cv2.COLOR_BGR2BGRA)
flip[:,:,3] = grad
# concatenate the original and the flipped versions
result = np.vstack((img, flip))
# save output
cv2.imwrite('bear2_vertical_gradient.png', grad)
cv2.imwrite('bear2_reflection.png', result)
# Display various images to see the steps
cv2.imshow('flip',flip)
cv2.imshow('grad',grad)
cv2.imshow('result',result)
cv2.waitKey(0)
cv2.destroyAllWindows()
Ramped (Gradient) Image:
Result:
Im trying to insert one picture(transparent .png) into another on certain coordinates.
While the solution from How to add an image over another image using x,y coordinates?
frame[y: y+insert_size[1], x: x+insert_size[0]] = image (where insert_size - width and height of inserted picture) works, i also dont want black pixels(thats how opencv represents transparent pixels) on the final image.
I wrote a function that iterates pixel by pixel, and while it works - it is horribly slow(it completes about 2 image inserts per second), code:
def insert_image(frame, image, insert_coordinates, masked_value):
img_height = len(image)
img_width = len(image[0])
mask = np.ndarray((3,), np.uint8, buffer=np.array(masked_value))
y_diff = 0 #current vertical position in insert picture
for y, line in enumerate(frame):
if y_diff == img_height-1:
continue #interested until last row
if y < insert_coordinates[1] or y > insert_coordinates[1]+img_height:
continue #interested only in rows that will be changed
else:
x_diff = 0 #current horizontal position in insert picture
for x, col in enumerate(line):
if x_diff == img_width-1:
continue #interested until last column
if x < insert_coordinates[0] or x > insert_coordinates[0]+img_width:
continue #interested only in columns that will be changed
else:
if (image[y_diff][x_diff] != mask).all():
frame[y][x] = image[y_diff][x_diff] #setting pixel value if its not of masked value
x_diff += 1
y_diff += 1
return frame
maybe there is a smarter way to do so?
opencv version 4.5.0
numpy version 1.20.0rc1
UPDATE:
By "insert" i do mean assign a pixel value from image to some pixel of frame.
i added data and code for reproducible example(also modified function so its a bit faster):
"frame" - original picture, that image will be added to to, has red square sized (500,500) at (100,100) coordinates
"image" - transparent .png, sized (500,500) that will be "inserted" into original frame
"result1" - result, where red pixels were replaced with black "transparent" pixels from inserted image
"result2" - desired result
code, requires opencv-python and numpy modules: example.py
import cv2
import numpy as np
import copy
def insert_image_v2(frame, image, insert_coordinates, masked_value):
img_height = len(image)
img_width = len(image[0])
mask = np.ndarray((3,), np.uint8, buffer=np.array(masked_value))
y_diff = 0
for y in range(insert_coordinates[1], insert_coordinates[1]+img_height, 1):
x_diff = 0
for x in range(insert_coordinates[0], insert_coordinates[0]+img_width, 1):
if (image[y_diff][x_diff] != mask).all():
frame[y][x] = image[y_diff][x_diff]
x_diff += 1
y_diff += 1
return frame
if __name__ == "__main__":
frame = cv2.imread('frame.png')
image = cv2.imread('image.png')
insert_size = (image.shape[0], image.shape[1])
insert_coordinates = (100, 100)
x = insert_coordinates[0]
y = insert_coordinates[1]
result1 = copy.deepcopy(frame)
result1[y: y+insert_size[1], x: x+insert_size[0]] = image
result2 = insert_image_v2(frame, image, insert_coordinates, [0,0,0])
cv2.imshow('result1', result1)
cv2.imshow('result2', result2)
cv2.imwrite('result1.jpg', result1)
cv2.imwrite('result2.jpg', result2)
print()
Found a solution in Image alpha composite with OpenCV, it is about 20 times faster than what i had. code:
import cv2
import numpy as np
import time
def insert_image_v2(frame, image, insert_coordinates):
x = insert_coordinates[0]
y = insert_coordinates[1]
insert_size = (image.shape[1], image.shape[0])
background = frame[y: y+insert_size[1], x: x+insert_size[0]]
foreground = image
kernel = np.ones((5,5), np.uint8)
image_gray = cv2.cvtColor(foreground, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(image_gray, 1, 255, cv2.THRESH_BINARY_INV)
opening = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
output = np.zeros(foreground.shape, dtype=foreground.dtype)
for i in range(3):
output[:,:,i] = background[:,:,i] * (opening/255)+foreground[:,:,i]*(1-opening/255)
frame[y: y+insert_size[1], x: x+insert_size[0]] = output
return frame
if __name__ == "__main__":
frame = cv2.imread('frame.png')
image = cv2.imread('image_1.png')
insert_size = (image.shape[0], image.shape[1])
insert_coordinates = (100, 100)
t1 = time.time()
frame = insert_image_v2(frame, image, insert_coordinates)
t2 = time.time()
print(f"{t2-t1}")
cv2.imshow('img', frame)
cv2.waitKey(0)
I'm trying to blend 3 images.
First i overlay photo.png over base.png with some positioning. And it works.
base_photo.png
Second i'm trying to overlay effect.png
effect.png
over resulting image from previous step.
The result looks like this
result.png
I'm using opencv blending described here. I've tried to use cv2.addWeighted but the result was the same
# blending function
def img_overlay(background, overlay, x_offset, y_offset):
y1, y2 = y_offset, y_offset + overlay.shape[0]
x1, x2 = x_offset, x_offset + overlay.shape[1]
alpha_s = overlay[:, :, 3] / 255.0
alpha_l = 1.0 - alpha_s
for c in range(0, 3):
background[y1:y2, x1:x2, c] = (alpha_s * overlay[:, :, c] +
alpha_l * background[y1:y2, x1:x2, c])
return background
# First step
background = cv2.imread('base.png')
overlay = cv2.imread('photo.png', -1)
x_offset = 386
y_offset = 70
base_photo = img_overlay(background, overlay, x_offset, y_offset)
# Second step
overlay = cv2.imread('effect.png', -1)
final_photo = img_overlay(base_photo, overlay, 0, 0)
cv2.imwrite(result, final_photo)
How i can fix img_overlay function so it will overlay effect.png correctly?
The problem is your 'effect' image is in the a different scale. Correct this by dividing by 255.
Yours 'Second Step' should look like this:
# Second step
overlay = cv2.imread('effect.png', -1)
overlay = overlay / 255.0
final_photo = img_overlay(base_photo, overlay, 0, 0)
Which works for me.