Related
I have an image where I am creating rectangle over a specified area. The image is :
I am reading this image passing it through yolo algorithm gives me co-ordinates for rectangle around this gesture
the x1 , y1 , x2 , y2 values are
print(x1 , y1 , x2 , y2)
tensor(52.6865) tensor(38.8428) tensor(143.1934) tensor(162.9857)
Using these to add a rectangle over the image
box_w = x2 - x1
box_h = y2 - y1
color = bbox_colors[int(np.where(unique_labels == int(cls_pred))[0])]
# Create a Rectangle patch
bbox = patches.Rectangle((x1, y1), box_w, box_h, linewidth=2, edgecolor=color, facecolor="none")
# Add the bbox to the plot
ax.add_patch(bbox)
It results in the follwing image :
Now, I want to blacken everything around this square. For this purpose i am saving the above image and reading it back then using opencv to blacken the rest using following code.
x1 = int(x1)
y1 = int(y1)
x2 = int(x2)
y2 = int(y2)
# read image
img = cv2.imread(output_path)
#creating black mask
mask = np.zeros_like(img)
mask = cv2.rectangle(mask, (x1, y1), (x2,y2), (255,255,255), -1)
# apply mask to image
result = cv2.bitwise_and(img, mask)
# save results
cv2.imwrite(output_path, result)
I am getting the following image as result :
There are 2 issues :
cv2.rectangle only takes integer values as co-ordinates
May be x, y axis has different direction in yolo and open cv. Just guessing cause integers co-ordinate values should not be giving such vast difference from the rectangle.
This is being done in Jupyter notebook on Win 10.
I'm trying to process image on OpenCV Python3. I searched and tried various methods to do the processing.
This is far as I could go, I need some help or tips or at least what should i look for.
This is what i use line detection for detecting outlines of carpet:
My sample input images are stored on the mega.nz site.
This is the code:
#pygame and camera lib
import os
import pygame, sys
from pygame.locals import *
import pygame.camera
import numpy as np
import cv2
def draw_lines(img, houghLines, color=[0, 255, 0], thickness=2):
for line in houghLines:
for rho,theta in line:
a = np.cos(theta)
b = np.sin(theta)
x0 = a*rho
y0 = b*rho
x1 = int(x0 + 1000*(-b))
y1 = int(y0 + 1000*(a))
x2 = int(x0 - 1000*(-b))
y2 = int(y0 - 1000*(a))
cv2.line(img,(x1,y1),(x2,y2),color,thickness)
def weighted_img(img, initial_img, α=0.8, β=1., λ=0.):
return cv2.addWeighted(initial_img, α, img, β, λ)
#Variables
width = 1024
height = 768
Threshold1 = 10;
Threshold2 = 40;
FilterSize = 4
rho_resolution = 1
theta_resolution = np.pi/180
threshold = 155
#initialise pygame
pygame.init()
pygame.camera.init()
cam = pygame.camera.Camera("/dev/video0",(width,height))
cam.start()
#setup window
windowSurfaceObj = pygame.display.set_mode((width,height),1,16)
pygame.display.set_caption('Camera')
i=1
while i<50 :
#take a picture
image = cam.get_image()
#display the picture
windowSurfaceObj.blit(image,(0,0))
pygame.display.update()
#pygame.image.save(windowSurfaceObj,'picture2.jpg')
#image=cv2.imread('picture2.jpg',1)
print(i,". frame")
i=i+1;
gray_image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
blurred_image = cv2.GaussianBlur(gray_image, (25, 25), 0)
edges_image = cv2.Canny(blurred_image, Threshold1, Threshold2, FilterSize)
hough_lines = cv2.HoughLines(edges_image, rho_resolution , theta_resolution , threshold)
hough_lines_image = np.zeros_like(image)
draw_lines(hough_lines_image, hough_lines)
original_image_with_hough_lines = weighted_img(hough_lines_image,image)
windowSurfaceObj.blit(image,(0,0))
pygame.display.update()
#imshow hough_lines_image & draw_lines & original_image_with_hough_lines
print("fin.")
pygame.display.quit()
cam.stop()
My problem is with this code: if I remove image processing just to observe, the image quality is kinda bad, but if I can process it, it might be OK for me.
If i save image and read it with cv2.imread() i get this error:
windowSurfaceObj.blit(image,(0,0)) TypeError: argument 1 must be pygame.Surface, not numpy.ndarray
If i dont save&read it and just use image from this line image = cam.get_image(), OpenCV doesnt like it and says:
TypeError: src is not a numpy array, neither a scalar
How can I convert these datatypes?
Is there any other way I can use for this process?
I want to process images like the ones on top and I use logitech c110 camera and Raspberry Pi 3 model B+. Could you, perhaps, suggest a more suitable way to approach this problem?
Context: I am performing Object Localisation and wanting to implement an Inhibition of Return mechanism (i.e. drawing a black cross on the image where the red bounding box is after a trigger action.)
Problem: I do not know how to accurately scale the bounding box (red) in relation to the original input (init_input). If this scaling is understood, then the black cross should be accurately placed in the middle of the red bounding box.
My current code for this function is as follows:
def IoR(b, init_input, prev_coord):
"""
Inhibition-of-Return mechanism.
Marks the region of the image covered by
the bounding box with a black cross.
:param b:
The current bounding box represented as [x1, y1, x2, y2].
:param init_input:
The initial input volume of the current episode.
:param prev_coord:
The previous state's bounding box coordinates (x1, y1, x2, y2)
"""
x1, y1, x2, y2 = prev_coord
width = 12
x_mid = (b[2] + b[0]) // 2
y_mid = (b[3] + b[1]) // 2
# Define vertical rectangle coordinates
ver_x1 = int(((x_mid) * IMG_SIZE / (x2 - x1)) - width)
ver_x2 = int(((x_mid) * IMG_SIZE / (x2 - x1)) + width)
ver_y1 = int((b[1]) * IMG_SIZE / (y2 - y1))
ver_y2 = int((b[3]) * IMG_SIZE / (y2 - y1))
# Define horizontal rectangle coordinates
hor_x1 = int((b[0]) * IMG_SIZE / (x2 - x1))
hor_x2 = int((b[2]) * IMG_SIZE / (x2 - x1))
hor_y1 = int(((y_mid) * IMG_SIZE / (y2 - y1)) - width)
hor_y2 = int(((y_mid) * IMG_SIZE / (y2 - y1)) + width)
# Draw vertical rectangle
cv2.rectangle(init_input, (ver_x1, ver_y1), (ver_x2, ver_y2), (0, 0, 0), -1)
# Draw horizontal rectangle
cv2.rectangle(init_input, (hor_x1, hor_y1), (hor_x2, hor_y2), (0, 0, 0), -1)
The desired effect can be seen below:
Note: I believe the complexity in this problem arises due to the image being resized (to 224, 224, 3) each time I take an action (and consequently move onto the next state). Therefore, the "anchor" to determine the scaling must be extracted from the previous states scaling, which is shown in the following code:
def next_state(init_input, b_prime, g):
"""
Returns the observable region of the next state.
Formats the next state's observable region, defined
by b_prime, to be of dimension (224, 224, 3). Adding 16
additional pixels of context around the original bounding box.
The ground truth box must be reformatted according to the
new observable region.
IMG_SIZE = 224
:param init_input:
The initial input volume of the current episode.
:param b_prime:
The subsequent state's bounding box.
:param g: (init_g)
The initial ground truth box of the target object.
"""
# Determine the pixel coordinates of the observable region for the following state
context_pixels = 16
x1 = max(b_prime[0] - context_pixels, 0)
y1 = max(b_prime[1] - context_pixels, 0)
x2 = min(b_prime[2] + context_pixels, IMG_SIZE)
y2 = min(b_prime[3] + context_pixels, IMG_SIZE)
# Determine observable region
observable_region = cv2.resize(init_input[y1:y2, x1:x2], (224, 224), interpolation=cv2.INTER_AREA)
# Resize ground truth box
g[0] = int((g[0] - x1) * IMG_SIZE / (x2 - x1)) # x1
g[1] = int((g[1] - y1) * IMG_SIZE / (y2 - y1)) # y1
g[2] = int((g[2] - x1) * IMG_SIZE / (x2 - x1)) # x2
g[3] = int((g[3] - y1) * IMG_SIZE / (y2 - y1)) # y2
return observable_region, g, (b_prime[0], b_prime[1], b_prime[2], b_prime[3])
Explanation:
There is a state t in which the agent is predicting the location of the target object. The target object has a ground truth box (yellow in image, dotted in sketch), and the agent's current "localising box" is the red bounding box. Say, at state t the agent decides it is best to move right. Consequently, the bounding box is moved to the right, and then the next state, t' is determined by adding an additional 16 pixels of context around the red bounding box, cropping the original image with respect to this boundary, and then upscaling the cropped image back to 224, 224 in dimensions.
Say the agent is now confident that its prediction is accurate, so it chooses the trigger action. This basically means, end the current target object's localisation episode and place a black cross on where the agent predicted the object was (i.e. in the middle of the red bounding box). Now, since the current state is zoomed in after being cropped following the previous action, the bounding box must be re-scaled with respect to the normal/original/initial image and then the black cross can be drawn accurately onto the image.
In the context of my problem, the first rescaling between states is working perfectly well (the second code in this post). However, scaling back to normal and drawing the black cross is what I cannot seem to get my head around.
Here is an image which hopefully helps the explanation:
Here is the output of my current solution (please click the image to zoom in):
I think it's better to save the coordinate globally instead of using a bunch of upscale/downscale. They give me headache and there might be loss of precision due to rounding.
That is, every time you detect something, you convert it to global (original image) coordinate first. I have written a small demo here, imitating your detection and trigger behavior.
Initial detection:
Zoomed in, another detection:
Zoomed in, another detection:
Zoomed in, another detection:
Zoomed back to original scale, with the detection box in the correct location
Code:
import cv2
import matplotlib.pyplot as plt
IMG_SIZE = 224
im = cv2.cvtColor(cv2.imread('lena.jpg'), cv2.COLOR_BGR2GRAY)
im = cv2.resize(im, (IMG_SIZE, IMG_SIZE))
# Your detector results
detected_region = [
[(10, 20) , (80, 100)],
[(50, 0) , (220, 190)],
[(100, 143) , (180, 200)],
[(110, 45) , (180, 150)]
]
# Global states
x_scale = 1.0
y_scale = 1.0
x_shift = 0
y_shift = 0
x1, y1 = 0, 0
x2, y2 = IMG_SIZE-1, IMG_SIZE-1
for region in detected_region:
# Detection
x_scale = IMG_SIZE / (x2-x1)
y_scale = IMG_SIZE / (y2-y1)
x_shift = x1
y_shift = y1
cur_im = cv2.resize(im[y1:y2, x1:x2], (IMG_SIZE, IMG_SIZE))
# Assuming the detector return these results
cv2.rectangle(cur_im, region[0], region[1], (255))
plt.imshow(cur_im)
plt.show()
# Zooming in, using part of your code
context_pixels = 16
x1 = max(region[0][0] - context_pixels, 0) / x_scale + x_shift
y1 = max(region[0][1] - context_pixels, 0) / y_scale + y_shift
x2 = min(region[1][0] + context_pixels, IMG_SIZE) / x_scale + x_shift
y2 = min(region[1][1] + context_pixels, IMG_SIZE) / y_scale + y_shift
x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
# Assuming the detector confirm its choice here
print('Confirmed detection: ', x1, y1, x2, y2)
# This time no padding
x1 = detected_region[-1][0][0] / x_scale + x_shift
y1 = detected_region[-1][0][1] / y_scale + y_shift
x2 = detected_region[-1][1][0] / x_scale + x_shift
y2 = detected_region[-1][1][1] / y_scale + y_shift
x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
cv2.rectangle(im, (x1, y1), (x2, y2), (255, 0, 0))
plt.imshow(im)
plt.show()
This also prevents resizing on a resized image which might create more artifacts and worsen the detector's performance.
Imagine a point (x, y) in a 500x500 image. Let it be (100, 200).
After scaling it to a different size, say 250x250 - the correct way to scale it would be to just look at the current co-ordinate and do new_coord = old_coord * NEW_SIZE/OLD_SIZE.
Thus, (100,200) will be transformed to (50,100)
If you replace your scaling using x2-x1 and use a simpler rescaling formula, it should fix your problem.
Update: NEW_SIZE and OLD_SIZE may be different for the two co-ordinates based on the shape of the original image and final image, if they are rectangular and not square.
I'm working on a image processing project and I'm trying to detect the shape of a square in a distorted image with opencv 3.4.0 and python 3.
The image is this one:
As you can see the black square is pretty distorted, but I'm interested in detecting only the right side that will always be on focus and not distorted.
I've already tried to use the Hough Line Transformation method with the code below, but it detects the line right to the square:
temp = cv2.imread("images/macro/noF3.jpg")
grTemp = cv2.cvtColor(temp, cv2.COLOR_BGR2GRAY)
asd = cv2.Canny(grTemp, 50, 150, apertureSize = 3)
lines = cv2.HoughLines(asd.copy(), 1, np.pi/180, 200)
for rho, theta in lines[0]:
a = np.cos(theta)
b = np.sin(theta)
x0 = a*rho
y0 = b*rho
x1 = int(x0 + 1000*(-b))
y1 = int(y0 + 1000*(a))
x2 = int(x0 - 1000*(-b))
y2 = int(y0 - 1000*(a))
cv2.line(temp, (x1,y1), (x2,y2), (255,0,0), 3)
I've also tried to use Canny and the findContours method, but it didn't work well, moreover I can't blur the image.
I would like to obtain the result seen in the image below:
Thanks
If I have the polar coordinates of a line, how can I draw it on an image in OpenCV & python?
Line function takes 2 points, but draws only the segment. I want to draw a line from one edge of the image to other.
Just calculate for 2 points outside. opencv's Line is fine with e.g. (-10,-10) for a point.
import cv2 # python-opencv
import numpy as np
width, height = 800, 600
x1, y1 = 0, 0
x2, y2 = 200, 400
image = np.ones((height, width)) * 255
line_thickness = 2
cv2.line(image, (x1, y1), (x2, y2), (0, 255, 0), thickness=line_thickness)
http://docs.opencv.org/2.4/modules/core/doc/drawing_functions.html#cv2.line
Take a look to the following solution, I firstly convert a line in polar equations to cartesian and then I use numpy.vectorize() to generate a vector that allows me to get represent the line in any point of the space.
import cv2
import numpy as np
img_size = (200,200)
img = np.ones(img_size) * 255
# polar equation
theta = np.linspace(0, np.pi, 1000)
r = 1 / (np.sin(theta) - np.cos(theta))
# polar to cartesian
def polar2cart(r, theta):
x = r * np.cos(theta)
y = r * np.sin(theta)
return x, y
x,y = polar2cart(r, theta)
x1, x2, y1, y2 = x[0], x[1], y[0], y[1]
# line equation y = f(X)
def line_eq(X):
m = (y2 - y1) / (x2 - x1)
return m * (X - x1) + y1
line = np.vectorize(line_eq)
x = np.arange(0, img_size[0])
y = line(x).astype(np.uint)
cv2.line(img, (x[0], y[0]), (x[-1], y[-1]), (0,0,0))
cv2.imshow("foo",img)
cv2.waitKey()
Result:
You can see how to do this in the Hough Line Transform tutorial.
import cv2
import numpy as np
img = cv2.imread('dave.jpg')
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
edges = cv2.Canny(gray,50,150,apertureSize = 3)
lines = cv2.HoughLines(edges,1,np.pi/180,200)
for rho,theta in lines[0]:
a = np.cos(theta)
b = np.sin(theta)
x0 = a*rho
y0 = b*rho
x1 = int(x0 + 1000*(-b))
y1 = int(y0 + 1000*(a))
x2 = int(x0 - 1000*(-b))
y2 = int(y0 - 1000*(a))
cv2.line(img,(x1,y1),(x2,y2),(0,0,255),2)
cv2.imwrite('houghlines3.jpg',img)
This is one way to solve the problem of drawing infinite line segment in OpenCV with two given points.
### function to find slope
def slope(p1,p2):
x1,y1=p1
x2,y2=p2
if x2!=x1:
return((y2-y1)/(x2-x1))
else:
return 'NA'
### main function to draw lines between two points
def drawLine(image,p1,p2):
x1,y1=p1
x2,y2=p2
### finding slope
m=slope(p1,p2)
### getting image shape
h,w=image.shape[:2]
if m!='NA':
### here we are essentially extending the line to x=0 and x=width
### and calculating the y associated with it
##starting point
px=0
py=-(x1-0)*m+y1
##ending point
qx=w
qy=-(x2-w)*m+y2
else:
### if slope is zero, draw a line with x=x1 and y=0 and y=height
px,py=x1,0
qx,qy=x1,h
cv2.line(image, (int(px), int(py)), (int(qx), int(qy)), (0, 255, 0), 2)
return image
You can use p1 and p2 according to your requirement and call the function drawLine.