Develop Reference

How i can use Lucas-Kanade Optical Flow to track 68 dlib facial landmarks in a image

Here is my code. I want Lucas-Kanade Optical Flow Algorithm to Track the Change in Facial Landmarks detected by shape_predictor_68_face_landmarks.dat. But I am Stuck.
I just want to compare 68 facial landmark points with the next frame.
cap = cv2.VideoCapture('1.mp4')
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
ret, old_frame = cap.read()
old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)
faces = detector(old_gray)
for face in faces:
landmark_vector=[]
landmarks = predictor(old_gray,face) # Shape predictor
for n in range(1, 68):
x = landmarks.part(n).x
y = landmarks.part(n).y
landmark_vector.append([x,y])
p0=np.array(landmark_vector)
# Parameters for lucas kanade optical flow
lk_params = dict( winSize = (15,15),
maxLevel = 2,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
# Create some random colors
color = np.random.randint(0,255,(100,3))
# Take first frame and find corners in it
# p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **predictor)
# Create a mask image for drawing purposes
mask = np.zeros_like(old_frame)
while(1):
ret,frame = cap.read()
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# calculate optical flow
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, landmark_vector, None, **lk_params)
# Select good points
good_new = p1[st==1]
good_old = p0[st==1]
# draw the tracks
for i,(new,old) in enumerate(zip(good_new,good_old)):
a,b = new.ravel()
c,d = old.ravel()
mask = cv2.line(mask, ((int(a)),(int(b))),((int(c)),(int(d))), color[i].tolist(), 2)
frame = cv2.circle(frame,((int(a)),(int(b))),5,color[i].tolist(),-1)
img = cv2.add(frame,mask)
cv2.imshow('frame',img)
if cv2.waitKey(1) & 0xFF == ord('q'):
# breaking the loop if the user types q
# note that the video window must be highlighted!
break
# Now update the previous frame and previous points
old_gray = frame_gray.copy()
p0 = good_new.reshape(-1,1,2)

cv2.GaussianBlur at multiple kernel sizes

Currently, I'm trying to perform motion detection with OpenCV. With each new frame, I use this bellow function to do compare with the previouse frame:
def detect(new_frame, kernel_size):
frame=cv2.cvtColor(new_frame,cv2.COLOR_BGR2GRAY) #Grayscale conversion of the frame
frame=cv2.GaussianBlur(frame, (kernel_size, kernel_size),0)
deltaFrame=cv2.absdiff(old_frame, frame)
old_frame = frame
threshFrame=cv2.threshold(deltaFrame, 5, 255, cv2.THRESH_BINARY)[1]
threshFrame=cv2.dilate(threshFrame, None, iterations=2)
(cnts,_)=cv2.findContours(threshFrame.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
return cnts
My problem is, I have to detect motion of objects of two types, each of them has it's own efficient value of kernel size parameter for that function (ie: 5 and 11). So I must to use that function 2 times with each new frame. But my device has resource limitations, so I want to reduce this process as much as I can. How can I do it?

Try bitwise functions over masks . Detect every pixel is moving. It's fast.
The trick for me is work with little resized images of frame .
import numpy as np
import cv2 as cv2
fid=0
video_path="videos/example.mp4"
cap = cv2.VideoCapture(video_path)
# Some characteristics from the original video
w_frame, h_frame = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps, num_frames = cap.get(cv2.CAP_PROP_FPS), cap.get(cv2.CAP_PROP_FRAME_COUNT)
print(fps,w_frame,h_frame)
x,y,h,w = 0,0,h_frame,w_frame
fnum=0
while(True):
ret, frame = cap.read()
if ret == None: pass
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
edges = gray
if fnum==0:
last_edges = edges.copy()
ret, mask1 = cv2.threshold(edges, 127, 255, cv2.THRESH_BINARY)
ret, mask2 = cv2.threshold(last_edges, 127 , 255, cv2.THRESH_BINARY)
dst1 = cv2.bitwise_and(mask2,mask1)
dst2 = cv2.bitwise_not(dst1)
dst4 = cv2.bitwise_and(dst2,dst2,mask=mask1)
scale_percent = 10 # percent of original size
width = int(dst4.shape[1] * scale_percent / 100)
height = int(dst4.shape[0] * scale_percent / 100)
dim = (width, height)
# resize image
mini = cv2.resize(dst4, dim, interpolation = cv2.INTER_AREA)
h,w = mini.shape
th=30 #my threshold
points=[]
for y in range(0, len(mini),4):
for x in range(0,len(mini[y]),4):
c1 = mini[y][x] > th and mini[y][x+1] > th and mini[y][x+2] > th and mini[y][x+3] > th
c2 = mini[y][x] > th and mini[y+1][x] > th and mini[y+2][x] > th and mini[y+3][x] > th
if c1 or c2:
start_point=(x*scale_percent,y*scale_percent)
points.append(start_point)
color1=(0,0,255)
color2=(0,255,255)
thickness=2
cv2.circle(frame, start_point, 20, color1, thickness)
if len(points) >= 2:
cx1 , cy1 = points[0][0] , points[0][1]
cx2 , cy2 = points[-1][0] , points[-1][1]
cxmin = min(cx1,cx2)
cymin = min(cy1,cy2)
cxmax = max(cx1,cx2)
cymax = max(cy1,cy2)
print(cymin,cymax , '--' , cxmin,cxmax)
cv2.rectangle(frame, (cxmin,cymin) , (cxmax,cymax), color2, thickness)
# Display the resulting frame
cv2.imshow('frame4', frame)
cv2.imshow('framemin', mini)
last_edges = edges.copy()
fnum+=1
if cv2.waitKey(33) & 0xFF == ord('q'):
break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
you can apply your own masks to detect one obj or another playing with blur values too.

using Opencv python2.7 to detect a rectangle

I have written a python code using opencv to detect a rectangle (displaying height, and width) also determine the distance from camera to the object but when i run the code i get this error(can be found below) i don't know what am doing wrong please i would appreciate it if anyone could help me out i have tried every possible solution that i could find but still can't rid of the error.
ERROR
yuv_red = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
error: C:\builds\master_PackSlaveAddon-win32-vc12-static\opencv\modules\imgproc\src\color.cpp:8059: error: (-215) scn == 3 || scn == 4 in function cv::cvtColor
import sys
sys.path.append('C:\Python27\Lib\site-packages')
import cv2
import numpy as np
import argparse
import math
############################# Capturing Video Through Camera ########################
cap = cv2.VideoCapture(0)
############# Distance to Camera initial value set to zero ##########################
Distance_to_Camera=0
while(True):
################################# Capture frame-by-frame ###########################
ret, frame = cap.read()
############################ Converting frame(img i.e BGR to YUV) ###################
yuv_red = cv2.cvtColor(frame, cv2.COLOR_BGR2YUV)
red_color = np.uint8([[[0,0,255]]])
yuv_color = cv2.cvtColor(red_color,cv2.COLOR_BGR2YUV)
print yuv_color
############################### Processing of Image ##############################
##################### Defining the Range of Red Colour ###########################
red_lower = np.array([136,87,111],np.uint8)
red_upper = np.array([180,255,255],np.uint8)
##################### Finding the Range of Red Colour in the image ###################
mask = cv2.inRange(yuv_red, red_lower,red_upper)
####################### Morphological Transformation, Dilation #######################
res = cv2.bitwise_and(frame, frame, mask = mask)
#####################################################################################
gray = cv2.cvtColor(res,cv2.COLOR_BGR2GRAY) #Converting the BGR res to Gray
blurred = cv2.GaussianBlur(gray, (5,5), 5) #Blur Image to remove noise
blur = cv2.bilateralFilter(blurred, 5,50,50) #Smooth the image
median = cv2.medianBlur(blur,5) #Reduce noise from image
thresh = cv2.threshold(median, 3, 255, cv2.THRESH_BINARY)[1] #To achieve a better output of white and black
frame2, contour, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
##################### Splitting and Merging Image channels ###########################
b,g,r = cv2.split(res)
ttl = res.size/3
Ra = float(np.sum(r))/ ttl
print Ra
if Ra > 1:
c = contours[0]
M = cv2.moments(c)
x = int(M['m10']/M['m00'])
y = int(M['m01']/M['m00'])
x,y,w,h = cv2.boundingRect(c)
epsilon = 0.01*cv2.arcLength(c, True)
approx = cv2.approxPolyDP(c, epsilon, True)
perimeter = cv2.approxLength(c,True)
area = cv2.contourArea(c)
ah = h/40
aw = w/40
Distance_to_Camera = round(math.sqrt(315/area),4)
print Distance_to_Camera
approx = cv2.approxPolyDP(c, epsilon, True)
print len(approx)
shape = len(approx)
if shape == 4:
print " 4 Sides"
rect = cv2.minAreaRect(c)
box = cv2.boxPoints(rect)
box = np.int0(box)
cv2.drawContours(frame,ArithmeticError[box],0,(0,0,255),0)
print box
if ah == aw:
################################ Displaying Text on the Image ################################
print "Unknown"
cv2.putTextputText(frame,"Unknown",(x+150,y+150),cv2.FONT_HERSHEY_SIMPLEX,2,(255,255,255),4)
else:
################################ Displaying Text on the Image ################################
print "Rectangle"
cv2.putTextputText(frame,"Rectangle",(x+150,y+150),cv2.FONT_HERSHEY_SIMPLEX,2,(255,255,255),4)
output = ("Distance="+str((round((distance+0.0004)*1000))) + "cm" + "X=" + str(aw)+"cm" +"Y="+str(ah)+"cm"+"Perimeter=" +str(round(perimeter/40))+"cm"+"Area="+str(round((area/1.64)/1000))+"cm^2")
cv2.imshow('gray',frame)
else:
cv2.imshow('gray',frame)
########################################## Output ##########################################
cv2.imshow('gray',frame)
cv2.imshow('grayscaled',thresh)
if cv2.waitKey(20) & 0xFF == 27:
break
##if k == 27: # wait for ESC key to exit
## cv2.destroyAllWindows()
# When everything done, release the capture
cv2.destroyAllWindows()
cap.release()
[1]: https://i.stack.imgur.com/qB2rx.png

In the line implementing a bilateral filter you have an erroneous parenthesis:
blur = cv2.bilateralFilter(blurred, (5,50,50)
Check out the docs for cv2.bilateralFilter() to see the proper use.

AR Drone feedback

thanks for taking the time to read, and hopefully help me.
I have an AR.Drone 2.0 that I have already started to program/develop. I am using python for coding with opencv for the image processing. I want to be able to feedback this code to the drone. I was thinking about obtaining frames from the video stream and have the AR.Drone perform some tasks based upon the images. I, however, don't know where to start. It would be helpful for me if someone can point me in the right direction.
import numpy as np
import cv2
# open the camera
cap = cv2.VideoCapture('tcp://192.168.1.1:5555')
def nothing(x):
pass
cv2.namedWindow('result')
# Starting with 100's to prevent error while masking
h,s,v = 100,100,100
# Creating track bar
cv2.createTrackbar('h', 'result',0,179,nothing)
cv2.createTrackbar('s', 'result',0,255,nothing)
cv2.createTrackbar('v', 'result',0,255,nothing)
while True:
#read the image from the camera
ret, frame = cap.read()
#You will need this later
frame = cv2.cvtColor(frame, 35)
#converting to HSV
hsv = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
# get info from track bar and appy to result
h = cv2.getTrackbarPos('h','result')
s = cv2.getTrackbarPos('s','result')
v = cv2.getTrackbarPos('v','result')
# Normal masking algorithm
lower_blue = np.array([h,s,v])
upper_blue = np.array([180,255,255])
mask = cv2.inRange(hsv,lower_blue, upper_blue)
result = cv2.bitwise_and(frame,frame,mask = mask)
cv2.imshow('result',result)
#find center
cnts=cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)[-2]
center=None
if len(cnts)>0:
c=max(cnts, key=cv2.contourArea)
((x,y),radius)=cv2.minEnclosingCircle(c)
M=cv2.moments(c)
center=(int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
if radius>10:
#cv2.circle(frame, (int(x),int(y)), int(radius), 2)
cv2.circle(frame, center,5,(0,0,255),-1)
# color detection limits
lB = 5
lG = 50
lR = 50
hB = 15
hG = 255
hR = 255
lowerLimits = np.array([lB, lG, lR])
upperLimits = np.array([hB, hG, hR])
# Our operations on the frame come here
thresholded = cv2.inRange(frame, lowerLimits, upperLimits)
outimage = cv2.bitwise_and(frame, frame, mask = thresholded)
cv2.imshow('original', frame)
# Display the resulting frame
cv2.imshow('processed',outimage)
# Quit the program when Q is pressed
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
print 'closing program'
cap.release()
cv2.destroyAllWindows()'

OpenCV Python: Error with using Mask parameter in GoodFeatureToDetect

I was trying to make a facial detection program in Python that combines Haar Cascade Classification and Lucas Kanade. But I am getting error saying something like this:
Error:
Traceback (most recent call last):
File "/home/anthony/Documents/Programming/Python/Computer-Vision/OpenCV-Doc/optical-flow-and-haar-detection-test.py", line 80, in <module>
corners_t = cv2.goodFeaturesToTrack(gray, mask = mask_use, **feature_params)
error: /build/buildd/opencv-2.4.8+dfsg1/modules/imgproc/src/featureselect.cpp:63: error: (-215) mask.empty() || (mask.type() == CV_8UC1 && mask.size() == image.size()) in function goodFeaturesToTrack
How my program works:
My program uses Haar Cascade to get coordinates of a detected face, copy whatever is in that area created by the coordinates (in this case, the face), take an image with nothing but black colors (all pixels are set to zero via numpy), and paste the copied face into the black background. By setting the new face with black background into the mask parameter, this would force Lucas Kanade (goodFeaturesToDetect) to create feature points on the face which will be tracked by optical flow.
Code:
from matplotlib import pyplot as plt
import numpy as np
import cv2
rectangle_x = 0
face_classifier = cv2.CascadeClassifier('haarcascades/haarcascade_frontalface_default.xml')
#cap = cv2.VideoCapture('video/sample.mov')
cap = cv2.VideoCapture(0)
# params for ShiTomasi corner detection
feature_params = dict( maxCorners = 200,
qualityLevel = 0.01,
minDistance = 10,
blockSize = 7 )
# Parameters for lucas kanade optical flow
lk_params = dict( winSize = (15,15),
maxLevel = 2,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
# Create some random colors
color = np.random.randint(0,255,(100,3))
# Take first frame and find corners in it
ret, old_frame = cap.read()
#old_frame = cv2.imread('images/webcam-first-frame-two.png')
######Adding my code###
cv2.imshow('Old_Frame', old_frame)
cv2.waitKey(0)
old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)
restart = True
#while restart == True:
face = face_classifier.detectMultiScale(old_gray, 1.2, 4)
if len(face) == 0:
print "This is empty"
for (x,y,w,h) in face:
focused_face = old_frame[y: y+h, x: x+w]
#cv2.rectangle(old_frame, (x,y), (x+w, y+h), (0,255,0),2)
#initalize all pixels to zero (picture completely black)
mask_use = np.zeros(old_frame.shape,np.uint8)
#Crop old_frame coordinates and paste it on the black mask)
mask_use[y:y+h,x:x+w] = old_frame[y:y+h,x:x+w]
height, width, depth = mask_use.shape
print "Height: ", height
print "Width: ", width
print "Depth: ", depth
height, width, depth = old_frame.shape
print "Height: ", height
print "Width: ", width
print "Depth: ", depth
cv2.imshow('Stuff', mask_use)
cv2.imshow('Old_Frame', old_frame)
#cv2.imshow('Zoom in', focused_face)
face_gray = cv2.cvtColor(old_frame,cv2.COLOR_BGR2GRAY)
gray = cv2.cvtColor(focused_face,cv2.COLOR_BGR2GRAY)
corners_t = cv2.goodFeaturesToTrack(gray, mask = mask_use, **feature_params)
corners = np.int0(corners_t)
#print corners
for i in corners:
ix,iy = i.ravel()
cv2.circle(focused_face,(ix,iy),3,255,-1)
cv2.circle(old_frame,(x+ix,y+iy),3,255,-1)
print ix, " ", iy
plt.imshow(old_frame),plt.show()
"""
print "X: ", x
print "Y: ", y
print "W: ", w
print "H: ", h
#face_array = [x,y,w,h]
"""
#############################
p0 = cv2.goodFeaturesToTrack(old_gray, mask = old_gray, **feature_params)
#############################
# Create a mask image for drawing purposes
mask = np.zeros_like(old_frame)
while(1):
ret,frame = cap.read()
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# calculate optical flow
p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params)
# Select good points
good_new = p1[st==1]
###print "Good_New"
###print good_new
good_old = p0[st==1]
# draw the tracks
for i,(new,old) in enumerate(zip(good_new,good_old)):
#print i
#print color[i]
a,b = new.ravel()
c,d = old.ravel()
cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2)
cv2.circle(frame,(a, b),5,color[i].tolist(),-1)
if i == 99:
break
#For circle, maybe replace (a,b) with (c,d)?
#img = cv2.add(frame,mask)
cv2.imshow('frame',frame)
k = cv2.waitKey(30) & 0xff
if k == 27:
break
# Now update the previous frame and previous points
old_gray = frame_gray.copy()
p0 = good_new.reshape(-1,1,2)
cv2.destroyAllWindows()
cap.release()
Can anyone see the problem and tell me how to fix it? Thanks.

I've had this error caused by using arrays that aren't the same size.
You have a for loop that dynamically assigns values to focused_face but the good_features to track uses a static size (= to the last instance of focused_face). Old_frame looks like it uses the shape of the first instance of focused_face.
Make sure you are using image and mask arrays of the same shape in goodFeaturesToTrack.