Develop Reference

MediaPipe pose estimator with multiprocessing hangs on its process function

I am currently trying to implement MediaPipe pose estimator as an independent event-based process with Python's multiprocessing library, but it hangs on the MediaPipe's Pose.process() function.
I input the frame with another process (readFrames). Whenever a frame is captured, it is written into a shared object and tells the MediaPipe process (MediaPipeRunner) to start working on the current image:
def readFrames(ns, event):
#initialize the video capture object
cap = cv2.VideoCapture(0)
while cap.isOpened():
ret, frame = cap.read()
if ret:
ns.frame = frame
event.set()
cv2.imshow('Orijinal Frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
cap.release()
cv2.destroyAllWindows()
return -1
else:
return
class MediaPipeRunner(mproc.Process):
def __init__(self, name, nsFrame, nsMediaPipe, eventWait, eventPublish):
super(MediaPipeRunner, self).__init__()
# Specify a name for the instance
self.name = name
# Input and output namespaces
self.nsFrame = nsFrame
self.nsMediaPipe = nsMediaPipe
# Waiter and publisher events
self.eventWait = eventWait
self.eventPublish = eventPublish
# Create a pose estimator from MediaPipe
mp_pose = mp.solutions.pose
# Specify pose estimator parameters (static)
static_image_mode = True
model_complexity = 1
enable_segmentation = True # DONT CHANGE
min_detection_confidence = 0.5
# Create a pose estimator here
self.pose = mp_pose.Pose(
static_image_mode=static_image_mode,
model_complexity=model_complexity,
enable_segmentation=enable_segmentation,
min_detection_confidence=min_detection_confidence,
smooth_landmarks=False,
)
def run(self):
while True:
eventFrame.wait()
# This part is where it gets stuck:
results = self.pose.process(cv2.cvtColor(self.nsFrame.frame, cv2.COLOR_BGR2RGB))
if not results.pose_landmarks:
continue
self.nsMediaPipe.segmentation = results.segmentation_mask
eventMP.set()
This is how I bind the processes, namespaces and events:
if __name__=="__main__":
mgr = mproc.Manager()
nsFrame = mgr.Namespace()
nsMP = mgr.Namespace()
eventFrame = mproc.Event()
eventMP = mproc.Event()
camCap = mproc.Process(name='camCap', target=readFrames, args=(nsFrame, eventFrame, ))
camCap.daemon=True
mpCap = MediaPipeRunner('mpCap', nsFrame, nsMP, eventFrame, eventMP, )
mpCap.daemon=True
camCap.start()
mpCap.start()
camCap.join()
mpCap.join()
Am I taking a wrong step on processes or MediaPipe is not getting along with the multiprocessing library of Python?
Any help will be appreciated, thanks in advance :)
P.S.: I installed MediaPipe by pip and version 0.8.9.1 is present.

I have found the problem: The process function behaves correctly when with structure is used in Python (idk why):
with mp_pose.Pose(
static_image_mode=static_image_mode,
model_complexity=model_complexity,
enable_segmentation=enable_segmentation,
min_detection_confidence=min_detection_confidence,
smooth_landmarks=False,
) as pose:
Now this part works!
results = self.pose.process(cv2.cvtColor(self.nsFrame.frame, cv2.COLOR_BGR2RGB))
I hope it might be helpful for you.

Flask python Script runs before values are given

I have a simple Flask python script where I want to via a form add 2 image paths, send those paths to python do stuff and send back a string. The problem I have is that the script runs before the form is filled out and the button is pressed. How to I make the script wait and only run on the press of the submit button?
Code for reference
#app.route('/', methods=['GET', 'POST'])
#app.route('/index')
def index():
PATH_REFERENCE = request.form.get("referencePhoto")
PATH_TEST = request.form.get("testPhoto")
testImage = cv2.imread(PATH_TEST)
reference = setupReference(PATH_REFERENCE)
face_locations, face_encodings = getFaceEmbeddingsFromImage(testImage, convertToRGB=True)
for location, face_encoding in zip(face_locations, face_encodings):
distances = face_recognition.face_distance(reference[0], face_encoding)
if distances <= 0.6:
result = 'Match!'
else:
result = 'Not Match!'
return render_template('index.html', title='Home', result=result)
The error is that The script cant do stuff with a NoneObject. Which makes sense condisering the form hasnt sent the paths needed.

Only do the form logic on a Post- on a get, just serve the form.
#app.route('/', methods=['GET', 'POST'])
#app.route('/index')
def index():
result = ""
if request.method == "POST":
PATH_REFERENCE = request.form.get("referencePhoto")
PATH_TEST = request.form.get("testPhoto")
testImage = cv2.imread(PATH_TEST)
reference = setupReference(PATH_REFERENCE)
face_locations, face_encodings = getFaceEmbeddingsFromImage(testImage, convertToRGB=True)
for location, face_encoding in zip(face_locations, face_encodings):
distances = face_recognition.face_distance(reference[0], face_encoding)
if distances <= 0.6:
result = 'Match!'
else:
result = 'Not Match!'
return render_template('index.html', title='Home', result=result)

How to call instance variable form another class and file

I have a question, I have 4 file app.py, face.py, camera.py and db.py Inside face.py file I have one variable call known_encoding_faces. If I put print code inside my face.py and run the app.py the result will display in my command prompt.
My question is how can i use known_encoding_faces variable in my camera.py? My expected result is when i run my app.py and open my webcam the command prompt will show the printed known_encoding_faces output. I believe if this work means this known_encoding_faces variable successfully can be used by camera.py file.
Here I attach my code. Hope someone can help me regarding on this matter.
app.py
from flask import Flask, Response, json, render_template
from werkzeug.utils import secure_filename
from flask import request
from os import path, getcwd
import time
from face import Face
from db import Database
app = Flask(__name__)
import cv2
from camera import VideoCamera
app.config['file_allowed'] = ['image/png', 'image/jpeg']
app.config['train_img'] = path.join(getcwd(), 'train_img')
app.db = Database()
app.face = Face(app)
def gen(camera):
while True:
frame = camera.get_frame()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
#app.route('/video_feed')
def video_feed():
return Response(gen(VideoCamera()),
mimetype='multipart/x-mixed-replace; boundary=frame')
#app.route('/')
def index():
return render_template('index.html')
def success_handle(output, status=200, mimetype='application/json'):
return Response(output, status=status, mimetype=mimetype)
face.py
import face_recognition
from os import path
import cv2
import face_recognition
class Face:
def __init__(self, app):
self.train_img = app.config["train_img"]
self.db = app.db
self.faces = []
self.face_user_keys = {}
self.known_encoding_faces = [] # faces data for recognition
self.load_all()
def load_user_by_index_key(self, index_key=0):
key_str = str(index_key)
if key_str in self.face_user_keys:
return self.face_user_keys[key_str]
return None
def load_train_file_by_name(self,name):
trained_train_img = path.join(self.train_img, 'trained')
return path.join(trained_train_img, name)
def load_unknown_file_by_name(self,name):
unknown_img = path.join(self.train_img, 'unknown')
return path.join(unknown_img, name)
def load_all(self):
results = self.db.select('SELECT faces.id, faces.user_id, faces.filename, faces.created FROM faces')
for row in results:
user_id = row[1]
filename = row[2]
face = {
"id": row[0],
"user_id": user_id,
"filename": filename,
"created": row[3]
}
self.faces.append(face)
face_image = face_recognition.load_image_file(self.load_train_file_by_name(filename))
face_image_encoding = face_recognition.face_encodings(face_image)[0]
index_key = len(self.known_encoding_faces)
self.known_encoding_faces.append(face_image_encoding)
index_key_string = str(index_key)
self.face_user_keys['{0}'.format(index_key_string)] = user_id
def recognize(self,unknown_filename):
unknown_image = face_recognition.load_image_file(self.load_unknown_file_by_name(unknown_filename))
unknown_encoding_image = face_recognition.face_encodings(unknown_image)[0]
results = face_recognition.compare_faces(self.known_encoding_faces, unknown_encoding_image);
print("results", results)
index_key = 0
for matched in results:
if matched:
# so we found this user with index key and find him
user_id = self.load_user_by_index_key(index_key)
return user_id
index_key = index_key + 1
return None
camera.py
import face_recognition
from os import path
import cv2
from db import Database
from face import Face
class VideoCamera(object):
def __init__(self):
# Using OpenCV to capture from device 0. If you have trouble capturing
# from a webcam, comment the line below out and use a video file
# instead.
self.video = cv2.VideoCapture(0)
# If you decide to use video.mp4, you must have this file in the folder
# as the main.py.
# self.video = cv2.VideoCapture('video.mp4')
def __del__(self):
self.video.release()
def get_frame(self):
success, image = self.video.read()
# We are using Motion JPEG, but OpenCV defaults to capture raw images,
# so we must encode it into JPEG in order to correctly display the
# video stream.
ret, jpeg = cv2.imencode('.jpg', image)
return jpeg.tobytes()

known_encoding_faces is a member of a Face object. That means it does not exist on its own - as evidence, note you only reference self.known_encoding_faces rather than just known_encoding_faces. You need to initialize some Face object, and than you can use it. Further more, it looks like you would need to call load_all on said object to properly initialize it. The minimal thing you need is some thing like:
from face import Face
aface = Face(app) #You would need an app here
aface.load_all()
known_encoding_faces = aface.known_encoding_faces
If you are expecting this to exist irrespective of an objects creation, you need to rethink your design, and take it out of your class.
If you are expecting this to be called from the main script, you can demand this variable to initialize your camera:
VideoCamera(app.face.known_encoding_faces) #Called from main script
and in camera.py:
class VideoCamera(object):
def __init__(self,known_face_encodings):
self.known_encoding_faces = known_face_encodings
self.video = cv2.VideoCapture(0)
and in this class you can now use self.known_encoding_faces.

How can I use OpenCV with Flask to capture video/image from the Client?

I am trying to create a Flask application which will allow the users to use their webcam as a motion detector and I have a working code in OpenCV which does detect motion but right now it uses the cam of the local machine only (i.e. server). I don't know how to capture images/video from the client instead of the server and then pass it on for motion detection with OpenCV.
I found this article https://kirupa.com/html5/accessing_your_webcam_in_html5.htm which uses HTML5 and JS to capture feed from the client but it doesn't show a way to use it with OpenCV.
Here's the code for the route in the flask app
#app.route('/camera')
#login_required
def camera():
return Response(get_frame(), mimetype='multipart/x-mixed- replace; boundary=frame')
And here's the 'get_frame()' passed in the Response above
def get_frame():
camera_port = 0 # Assigns which webcame to detect if user has more than one webcam
camera = cv2.VideoCapture(camera_port) # Creates a camera object
first_frame = None # This variable will store the first image as a base image for comparing with images thereafter
# Starts the while loop to stream feed
while True:
status = 0
check, frame = camera.read()
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Converts to gray scale
gray = cv2.GaussianBlur(gray, (21, 21), 0)
....
....
imgencode = cv2.imencode('.jpg',frame)[1]
image_data = imgencode.tostring()
yield (b'--frame\r\n'
b'Content-Type: text/plain\r\n\r\n' + imageData + b'\r\n')

How to stop htmlpy ui blocking

I was working on a small downloader for Youtube videos using trollius (for async), htmlpy and pafy (and youtube-dl). Now i have a problem where once i click the download button, the UI freezes up while the backend is downloading the video.
I have tried to make the downloader class its own thread and run that alongside the UI but that doesn't seem to work. I also tried to uses coroutine to have the video download asynchronously while the UI continues. Neither seemed to work.
Here is some code
class Downloader(htmlPy.Object,threading.Thread):
dd = os.path.join(os.getenv('USERPROFILE'), 'Downloads') # dd = download director
dd = dd + "/vindownload/"
def __init__(self, app):
threading.Thread.__init__(self)
super(Downloader, self).__init__()
# Initialize the class here, if required.
self.app = app
return
#htmlPy.Slot(str)
def download_single(self, json_data):
form_data = json.loads(json_data)
print json_data
url = form_data["name"]
dt = form_data["dt"] # Download type is audio or video
if url.__contains__("https://www.youtube.com/watch?v="):
if dt == 'audio':
print "hello1"
loop = trollius.get_event_loop()
loop.run_until_complete(self._downloadVid(url, vid=False))
loop.stop()
else:
print "hello1"
loop = trollius.get_event_loop()
loop.run_until_complete(self._downloadVid(url, vid=True))
loop.stop()
self.app.evaluate_javascript("document.getElementById('form').reset()")
else:
print "Incorrect url"
print form_data
#trollius.coroutine
def _downloadVid(self, url, vid=False, order_reverse=False, vinName=None):
print "hello123"
video = pafy.new(url)
print video
name = u''.join(video.title).encode('utf8')
name = re.sub("[<>:\"/\\|?*]", "", name)
if not vid:
file = video.getbestaudio()
else:
file = video.getbest()
if (order_reverse):
file.download(self.dd + vinName + name + ".mp4", quiet=False,callback=self.mycb)
else:
file.download(self.dd + name + ".mp4", quiet=False,callback=self.mycb)
def mycb(self,total, recvd, ratio, rate, eta):
print(recvd, ratio, eta)
and my initialize.py
BASE_DIR = os.path.abspath(os.path.dirname("initilize.py"))
app = htmlPy.AppGUI(title=u"Vin download", width=700, height=400, resizable=False)
app.static_path = os.path.join(BASE_DIR, "static/")
app.template_path = os.path.join(BASE_DIR, "templates/")
app.web_app.setMaximumWidth(830)
app.web_app.setMaximumHeight(600)
download = Downloader(app)
download.start()
# Register back-end functionalities
app.bind(download)
app.template = ("./index.html", {"template_variable_name": "value"})
# Instructions for running application
if __name__ == "__main__":
# The driver file will have to be imported everywhere in back-end.
# So, always keep app.start() in if __name__ == "__main__" conditional
app.start()
Now my question is. Is there a way where i can free up my UI while downloading so it doesn't look like the application crashed.
I am using: Python 2.7, Trollius, Pafy, Youtube-dl, HTMLPY.
Thank you for your time.

Alright i found the answer to my question, here is what i did. i changed my download_single method to the following:
#htmlPy.Slot(str)
def download_single(self, json_data):
form_data = json.loads(json_data)
print json_data
url = form_data["name"]
dt = form_data["dt"] # Download type is audio or video
videoDown = videoDownload(url, dt, self.app, self.dd)
videoDown.start()
print form_data
And the code i had in it earlier is now transferred to a new class called videoDownload taking all those attributes stated above.
videoDownload.py
class videoDownload(threading.Thread):
def __init__(self, url, dt, app, dd):
threading.Thread.__init__(self)
self.url = url
self.dt = dt
self.app = app
self.dd = dd
def run(self):
threads.append(self)
if self.url.__contains__("https://www.youtube.com/watch?v="):
if self.dt == 'audio':
print "hello1"
self._downloadVid(self.url, vid=False)
else:
print "hello1"
self._downloadVid(self.url, vid=True)
else:
print "Incorrect url"
def _downloadVid(self, url, vid=False, order_reverse=False, vinName=None):
print "hello123"
video = pafy.new(url)
print video
name = u''.join(video.title).encode('utf8')
name = re.sub("[<>:\"/\\|?*]", "", name)
if not vid:
file = video.getbestaudio()
else:
file = video.getbest()
if (order_reverse):
file.download(self.dd + vinName + name + ".mp4", quiet=False, callback=self.mycb)
else:
file.download(self.dd + name + ".mp4", quiet=False, callback=self.mycb)
threads.remove(self)
def mycb(self, total, recvd, ratio, rate, eta):
pass
This solved the issues i had with ui being blocked. The thread will end itself once leaving the run method and it will be removed from a threads array that is defined above the class.
Have a great day all
~Ellisan