Develop Reference

Error when attempting to buffer video in opencv using multiple threads

I'm trying to read video into a buffer using threads in OpenCV and I get "Assertion fctx->async_lock failed at libavcodec/pthread_frame.c:167" the reason I want to use threads to do this is so I can read a lot of frames into a list at a time so it's fast. I need all the frames and can not skip any frames. I thought multi-threading would be the way to go. my code works single-threaded aka with "buffer_refill = 1".
import threading
import cv2
cap = cv2.VideoCapture('data/temp/testing/test.mp4')
frames = []
total_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)
frames_left = total_frames
threads = []
print(total_frames)
min_buffer = 120
buffer_refill = 60
def buffer():
print('buffer')
frame = cap.grab()
ret, frame = cap.retrieve()
frames.append(frame)
def get_buffer(num):
global frames_left
for i in range(num):
frames_left -= 1
if frames_left > 0:
t = threading.Thread(target=buffer)
t.start()
threads.append(t)
for thread in threads:
thread.join()
print('block')
while(cap.isOpened()):
if frames_left > 0 and len(frames) < min_buffer:
get_buffer(buffer_refill)
else:
cv2.imshow('Frame',frames[0])
frames.pop(0)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()

How to have a thread/process always active and request the updated value of one of its variables every certain time interval in another thread/process

import numpy as np
import cv2
import multiprocessing
import time
import random
finish_state = multiprocessing.Event()
#function that requests frames
def actions_func(frame):
while True:
time.sleep(random.randint(1,5))
cv2.imshow('requested_frame_1',frame)
time.sleep(random.randint(1,5))
cv2.imshow('requested_frame_2',frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
#function that keeps the camera always on and should return the frame value with the last image only when requested
def capture_cam():
cap = cv2.VideoCapture(1)
if (cap.isOpened() == False):
print("Unable to read camera feed")
# Default resolutions of the frame are obtained. The default resolutions are system dependent.
# We convert the resolutions from float to integer.
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
while(True):
ret, frame = cap.read()
if ret == True:
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
else:
break
def main_process(finish_state):
thr1, frame = multiprocessing.Process(target=capture_cam)
thr1.start()
thr2 = multiprocessing.Process(target=actions_func, args=(frame,))
thr2.start()
if __name__ == '__main__':
main_process(finish_state)
print("continue the code with other things after all threads/processes except the main one were closed with the loop that started them... ")
I want a webcam to be open all the time capturing an image, for this I have created thread1 where it is supposed to run all the time regardless of the program.
What you need is to fix this program that is supposed to ask for frames from the function that always runs on thread1.
The problem is that I don't know when it might be time to ask thread1 for the last frame it showed, and to represent that I put the random.randint(1,5), although in reality I won't have knowledge of the maximum or minimum time in which the last frame will be requested from thread1
The truth is that I'm getting complicated with this program, and I really don't know if it's convenient to create a thread2 to do the frame requests or if it's better to just have thread1 and have the frame requests inside the main thread
Although they say thread they are actually parallel processes, try with threads but I think it is more convenient to use processes, right?
Traceback (most recent call last):
File "request_frames_thread.py", line 58, in <module>
main_process(finish_state)
File "request_frames_thread.py", line 50, in main_process
thr1, frame = multiprocessing.Process(target=capture_cam)
TypeError: cannot unpack non-iterable Process object

I would have the main process create a full duplex multiprocessing.Pipe instance which returns two multiprocessing.connection.Connection instances and pass one connection to each of your processes. These connections can be used for a simple two way communication vehicle for sending and receiving objects to one another. I would have the capture_cam process start a dameon thread (it will terminate when all your regular threads terminate and so it can be in an infinite loop) that is passed on of these connections to handle requests for the latest frame, which is stored in a global variable.
The only requirement is that a frame be serializable by the pickle module.
import multiprocessing
from threading import Thread
import time
import random
#function that requests frames
def actions_func(conn):
try:
while True:
time.sleep(random.randint(1,5))
# Ask for latest frame by sending any message:
conn.send('frame')
frame = conn.recv() # This is the response
cv2.imshow('requested_frame_1',frame)
time.sleep(random.randint(1,5))
# Ask for latest frame by sending any message:
conn.send('frame')
frame = conn.recv() # This is the response
cv2.imshow('requested_frame_2',frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
except BrokenPipeError:
# The capture_cam process has terminated.
pass
def handle_frame_requests(conn):
try:
while True:
# Any message coming in is a request for the latest frame:
request = conn.recv()
conn.send(frame) # The frame must be pickle-able
except EOFError:
# The actions_func process has ended
# and its connection has been closed.
pass
#function that keeps the camera always on and should return the frame value with the last image only when requested
def capture_cam(conn):
global frame
frame = None
# start dameon thread to handle frame requests:
Thread(target=handle_frame_requests, args=(conn,), daemon=True).start()
cap = cv2.VideoCapture(1)
if (cap.isOpened() == False):
print("Unable to read camera feed")
# Default resolutions of the frame are obtained. The default resolutions are system dependent.
# We convert the resolutions from float to integer.
frame_width = int(cap.get(3))
frame_height = int(cap.get(4))
while(True):
ret, frame = cap.read()
if ret == True:
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
else:
break
def main_process(finish_state):
conn1, conn2 = multiprocessing.Pipe(duplex=True)
p1 = multiprocessing.Process(target=capture_cam, args=(conn1,))
p1.start()
p2 = multiprocessing.Process(target=actions_func, args=(conn2,))
p2.start()
if __name__ == '__main__':
finish_state = multiprocessing.Event()
main_process(finish_state)

How To Use Memory Mapped File in Python Linux

As I saw in the Python documentation,
https://docs.python.org/3/library/mmap.html.
Python in Linux can fully support the memory-mapped file. However, while I am trying to apply this idea to my application. I cannot run the sample.
My application is that sending the frames from Python file (client) to the another Python file (server).
Client Code
import mmap
import time
import os
import cv2 as cv
print("Opening camera...")
cap = cv.VideoCapture('/home/hunglv/Downloads/IMG_8442.MOV')
mm = None
try:
while True:
ret, img = cap.read()
if not ret:
break
if mm is None:
mm = mmap.mmap(-1,img.size,mmap.MAP_SHARED, mmap.PROT_WRITE)
# write image
start = time.time()
buf = img.tobytes()
mm.seek(0)
mm.write(buf)
mm.flush()
stop = time.time()
print("Writing Duration:", (stop - start) * 1000, "ms")
except KeyboardInterrupt:
pass
print("Closing resources")
cap.release()
mm.close()
Server Code
import mmap
import time
import os
import cv2 as cv
import numpy as np
shape = (1080, 1920, 3)
n = np.prod(shape)
mm = mmap.mmap(-1, n)
while True:
# read image
print (mm)
start = time.perf_counter()
mm.seek(0)
buf = mm.read(12)
img = np.frombuffer(buf, dtype=np.uint8).reshape(shape)
stop = time.perf_counter()
print("Reading Duration:", (stop - start) * 1000, "ms")
cv.imshow("img", img)
key = cv.waitKey(1) & 0xFF
key = chr(key)
if key.lower() == "q":
break
cv.destroyAllWindows()
mm.close()
On the server-side, I set the memory index at 0, and try to read the bytes from memory. However, it seems to be that the server cannot read correctly the data from client.
[Updated]
I've tried to read out the first 12 bytes at the server-side. The value is constant, not changing anymore.
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
Besides,
The first 12 bytes of a random frame is
b'\xf5\xff\xff\xf0\xfa\xfe\xdf\xe9\xed\xd2\xdc\xe0'

First I found example which probably works but it uses tagName (the same for client and server) and it means it is only for Window:
python-mmap-ipc
Next I found code which works on Linux:
Sharing Python data between processes using mmap.
It creates real file on disk, resizes it to size of image and then it uses its fd in mmap()
I use web camera for test.
Server
import mmap
import time
import os
import cv2
print("Opening camera...")
cap = cv2.VideoCapture(0)
#print(cap.get(cv.CAP_PROP_FRAME_WIDTH)) # 640
#print(cap.get(cv.CAP_PROP_FRAME_HEIGHT)) # 480
shape = (480, 640, 3)
n = (480*640*3)
fd = os.open('/tmp/mmaptest', os.O_CREAT | os.O_TRUNC | os.O_RDWR)
#os.write(fd, b'\x00' * n) # resize file
os.truncate(fd, n) # resize file
mm = None
try:
while True:
ret, img = cap.read()
if not ret:
break
if mm is None:
mm = mmap.mmap(fd, n, mmap.MAP_SHARED, mmap.PROT_WRITE) # it has to be only for writing
# write image
start = time.perf_counter()
buf = img.tobytes()
mm.seek(0)
mm.write(buf)
mm.flush()
stop = time.perf_counter()
print("Writing Duration:", (stop - start) * 1000, "ms")
except KeyboardInterrupt:
pass
print("Closing resources")
cap.release()
mm.close()
Client
import mmap
import time
import os
import cv2
import numpy as np
shape = (480, 640, 3)
n = (480*640*3)
fd = os.open('/tmp/mmaptest', os.O_RDONLY)
mm = mmap.mmap(fd, n, mmap.MAP_SHARED, mmap.PROT_READ) # it has to be only for reading
while True:
# read image
start = time.perf_counter()
mm.seek(0)
buf = mm.read(n)
img = np.frombuffer(buf, dtype=np.uint8).reshape(shape)
stop = time.perf_counter()
print("Reading Duration:", (stop - start) * 1000, "ms")
cv2.imshow("img", img)
key = cv2.waitKey(1) & 0xFF
key = chr(key)
if key.lower() == "q":
break
cv2.destroyAllWindows()
mm.close()
BTW: probably mmap() with -1 (without creating file on disk) could work with threads (or forks) because they share the same memory.

Low FPS using OpenCv with PiCamera (python)

I am trying to interface my OpenCV program with my Raspberry Pi PiCamera. Every time I use OpenCV to capture video, it drastically drops the FPS. When I capture video using PiCamera's Library, everything is fine and smooth.
Why is this happening?
Is there a way to fix it?
This is my code:
import time
import RPi.GPIO as GPIO
from PCA9685 import PCA9685
import numpy as np
import cv2
try:
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FPS, 90)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 800)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 700)
while(True):
ret, frame = cap.read()
cv2.imshow('frame',frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything is done, release the capture
except:
pwm.exit_PCA9685()
print ("\nProgram end")
exit()
cap.release()
cv2.destroyAllWindows()
This is the error I'm getting:

First of all, those are warnings not errors.
Reduce the video dimension. Specify the dimension.
cv2.VideoCapture has some problems as it buffers the frames, and the frames are queued so if you're doing some processing and the speed is less than the bandwidth of VideoCapture the video will be slowed down.
So, here is a bufferless VideoCapture.
video_capture_Q_buf.py
import cv2, queue as Queue, threading, time
is_frame = True
# bufferless VideoCapture
class VideoCaptureQ:
def __init__(self, name):
self.cap = cv2.VideoCapture(name)
self.q = Queue.Queue()
t = threading.Thread(target=self._reader)
t.daemon = True
t.start()
# read frames as soon as they are available, keeping only most recent one
def _reader(self):
while True:
ret, frame = self.cap.read()
if not ret:
global is_frame
is_frame = False
break
if not self.q.empty():
try:
self.q.get_nowait() # discard previous (unprocessed) frame
except Queue.Empty:
pass
self.q.put(frame)
def read(self):
return self.q.get()
Using it:
test.py
import video_capture_Q_buf as vid_cap_q # import as alias
from video_capture_Q_buf import VideoCaptureQ # class import
import time
cap = VideoCaptureQ(vid_path)
while True:
t1 = time.time()
if vid_cap_q.is_frame == False:
print('no more frames left')
break
try:
ori_frame = cap.read()
# do your stuff
except Exception as e:
print(e)
break
t2 = time.time()
print(f'FPS: {1/(t2-t1)}')

reading video frames in parallel with opencv and Python

I have to read frames from a video in parallel with multiprocessing & queues using opencv in Python and I'm getting an error with my code.
This is my code, and I don't know where my problem is.
#! /usr/bin/python
import numpy as np
import cv2
import multiprocessing as mp
import time
def read_frames(q1,q2):
while True :
frame = q1.get()
if frame=='Done':
break
R=frame[:,:,0]
G=frame[:,:,1]
B=frame[:,:,2]
y = (np.uint8)((0.299 * R)+ (0.587 *G) +( 0.114 *B))
q2.put(y)
if __name__ == '__main__':
q1 = mp.Queue()
q2 = mp.Queue()
processes =[mp.Process(target=read_frames, args= (q1,q2)) for i in rang$
for p in processes:
p.start()
# feed the processes
# read input file and send to the processes the frames:
cap = cv2.VideoCapture('gou.avi')
lines = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
cols = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
fps = int(cap.get(cv2.CAP_PROP_FPS))
fourcc_ver = int(cap.get(cv2.CAP_PROP_FOURCC))
out = cv2.VideoWriter('output.avi',fourcc_ver, fps, (cols,lines),False)
# y = np.empty(shape=(lines,cols),dtype=np.uint8)
while(cap.isOpened()):
ret, frame = cap.read()
# as long as new frames are there
if ret==True:
q1.put(frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
break
q1.put('Done')
for p in processes:
p.join()
for p in processes:
result=[q2.get()]
# result.sort()
# results = [r[1] for r in results]
for i in result:
out.write(i)
# Release everything if job is finished
cap.release()
out.release()
cv2.destroyAllWindows()

What you could do is add all the frames to a list[] and then use list references such as frame = list[] and then address frames by location on the list: frame[0] or frame[1:4]