My project uses socket.io to send/receive data.
I added aiohttp to help display the results on the browser.
import asyncio
from aiohttp import web
sio = socketio.AsyncServer(async_mode='`aiohttp`')
app = web.Application()
sio.attach(app)
I followed
https://us-pycon-2019-tutorial.readthedocs.io/aiohttp_file_uploading.html
to upload an image but I cannot upload a video.
def gen1():
# while True:
# if len(pm.list_image_display) > 1 :
image = cv2.imread("/home/duong/Pictures/Chess_Board.svg")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# img = PIL.Image.new("RGB", (64, 64), color=(255,255,0))
image_pil = PIL.Image.fromarray(image)
fp = io.BytesIO()
image_pil.save(fp, format="JPEG")
content = fp.getvalue()
return content
async def send1():
print("11")
return web.Response(body=gen1(), content_type='image/jpeg')
How to display video via aiohttp on browsers?
To stream a video in aiohttp you may open a StreamResponse in response to the fetching of a img HTML node:
#routes.get('/video')
async def video_feed(request):
response = web.StreamResponse()
response.content_type = 'multipart/x-mixed-replace; boundary=frame'
await response.prepare(request)
for frame in frames('/dev/video0'):
await response.write(frame)
return response
and send your frames in the form of bytes:
def frames(path):
camera = cv2.VideoCapture(path)
if not camera.isOpened():
raise RuntimeError('Cannot open camera')
while True:
_, img = camera.read()
img = cv2.resize(img, (480, 320))
frame = cv2.imencode('.jpg', img)[1].tobytes()
yield b'--frame\r\nContent-Type: image/jpeg\r\n\r\n'+frame+b'\r\n'
This may be however network demanding as the bitrate required to send each frame individually is high. For real-time streaming with further compression you may want to use WebRTC implementations like aiortc.
Related
from flask import Flask, Response, request, send_file
from moviepy.editor import VideoFileClip
import socket
import cv2 as cv2
app = Flask(__name__)
video_path = "videos/video.avi"
#app.route('/video_feed/')
def video_feed():
start_frame = int(request.args.get("start_frame"))
end_frame = int(request.args.get("end_frame"))
return Response(gen(start_frame, end_frame), mimetype='multipart/x-mixed-replace; boundary=frame')
def gen(start_frame, end_frame):
cap = cv2.VideoCapture(video_path)
cap.set(1, start_frame)
while True:
success, img = cap.read()
current_frame = cap.get(cv2.CAP_PROP_POS_FRAMES)
if current_frame > end_frame:
break
imgencode = cv2.imencode('.jpg', img)[1]
stringData = imgencode.tobytes()
# can also use tostring()
yield (b'--frame\r\n'
b'Content-Type: text/plain\r\n\r\n' + stringData + b'\r\n')
if __name__ == '__main__':
app.run(host='0.0.0.0', port=8000, debug=True)
So, this is the Flask server I am running. I am able to view the correct frames being yielded by the flask server at this URL: (It doesn't matter that the video playback is too fast)
http://127.0.0.1:8000/video_feed/?start_frame=0&end_frame=5000
But I can't seem to figure out how to use this URL into a development environment like PyCharm to be able to read the frames from this URL into a python script, for example.
url="http://127.0.0.1:8000/video_feed/?start_frame=4000&end_frame=5001"
while True:
resp = urllib.request.urlopen(url)
response = resp.read()
data = resp.split(b'\r\n\r\n', 1)[1]
i = np.frombuffer(data, dtype=np.uint8)
img = cv2.imdecode(i, cv2.IMREAD_UNCHANGED)
cv2.imshow("frame", img)
if cv2.waitKey(16) & 0xFF==ord('q'):
break
cv2.destroyAllWindows()
So this is what I have tried for reading the frames into PyCharm but it only reads the first frame. I want it to be able to ingest all of the frames from the URL. I know that there is something I am not understanding when it comes to URL's or generator functions, so any refactoring or help is greatly appreciated!
On the Flask server side the adjustment is made to the generator function:
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + stringData + b'\r\n')
And then read into an IDE like pycharm as such:
import cv2
vcap = cv2.VideoCapture('http://127.0.0.1:8000/video_feed/?
start_frame=4000&end_frame=5001')
while True:
ret, frame = vcap.read()
if frame is not None:
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
else:
print("Frame is None")
break
vcap.release()
cv2.destroyAllWindows()
print("Video stop")
I want to get the size of the image from a buffer.
import io
from PIL import Image
class Scraper:
asnyc _save_image(res):
buffer = await res.body()
img = Image.open(io.BytesIO(buffer))
img_w, img_h = img.size
async def scrape():
playwright = await async_playwright().start()
browser = await playwright.chromium.launch( headless = True, devtools = False )
page = browser.new_page()
page.on('response', _save_image)
await page.goto('https://www.example.com')
scraper = Scraper(key, id)
asyncio.run(scraper.scrape())
img = Image.open(io.BytesIO(buffer))
This code above is not an async function. I want know the size of images from the buffer asynchronously. How to so this?
I have a flask web application which reads one image and display it in my web browser
app.py
from flask import Response
from flask import Flask
from flask import render_template
import cv2
app = Flask(__name__)
#app.route("/")
def index():
return render_template("index.html")
def GetImage():
global img
while True:
yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(img) + b'\r\n')
#app.route("/stream")
def stream():
return Response(GetImage(), mimetype = "multipart/x-mixed-replace; boundary=frame")
if(__name__ == "__main__"):
img = cv2.imread("Cat.jpg", 0)
app.run(debug = True, threaded = True, use_reloader = False)
index.html
<html>
<body>
<img src="{{ url_for('stream') }}">
</body>
</html>
This example doesn´t work, because the image isn´t displayed in the browser.
But when I change GetImage in the following way the image will be displayed in the browser:
def GetImage():
global img
(flag, encodedImage) = cv2.imencode(".jpg", img)
while True:
yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + bytearray(encodedImage) + b'\r\n')
So why do I need this imencode? The image is stored as jpg on my harddisk so why do I have to encode it again as JPG?
If you have JPEG file then you can use standard open() and read() to read it as raw bytes data without decompressing it to array with all pixels - so later you don't have to compress it back to JPEG data using imencode()
img = open("Cat.jpg", "rb").read()
and then you can display it
b'Content-Type: image/jpeg\r\n\r\n'+ img + b'\r\n'
I read it directly in bytes mode - open(..., 'rb') - so I don't have to use bytearray() to convert string to bytes. Besides, reading in text mode it could convert some chars (like "new line") and create incorrect data.
But to send single file you can use send_file()
#app.route("/image")
def image():
return send_file('Cat.jpg')
Working example.
Opening http://127.0.0.1:5000/stream in Chrome it shows image.
My Firefox had problem to display image - it was reading data all time - till I added time.sleep()
I also added version which adds header Content-Length:
from flask import Flask, Response, send_file
import time
app = Flask(__name__)
#app.route("/")
def index():
return "Hello World"
def get_image():
while True:
yield(b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n'+ img + b'\r\n')
time.sleep(0.01) # my Firefox needs some time to display image / Chrome displays image without it
def get_image_with_size():
length = str(len(img)).encode() # convert to bytes
while True:
yield(b'--frame\r\n'
b'Content-Type: image/jpeg\r\n'
b'Content-Length: ' + length + b'\r\n'
b'\r\n'+ img + b'\r\n')
time.sleep(0.01) # my Firefox needs some time to display image / Chrome displays image without it
#app.route("/stream")
def stream():
return Response(get_image(), mimetype="multipart/x-mixed-replace; boundary=frame")
#app.route("/image")
def image():
return send_file('Cat.jpg')
if(__name__ == "__main__"):
img = open('Cat.jpg', 'rb').read()
app.run()
I'm using opencv to capture a video from my webcam. Every 5 seconds, I'm processing a single frame / an image which can take some seconds. So far everything works. But whenever a frame is processed the entire video is freezing for a couple of seconds (Until the process is finished). I'm trying to get rid of it by using Threading. Here is what I did so far:
Inside the while loop which is capturing the video:
while True:
ret, image = cap.read()
if next_time <= datetime.now():
content_type = 'image/jpeg'
headers = {'content-type': content_type}
_, img_encoded = cv2.imencode('.jpg', image)
loop = asyncio.get_event_loop()
future = asyncio.ensure_future(self.async_faces(img_encoded, headers))
loop.run_until_complete(future)
next_time += period
...
cv2.imshow('img', image)
Here are the methods:
async def async_faces(self, img, headers):
with ThreadPoolExecutor(max_workers=10) as executor:
loop = asyncio.get_event_loop()
tasks = [
loop.run_in_executor(
executor,
self.face_detection,
*(img, headers) # Allows us to pass in multiple arguments to `fetch`
)
]
for response in await asyncio.gather(*tasks):
pass
def face_detection(self, img, headers):
try:
response = requests.post(self.url, data=img.tostring(), headers=headers)
...
except Exception as e:
...
...
But unfortunately it's not working.
EDIT 1
In the following I add what the whole thing is supposed to do.
Originally, the function looked like:
import requests
import cv2
from datetime import datetime, timedelta
def face_recognition(self):
# Start camera
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)
emotional_states = []
font = cv2.FONT_HERSHEY_SIMPLEX
period = timedelta(seconds=self.time_period)
next_time = datetime.now() + period
cv2.namedWindow('img', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('img', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
while True:
ret, image = cap.read()
if next_time <= datetime.now():
# Prepare headers for http request
content_type = 'image/jpeg'
headers = {'content-type': content_type}
_, img_encoded = cv2.imencode('.jpg', image)
try:
# Send http request with image and receive response
response = requests.post(self.url, data=img_encoded.tostring(), headers=headers)
emotional_states = response.json().get("emotions")
face_locations = response.json().get("locations")
except Exception as e:
emotional_states = []
face_locations = []
print(e)
next_time += period
for i in range(0, len(emotional_states)):
emotion = emotional_states[i]
face_location = face_locations[i]
cv2.putText(image, emotion, (int(face_location[0]), int(face_location[1])),
font, 0.8, (0, 255, 0), 2, cv2.LINE_AA)
cv2.imshow('img', image)
k = cv2.waitKey(1) & 0xff
if k == 27:
cv2.destroyAllWindows()
cap.release()
break
if k == ord('a'):
cv2.resizeWindow('img', 700,700)
I use the above method to film myself. This film will be shown live on my screen. Further, every 5 seconds one frame is send to an API where the image is processed in such a way that the emotion of the person in the image is returned. This emotion is displayed on my screen, next to myself. The problem is, that the live video is freezing for a couple of seconds until the emotion is returned from the API.
My OS is Ubuntu.
EDIT 2
The API is running locally. I created a Flask App and the following method is receiving the request:
from flask import Flask, request, Response
import numpy as np
import cv2
import json
#app.route('/api', methods=['POST'])
def facial_emotion_recognition():
# Convert string of image data to uint8
nparr = np.fromstring(request.data, np.uint8)
# Decode image
img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
# Analyse the image
emotional_state, face_locations = emotionDetection.analyze_facial_emotions(img)
json_dump = json.dumps({'emotions': emotional_state, 'locations': face_locations}, cls=NumpyEncoder)
return Response(json_dump, mimetype='application/json')
I'm trying to capture two rtsp streams with opencv and then present them with a simple flask server. I can show the two streams together when just using opencv, but when I try to display it through flask it just picks either stream and shows it twice.
Here's the original creators blog
Here is my flask code:
#!/usr/bin/env python
from importlib import import_module
import os
from flask import Flask, render_template, Response
# import camera driver
'''
if os.environ.get('CAMERA'):
Camera = import_module('camera_' + os.environ['CAMERA']).Camera
else:
from camera import Camera
'''
#
from camera_opencv import Camera1, Camera2
app = Flask(__name__)
#app.route('/')
def index():
"""Video streaming home page."""
return render_template('index.html')
def gen(camera):
"""Video streaming generator function."""
while True:
frame = camera.get_frame()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
#app.route('/video_feed1')
def video_feed1():
"""Video streaming route. Put this in the src attribute of an img tag."""
return Response(gen(Camera1()),
mimetype='multipart/x-mixed-replace; boundary=frame')
#app.route('/video_feed2')
def video_feed2():
"""Video streaming route. Put this in the src attribute of an img tag."""
return Response(gen(Camera2()),
mimetype='multipart/x-mixed-replace; boundary=frame')
if __name__ == '__main__':
app.run(host='0.0.0.0', threaded=True, port=8888)
Here's the camera_opencv file
import cv2
from base_camera import BaseCamera
class Camera1(BaseCamera):
video_source = 0
#staticmethod
def set_video_source(source):
Camera1.video_source = source
#staticmethod
def frames():
camera = cv2.VideoCapture(Camera1.video_source)
if not camera.isOpened():
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img = camera.read()
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()
class Camera2(BaseCamera):
video_source = 1
#staticmethod
def set_video_source(source):
Camera2.video_source = source
#staticmethod
def frames():
camera = cv2.VideoCapture(Camera2.video_source)
if not camera.isOpened():
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img = camera.read()
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()
Base camera file
import time
import threading
try:
from greenlet import getcurrent as get_ident
except ImportError:
try:
from thread import get_ident
except ImportError:
from _thread import get_ident
class CameraEvent(object):
"""An Event-like class that signals all active clients when a new frame is
available.
"""
def __init__(self):
self.events = {}
def wait(self):
"""Invoked from each client's thread to wait for the next frame."""
ident = get_ident()
if ident not in self.events:
# this is a new client
# add an entry for it in the self.events dict
# each entry has two elements, a threading.Event() and a timestamp
self.events[ident] = [threading.Event(), time.time()]
return self.events[ident][0].wait()
def set(self):
"""Invoked by the camera thread when a new frame is available."""
now = time.time()
remove = None
for ident, event in self.events.items():
if not event[0].isSet():
# if this client's event is not set, then set it
# also update the last set timestamp to now
event[0].set()
event[1] = now
else:
# if the client's event is already set, it means the client
# did not process a previous frame
# if the event stays set for more than 5 seconds, then assume
# the client is gone and remove it
if now - event[1] > 5:
remove = ident
if remove:
del self.events[remove]
def clear(self):
"""Invoked from each client's thread after a frame was processed."""
self.events[get_ident()][0].clear()
class BaseCamera(object):
thread = None # background thread that reads frames from camera
frame = None # current frame is stored here by background thread
last_access = 0 # time of last client access to the camera
event = CameraEvent()
def __init__(self):
"""Start the background camera thread if it isn't running yet."""
if BaseCamera.thread is None:
BaseCamera.last_access = time.time()
# start background frame thread
BaseCamera.thread = threading.Thread(target=self._thread)
BaseCamera.thread.start()
# wait until frames are available
while self.get_frame() is None:
time.sleep(0)
def get_frame(self):
"""Return the current camera frame."""
BaseCamera.last_access = time.time()
# wait for a signal from the camera thread
BaseCamera.event.wait()
BaseCamera.event.clear()
return BaseCamera.frame
#staticmethod
def frames():
""""Generator that returns frames from the camera."""
raise RuntimeError('Must be implemented by subclasses.')
#classmethod
def _thread(cls):
"""Camera background thread."""
print('Starting camera thread.')
frames_iterator = cls.frames()
for frame in frames_iterator:
BaseCamera.frame = frame
BaseCamera.event.set() # send signal to clients
time.sleep(0)
# if there hasn't been any clients asking for frames in
# the last 10 seconds then stop the thread
if time.time() - BaseCamera.last_access > 10:
frames_iterator.close()
print('Stopping camera thread due to inactivity.')
break
BaseCamera.thread = None
Index.html
<html>
<head>
<title>Video Streaming Demonstration</title>
</head>
<body>
<h1>Video Streaming Demonstration</h1>
<img src="{{ url_for('video_feed1') }}">
<img src="{{ url_for('video_feed2') }}">
</body>
</html>
So I kind of managed to make a hackey workaround. For whatever reason I could not resolve, the app just couldn't handle multiple streams individually.
So I changed the camera class and added multiple sources to it and used numpy.hstack(()) to merge both the frames together thus returning one unique stream.
Will be very grateful if someone could help out here as my method is not at all scalable.
import cv2
from base_camera import BaseCamera
import numpy as np
class Camera(BaseCamera):
video_source1 = 0
video_source2 = 1
#staticmethod
def set_video_source(sources):
Camera.video_source1 = sources[0]
Camera.video_source2 = sources[1]
#staticmethod
def frames():
camera1 = cv2.VideoCapture(Camera.video_source1)
camera2 = cv2.VideoCapture(Camera.video_source2)
if not (camera1.isOpened() or camera2.isOpened()):
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img1 = camera1.read()
_, img2 = camera2.read()
img1 = cv2.resize(img1, (704, 396))
img2 = cv2.resize(img2, (704, 396))
img = np.hstack((img1, img2))
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()
According to the blog, I use two generator and send the image to the index.html. And I can see tow streaming.
def generate2():
# it is a generator
global outputFrame2, lock
while True:
with lock:
if outputFrame is None:
continue
(flag, encodedImage2) = cv2.imencode(".jpg", outputFrame2)
if not flag:
continue
yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' +
bytearray(encodedImage2) + b'\r\n')
And here is videofeed2
#app.route("/video_feed2")
def video_feed2():
return Response(generate2(),
mimetype = "multipart/x-mixed-replace; boundary=frame")