Is it possible in cross platform to create an rtmp connection with ffmpeg and pyaudio?
This the code:
Camera
import sys, pyaudio, subprocess, cv2
rtmp_url = "rtmp://MYURL"
cap = cv2.VideoCapture(0)
fps = int(cap.get(cv2.CAP_PROP_FPS))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
Command ffmpeg
command = ['']
Init the microphone
CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1 if sys.platform == 'darwin' else 2
RATE = 44100
RECORD_SECONDS = 20
PyA = pyaudio.PyAudio()
stream = PyA.open(format=FORMAT, channels=CHANNELS, rate=RATE, input=True)
subProcess = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
Write the bytes
while stream.is_active():
ret, frame = cap.read()
if not ret:
print("frame read failed")
break
subProcess.stdin.write(frame.tobytes())
subProcess.stdin.write(stream.read(CHUNK))
subProcess.stdin.close()
All in multiplatform.
Related
When I try updating a string variable in self.graphWidget.setTitle(phrase, ...) the variable doesn't update.
I'm plotting a real time waveform graph while recording from mic using pyaudio and PyQT, what I want to do is instead of printing in terminal I want the speech to text phrase to be shown in MainWindow after recognition is done
'''GUI'''
import struct
from PyQt5 import QtWidgets
from PyQt5.QtWidgets import QApplication
import sys
'''Graph'''
import pyqtgraph as pg
from PyQt5 import QtCore
import numpy as np
'''Audio Processing'''
import pyaudio
import wave
import speech_recognition as sr
import multiprocessing as mlti
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
CHUNK = 1024 * 2
p = pyaudio.PyAudio()
stream = p.open(
format=FORMAT,
channels=CHANNELS,
rate=RATE,
input=True,
output=True,
frames_per_buffer=CHUNK,
)
frames = []
seconds = 6
phrase = "..."
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
pg.setConfigOptions(antialias=True)
self.traces = dict()
'''Display'''
self.graphWidget = pg.PlotWidget()
self.setCentralWidget(self.graphWidget)
self.setWindowTitle("Waveform")
self.setGeometry(55, 115, 970, 449)
'''Data'''
self.x = np.arange(0, 2 * CHUNK, 2)
self.f = np.linspace(0, RATE // 2, CHUNK // 2)
'''Animate'''
self.timer = QtCore.QTimer()
self.timer.setInterval(50)
self.timer.timeout.connect(self.update)
self.timer.start()
def set_plotdata(self, name, data_x, data_y):
if name in self.traces:
self.traces[name].setData(data_x, data_y)
else:
if name == 'waveform':
self.traces[name] = self.graphWidget.plot(pen='c', width=3)
self.graphWidget.setYRange(0, 255, padding=0)
self.graphWidget.setXRange(0, 2 * CHUNK, padding=0.005)
def update(self):
self.wf_data = stream.read(CHUNK)
self.wf_data = struct.unpack(str(2 * CHUNK) + 'B', self.wf_data)
self.wf_data = np.array(self.wf_data, dtype='b')[::2] + 128
self.set_plotdata(name='waveform', data_x=self.x, data_y=self.wf_data)
self.graphWidget.setTitle(phrase, color="w", size="30pt") #### Change it
def main():
app = QtWidgets.QApplication(sys.argv)
win = MainWindow()
win.show()
sys.exit(app.exec_())
def Record():
for i in range(0, int(RATE/CHUNK*seconds)):
data = stream.read(CHUNK)
frames.append(data)
print(i)
def Refine_Stream():
stream.stop_stream()
stream.close()
p.terminate()
obj = wave.open("output.wav", "wb")
obj.setnchannels(CHANNELS)
obj.setsampwidth(p.get_sample_size(FORMAT))
obj.setframerate(RATE)
obj.writeframes(b"".join(frames))
obj.close()
def Speech_Recog():
print("Function Started")
r = sr.Recognizer()
#usando o microfone
with sr.AudioFile("output.wav") as source:
r.adjust_for_ambient_noise(source, duration=1)
#Armazena o que foi dito numa variavel
audio = r.listen(source)
phrase = ""
try:
#Into recog
phrase = r.recognize_google(audio,language='pt-BR') #### To it
print(phrase)
#Couldn't make it
except sr.UnknownValueError:
phrase = "Not understood"
print(phrase)
if __name__ == '__main__':
p1 = mlti.Process(target=main)
p1.start()
Record()
Refine_Stream()
Speech_Recog()
Hope it was clean code enough
I studied quite a lot and I figured out how to do exactly what I wanted
I'm using a signal and slot system as my_signal = QtCore.pyqtSignal(str) to send my string-variable as a signal to setTitle that is activated using QTimer after some time
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
'''Display'''
self.graphWidget = pg.PlotWidget()
self.setCentralWidget(self.graphWidget)
self.setWindowTitle("Waveform")
self.setGeometry(55, 115, 970, 449)
self.graphWidget.setTitle(phrase, color="w", size="30pt")
'''Animate Title'''
self.picktimer = QTimer()
self.picktimer.setInterval(7000) ###seconds needs to be more than seconds recording
self.picktimer.timeout.connect(self.Transitioning)
self.picktimer.start()
def Transitioning(self):
self.example_class = Speech_Recognition(self)
self.example_class.my_signal.connect(self.Update_Title)
self.example_class.Speech_Recog() ###init SpeechRecog
def Update_Title(self, my_title_phrase):
self.graphWidget.setTitle(my_title_phrase, color="w", size="30pt")
QtWidgets.qApp.processEvents()
def main():
app = QtWidgets.QApplication(sys.argv)
win = MainWindow()
win.show()
sys.exit(app.exec_())
def Record():
for i in range(0, int(RATE/CHUNK*seconds)):
data = stream.read(CHUNK)
frames.append(data)
print(i)
def Refine_Stream():
stream.stop_stream()
stream.close()
p.terminate()
obj = wave.open("output.wav", "wb")
obj.setnchannels(CHANNELS)
obj.setsampwidth(p.get_sample_size(FORMAT))
obj.setframerate(RATE)
obj.writeframes(b"".join(frames))
obj.close()
class Speech_Recognition(QtWidgets.QWidget):
my_signal = QtCore.pyqtSignal(str)
def Speech_Recog(self):
r = sr.Recognizer()
recorded_phrase = ""
#usando o microfone
with sr.AudioFile("output.wav") as source:
r.adjust_for_ambient_noise(source, duration=1)
#Armazena o que foi dito numa variavel
audio = r.listen(source)
try:
recorded_phrase = r.recognize_google(audio,language='pt-BR')
except sr.UnknownValueError:
recorded_phrase = "Not understood"
self.my_signal.emit(recorded_phrase)
if __name__ == '__main__':
p2 = mlti.Process(target=main)
p2.start()
Record()
Refine_Stream()
Still needs to be clean code though
i have try to do this from thread to an imagebox of gtk . The frames are showing successfull for some times but after 5 seconds the gui freezes and the image appears to be blank , but the thread is still running
import gi
import threading
import cv2
import time
gi.require_version("Gtk" , "3.0")
from gi.repository import Gtk , GLib , GObject , GdkPixbuf
run = True
cam = cv2.VideoCapture(0)
ret , frame = cam.read()
frame = cv2.cvtColor(frame , cv2.COLOR_BGR2RGB)
cam.release()
class MyWindow(Gtk.Window):
def __init__(self):
global cam, frame , run
Gtk.Window.__init__(self,title="Hello")
self.set_border_width(10)
image = Gtk.Image()
h, w, d = frame.shape
pixbuf = GdkPixbuf.Pixbuf.new_from_data(frame.tostring(),GdkPixbuf.Colorspace.RGB, False, 8, w, h, w*d)
image.set_from_pixbuf (pixbuf.copy())
self.add(image)
def thread_running_example():
cam = cv2.VideoCapture(0)
while run:
ret , frame = cam.read()
frame = cv2.cvtColor(frame , cv2.COLOR_BGR2RGB)
h, w, d = frame.shape
pixbuf = GdkPixbuf.Pixbuf.new_from_data(frame.tostring(), GdkPixbuf.Colorspace.RGB, False, 8, w, h, w*d)
image.set_from_pixbuf (pixbuf.copy())
thread = threading.Thread( target = thread_running_example )
thread.daemon=True
thread.start()
win = MyWindow()
win.connect("destroy" , Gtk.main_quit )
win.show_all()
Gtk.main()
I'm trying to use tensorflow lite in raspberry pi to detect specific category (motorcycle only) using the pre-trained model. Since the motorcycle category is already existing in the pre-trained model, I assume that I don't need any to retrain it. Is there anyway to remove other objects in the model? I am using the code from Edje Electronics provided by this link: https://github.com/EdjeElectronics/TensorFlow-Lite-Object-Detection-on-Android-and-Raspberry-Pi
'''
# Import packages
import os
import argparse
import cv2
import numpy as np
import sys
import time
from threading import Thread
import importlib.util
from utils import label_map_util
from utils import visualization_utils as vis_util
# Define VideoStream class to handle streaming of video from webcam in separate processing thread
class VideoStream:
"""Camera object that controls video streaming from the Picamera"""
def __init__(self,resolution=(640,480),framerate=30):
# Initialize the PiCamera and the camera image stream
self.stream = cv2.VideoCapture(0)
ret = self.stream.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
ret = self.stream.set(3,resolution[0])
ret = self.stream.set(4,resolution[1])
# Read first frame from the stream
(self.grabbed, self.frame) = self.stream.read()
# Variable to control when the camera is stopped
self.stopped = False
def start(self):
# Start the thread that reads frames from the video stream
Thread(target=self.update,args=()).start()
return self
def update(self):
# Keep looping indefinitely until the thread is stopped
while True:
# If the camera is stopped, stop the thread
if self.stopped:
# Close camera resources
self.stream.release()
return
# Otherwise, grab the next frame from the stream
(self.grabbed, self.frame) = self.stream.read()
def read(self):
# Return the most recent frame
return self.frame
def stop(self):
# Indicate that the camera and thread should be stopped
self.stopped = True
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir', help='Folder the .tflite file is located in',
required=True)
parser.add_argument('--graph', help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument('--labels', help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument('--resolution', help='Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='1280x720')
parser.add_argument('--edgetpu', help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
# Initialize video stream
videostream = VideoStream(resolution=(imW,imH),framerate=30).start()
time.sleep(1)
#for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
while True:
# Start timer (for calculating frame rate)
t1 = cv2.getTickCount()
# Grab frame from video stream
frame1 = videostream.read()
# Acquire frame and resize to expected shape [1xHxWx3]
frame = frame1.copy()
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (width, height))
input_data = np.expand_dims(frame_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'],input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects
scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1,(boxes[i][0] * imH)))
xmin = int(max(1,(boxes[i][1] * imW)))
ymax = int(min(imH,(boxes[i][2] * imH)))
xmax = int(min(imW,(boxes[i][3] * imW)))
cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)
# Draw label
object_name = labels[int(classes[i])] # Look up object name from "labels" array using class index
label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
# Draw framerate in corner of frame
cv2.putText(frame,'FPS: {0:.2f}'.format(frame_rate_calc),(30,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,0),2,cv2.LINE_AA)
# All the results have been drawn on the frame, so it's time to display it.
cv2.imshow('Object detector', frame)
# Calculate framerate
t2 = cv2.getTickCount()
time1 = (t2-t1)/freq
frame_rate_calc= 1/time1
# Press 'q' to quit
if cv2.waitKey(1) == ord('q'):
break
# Clean up
cv2.destroyAllWindows()
videostream.stop()
'''
I've solved my own problem by adding these lines of code inside the for loop.
for i in range(len(scores)):
if classes[i] != 3:
scores[i]=0
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
NOTE: number 3 represents the motorcycle label in the labelmap
I can't synchronize the display of two ip cameras.
I am a newbie, I can't think of a better way.
import cv2
ip_camera_url_1 = 'rtsp://192.168.xx.xx:xx/xx'
ip_camera_url_2 = 'rtsp://192.168.xx.xx:xx/xx'
cap1 = cv2.VideoCapture(ip_camera_url_1)
cap2 = cv2.VideoCapture(ip_camera_url_2)
while True:
ret1, frame_left = cap1.read()
ret2, frame_right = cap2.read()
cv2.imshow('left', frame_left)
cv2.imshow('right', frame_right)
cv2.waitKey(1)
I created a python coding that apply HPF to the input audio in real-time. It works just fine.
import pyaudio
import time
import numpy as np
from scipy.signal import firwin, firwin2, lfilter, freqz
WIDTH = 2
CHANNELS = 2
RATE = 44100
p = pyaudio.PyAudio()
f1 = 500/1000
def bytes_to_float(byte_array):
int_array = np.frombuffer(byte_array, dtype=np.float32)
return int_array
def float_to_bytes(float_array):
int_array = float_array.astype(np.float32)
return int_array.tostring()
def callback(in_data, frame_count, time_info, flag):
signal = bytes_to_float(in_data)
a = [1]
b = firwin(129, f1, pass_zero=False)
filtered= lfilter(b, a, signal)
output = float_to_bytes(filtered)
return (output, pyaudio.paContinue)
stream = p.open(format=pyaudio.paFloat32,
channels=CHANNELS,
rate=RATE,
output=True,
input=True,
stream_callback=callback)
stream.start_stream()
while stream.is_active():
time.sleep(0.1)
stream.stop_stream()
stream.close()
p.terminate()
Now, I wanted to display the frequency spectrum of filtered audio. So, I edited the python coding above.
from pyqtgraph.Qt import QtGui, QtCore
import numpy as np
import time
import pyqtgraph as pg
import pyaudio
from scipy.signal import firwin, firwin2, lfilter, freqz
app = QtGui.QApplication([])
win = pg.GraphicsWindow(title="Basic plotting examples")
win.resize(1000,600)
win.setWindowTitle('pyqtgraph example: Plotting')
pg.setConfigOptions(antialias=True)
p6 = win.addPlot(title="Updating plot")
p6.setYRange(-100, 20)
curve = p6.plot(pen='y')
chunks = 1024
freq = np.linspace(1e-6, (chunks / 2), chunks, endpoint=True)
def bytes_to_float(byte_array):
int_array = np.frombuffer(byte_array, dtype=np.float32)
return int_array
def float_to_bytes(float_array):
int_array = float_array.astype(np.float32)
return int_array.tostring()
def update_freq_plot(x):
global curve, freq
h = 20 * np.log10(np.abs(np.fft.rfft(x, n=2 * chunks)[:chunks]))
curve.setData(x=freq, y=h, clear=True)
def update():
WIDTH = 2
CHANNELS = 2
RATE = 44100
p = pyaudio.PyAudio()
f1 = 500/1000
def callback(in_data, frame_count, time_info, flag):
signal = bytes_to_float(in_data)
a = [1]
b = firwin(129, f1, pass_zero=False)
z = np.zeros(129 - 1)
filtered, z = lfilter(b, a, signal, zi=z)
update_freq_plot(filtered)
output = float_to_bytes(filtered)
return (output, pyaudio.paContinue)
stream = p.open(format=pyaudio.paFloat32,
channels=CHANNELS,
rate=RATE,
output=True,
input=True,
stream_callback=callback)
stream.start_stream()
timer = QtCore.QTimer()
timer.timeout.connect(update)
timer.start(20)
## Start Qt event loop unless running in interactive mode or using pyside.
if __name__ == '__main__':
import sys
if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
QtGui.QApplication.instance().exec_()
I used this as a reference. When I run the coding, it does display the spectrum, but it is lagging. I'm using jupyter to run the coding.
I would like to ask how to improve my coding so that the lagging issue won't appear? Any help is much appreciated. Thank you.