Develop Reference

Parallel downloading don't work in python threading

I'm building a parallel download library using threading module.
When I use my library, it downloads the file without error, but the video file doesn't have the same content as if I downloaded it through the browser.
I use threading for parallel downloading and I think I have a problem with threading.Lock and file.seek, but I can't figure out how to fix the problem.
This is my code:
import requests
import threading
from tqdm import tqdm
DOWNLOAD_CHUNK_SIZE = 1 << 20 # 1 MiB
class DownloadPart:
def __init__(self, url, byte_range) -> None:
self.url = url
self.byte_range = byte_range
self.lock = threading.Lock()
def download(self, file, pbar=None):
response = requests.get(
self.url,
headers={"Range": "bytes={}-{}".format(*self.byte_range)},
allow_redirects=True,
stream=True,
)
written = 0
for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
if chunk:
self.lock.acquire()
file.seek(self.byte_range[0] + written)
length = file.write(chunk)
file.flush()
written += length
pbar.update(length)
self.lock.release()
class Downloader:
def __init__(self, url, parts=10):
self.url = url
self.parts = parts
def _get_file_size(self) -> int:
info = requests.head(self.url, allow_redirects=True)
info.raise_for_status()
size = info.headers.get("content-length", None)
assert size
return int(size)
def download(self, filename):
file_size = self._get_file_size()
# file_size = 1024
size_per_part = file_size // self.parts
print(file_size, size_per_part)
file = open(filename, "wb")
pbar = tqdm(total=file_size)
threads = []
for index in range(self.parts):
# fix last part have more bytes
if index + 1 == self.parts:
byte_range = (size_per_part * index, file_size - 1)
else:
byte_range = (size_per_part * index, size_per_part * (index + 1) - 1)
thread = threading.Thread(
target=DownloadPart(self.url, byte_range).download, args=(file,), kwargs={"pbar": pbar}
)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
file.close()
URL = "https://s-delivery38.mxdcontent.net/v/8a5f59673042ed97c402be84ceeb20d9.mp4?s=TfiDzO2oBLrhub_GhToCiQ&e=1676489987&_t=1676476332"
d = Downloader(URL)
d.download("video.mp4")
How can I solve the problem with my library and get the same data in the file? Thank you for any help.

There were two problems with my code:
I found a solution to the first problem here. https://stackoverflow.com/a/25165183/14900791:
The Lock() function creates an entirely new lock - one that only the
thread calling the function can use. That's why it doesn't work,
because each thread is locking an entirely different lock.
Mixdrop (mxdcontent.net) only allows two videos in the same ip, so the code only works for two parts, the others got status code 509 (I didn't checked the status code so I didn't get an error).
import requests
import threading
from tqdm import tqdm
DOWNLOAD_CHUNK_SIZE = 1 << 20 # 1 MiB
# global lock instance
lock = threading.Lock()
class DownloadPart:
def __init__(self, url, byte_range) -> None:
self.url = url
self.byte_range = byte_range
def download(self, file, pbar=None):
response = requests.get(
self.url,
headers={"Range": "bytes={}-{}".format(*self.byte_range)},
allow_redirects=True,
stream=True,
)
written = 0
for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE):
if chunk:
lock.acquire()
file.seek(self.byte_range[0] + written)
length = file.write(chunk)
file.flush()
written += length
pbar.update(length)
lock.release()
class Downloader:
def __init__(self, url, parts=10):
self.url = url
self.parts = parts
def _get_file_size(self) -> int:
info = requests.head(self.url, allow_redirects=True)
info.raise_for_status()
size = info.headers.get("content-length", None)
assert size
return int(size)
def download(self, filename):
file_size = self._get_file_size()
# file_size = 1024
size_per_part = file_size // self.parts
print(file_size, size_per_part)
file = open(filename, "wb")
pbar = tqdm(total=file_size)
threads = []
for index in range(self.parts):
# fix last part have more bytes
if index + 1 == self.parts:
byte_range = (size_per_part * index, file_size - 1)
else:
byte_range = (size_per_part * index, size_per_part * (index + 1) - 1)
thread = threading.Thread(
target=DownloadPart(self.url, byte_range).download, args=(file,), kwargs={"pbar": pbar}
)
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
file.close()
URL = "https://s-delivery38.mxdcontent.net/v/8a5f59673042ed97c402be84ceeb20d9.mp4?s=TfiDzO2oBLrhub_GhToCiQ&e=1676489987&_t=1676476332"
d = Downloader(URL)
d.download("video.mp4")

How to Stream Mp3 music from web using Python PyQt5?

I want to stream mp3 file from web using python PyQt5.I have researched a lot and only found code for streaming wav file.
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtMultimedia import *
import urllib.request
import threading
import time
class Streamer:
def __init__(self,url):
self.url = url
self.fancy = urllib.request.URLopener()
self.web = self.fancy.open(self.url)
self.content_len = int(self.web.headers["Content-Length"])
self.data = self.web.read(1024*1024)
self.buffer = QBuffer()
self.buffer.writeData(self.data[250:])
self.buffer.open(QBuffer.ReadWrite)
threading.Thread(target=self.stream).start()
self.format = QAudioFormat()
self.format.setSampleRate(48000)
self.format.setChannelCount(2)
self.format.setSampleSize(16)
self.format.setCodec("audio/pcm")
self.format.setByteOrder(QAudioFormat.LittleEndian)
self.format.setSampleType(QAudioFormat.SignedInt)
self.audio = QAudioOutput(self.format)
self.audio.start(self.buffer)
def stream(self):
# while True:
# self.sound_data = self.web.read(1024*1024)
# if not self.sound_data:
# break
# self.buffer.buffer().append(self.sound_data)
# time.sleep(2)
while len(self.data) < self.content_len:
self.sound_data = self.web.read(1024*1024)
self.buffer.buffer().append(self.sound_data)
self.data+=self.sound_data
time.sleep(2)
self.buffer.buffer().clear()
del self.data
if __name__ == "__main__":
app = QApplication([])
streamer = Streamer("https://raw.githubusercontent.com/PremKumarMishra/Stream-Songs/main/Audio.wav")
app.exec_()
I checked but cant add MPEG-3(mp3 codec) codec in QAudioFormat.So this current code does not work for mp3.

The basic behavior of QMediaPlayer should be enough to manage buffering of simple audio streams, as the backend will consider the 100% of buffer size as enough to guarantee playing.
In case you want more control over the buffer state, you need to implement a custom QIODevice subclass to act as a middle layer between QMediaPlayer and the download process.
In the following example I'm using QNetworkAccessManager to download the stream, the readyRead signal of the QNetworkReply is then connected to a function that reads the raw bytes, and emits a buffer status considering the current available read data and the minimum size set for the buffer.
The first time the received data has reached the minimum size, it begins to emit the readyRead signal, and if the player has not been started yet (no media set), it sets the media using the Buffer instance, and is then ready to play.
from PyQt5 import QtCore, QtWidgets, QtNetwork, QtMultimedia
url = 'https://url.to/stream'
Errors = {}
for k, v in QtMultimedia.QMediaPlayer.__dict__.items():
if isinstance(v, QtMultimedia.QMediaPlayer.Error):
Errors[v] = k
class Buffer(QtCore.QIODevice):
buffering = QtCore.pyqtSignal(object, object)
fullBufferEmitted = False
def __init__(self, reply, minBufferSize=250000):
super().__init__()
self.minBufferSize = max(200000, minBufferSize)
self.reply = reply
self.data = bytes()
# the network reply is on another thread, use a mutex to ensure that
# no simultaneous access is done in the meantime
self.mutex = QtCore.QMutex()
# this is important!
self.setOpenMode(self.ReadOnly|self.Unbuffered)
self.reply.readyRead.connect(self.dataReceived)
def dataReceived(self):
self.mutex.lock()
self.data += self.reply.readAll().data()
dataLen = len(self.data)
self.mutex.unlock()
self.buffering.emit(dataLen, self.minBufferSize)
if not self.fullBufferEmitted:
if dataLen < self.minBufferSize:
return
self.fullBufferEmitted = True
self.readyRead.emit()
def isSequential(self):
return True
def readData(self, size):
self.mutex.lock()
data = self.data[:size]
self.data = self.data[size:]
self.mutex.unlock()
return data
def bytesAvailable(self):
return len(self.data) + super().bytesAvailable()
class Player(QtWidgets.QWidget):
def __init__(self):
super().__init__()
layout = QtWidgets.QVBoxLayout(self)
self.playButton = QtWidgets.QPushButton('Play', enabled=False)
layout.addWidget(self.playButton)
self.volumeSlider = QtWidgets.QSlider(QtCore.Qt.Horizontal)
layout.addWidget(self.volumeSlider)
self.statusLabel = QtWidgets.QLabel('Waiting')
self.statusLabel.setFrameShape(
self.statusLabel.StyledPanel|self.statusLabel.Sunken)
layout.addWidget(self.statusLabel)
self.player = QtMultimedia.QMediaPlayer(volume=16)
self.volumeSlider.setValue(self.player.volume())
self.networkManager = QtNetwork.QNetworkAccessManager()
self.url = QtCore.QUrl(url)
self.media = QtMultimedia.QMediaContent(self.url)
reply = self.networkManager.get(QtNetwork.QNetworkRequest(self.url))
self.buffer = Buffer(reply)
self.playButton.clicked.connect(self.play)
self.volumeSlider.valueChanged.connect(self.player.setVolume)
self.player.error.connect(self.error)
self.buffer.buffering.connect(self.buffering)
def error(self, error):
errorStr = 'Error: {} ({})'.format(
Errors.get(error, 'Unknown error'), int(error))
self.statusLabel.setText(errorStr)
print(errorStr)
def buffering(self, loaded, minBufferSize):
self.statusLabel.setText('Buffer: {}%'.format(int(loaded / minBufferSize * 100)))
if self.player.media().isNull() and loaded >= minBufferSize:
self.player.setMedia(self.media, self.buffer)
self.playButton.setEnabled(True)
self.playButton.setFocus()
self.statusLabel.setText('Ready to play')
def play(self):
if self.player.state() == self.player.PlayingState:
self.player.pause()
self.playButton.setText('Play')
else:
self.player.play()
self.playButton.setText('Pause')
app = QtWidgets.QApplication([])
w = Player()
w.show()
app.exec_()
Note that:
as soon as QMediaPlayer begins to read the stream, the buffer length will obviously become smaller, as there's no way to know or control how the backend access the stream: when the player is reading (which doesn't mean it's playing), it will read the stream anyway;
due to the reason above, the shown buffer size is only "guessed" as soon as the media is set, based on the data read and the data received from the network reply;
you might want to control the media player status in case the buffer goes too low (but you must consider what explained above), and eventually pause it;

Flask Opencv Recode mini Video clips

I am struggle with generate a mini video clips(Each are 10s video clips) when recording my webcam using open cv and python.
Basically I want to do cut the webcam video into 10S duration clips and store in a folder.
When I doing this video clips cut but when I checked first video clip has 100% of full video.
Second has about 75% of full video and third has about 50% and etc.
So how I can solve this.
I will put my hole code in below. Hope your help to fix that one
camera = cv2.VideoCapture(0)
global rec, img, out
rec = 0
def gen_frames():
global img
while True:
success, img = camera.read()
def record(out):
global rec, img
while(rec != False):
time.sleep(0.05)
out.write(img)
#app.route('/requests',methods=['POST','GET'])
def tasks():
if request.form.get('rec') == 'Start/Stop Recording':
global rec, img, out
rec= not rec
############### This Part work when manualy recode on and off ###############
# if rec:
# print("start")
# global out
# now=datetime.datetime.now()
# fourcc = cv2.VideoWriter_fourcc(*'XVID')
# p = os.path.sep.join(['clips', "vid_{}.avi".format(str(now).replace(":",''))])
# out = cv2.VideoWriter(p, fourcc, 25.0, size)
# thread = Thread(target = record, args=[out,])
# thread.start()
# if(rec==False):
# print("stop")
# out.release()
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
def run(self):
while (rec != False) and not self.event.is_set():
now=datetime.datetime.now()
fourcc = cv2.VideoWriter_fourcc(*'XVID')
p = os.path.sep.join(['clips', "vid_{}.avi".format(str(now).replace(":",''))])
out = cv2.VideoWriter(p, fourcc, 25.0, size)
thread = Thread(target = record, args=[out,])
thread.start()
self.event.wait(10)
def stop(self):
self.event.set()
tmr = TimerClass()
if(rec):
print("start")
tmr.start()
if(rec==False):
print("stop")
tmr.stop()
elif request.method=='GET':
return render_template('index.html')
return render_template('index.html')
if __name__ == '__main__':
app.run(host='0.0.0.0', threaded=True)

As for me problem is that in loop in TimerClass you create new thread every 10 seconds but you never stop previous thread and it still records frames. I would write frames directly in TimerClass instead of using another thread.
OR thread record() should check time and stop after 10 seconds.
I use timedelta to calculate when to stop recording one file and to create next one. And I do it in TimerClass but you could do something similar in record()
VideoWriter(..., 25.0) doesn't write with 25fps but it is only information for video players how fast to display this video. To get 10 seconds video you need sleep 0.04 because 1s / 25fps = 0.04 or you would have to write 250 frames (10s * 25fps = 250fps).
Because code needs also some time to work I would have to use 0.03 instead of 0.04 to get 10 seconds video.
Full working code.
I use render_template_string instead of render_template to have all in one file - and everyone can simply copy and test it.
from flask import Flask, Response, request, render_template, render_template_string
import cv2
import os
import datetime, time
import threading
# global varaibles
capture = False
rec = False
out = None
img = None
app = Flask(__name__)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 300
camera = cv2.VideoCapture(0)
frame_width = int(camera.get(3))
frame_height = int(camera.get(4))
size = (frame_width, frame_height)
os.makedirs('./shots', exist_ok=True)
os.makedirs('./clips', exist_ok=True)
def gen_frames():
global capture
global img
print('[DEBUG] gen_frames: start')
while True:
success, img = camera.read()
if not success:
break
if capture:
capture = False
now = datetime.datetime.now()
filename = "shot_{}.png".format(str(now).replace(":",''))
path = os.path.sep.join(['shots', filename])
print('[DEBUG] capture:', path)
cv2.imwrite(path, img)
frame = cv2.imencode('.jpg', img)[1].tobytes()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
#app.route('/video_feed')
def video_feed():
return Response(gen_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')
#app.route('/')
def index():
#return render_template('index.html')
return render_template_string('''
Go to FORM
''')
# define class only once
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
def run(self):
seconds_10 = datetime.timedelta(seconds=10)
while rec and not self.event.is_set():
now = datetime.datetime.now()
filename = "vid_{}.avi".format(str(now).replace(":", ''))
path = os.path.sep.join(['clips', filename])
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(path, fourcc, 25.0, size)
end = now + seconds_10
print('[DEBUG] end:', end)
while now < end and rec and not self.event.is_set():
if img is not None: # `img` can be `numpy.array` so it can't check `if img:`
out.write(img)
time.sleep(0.03) # 1s / 25fps = 0.04 # it needs some time for code.
now = datetime.datetime.now()
out.release()
def stop(self):
self.event.set()
#app.route('/requests', methods=['POST', 'GET'])
def tasks():
global capture
global rec
print('[DEBUG] click:', request.form.get('click'))
print('[DEBUG] rec :', request.form.get('rec'))
if request.method == 'POST':
if request.form.get('click') == 'Capture':
capture = True
if request.form.get('rec') == 'Start/Stop Recording':
rec = not rec
tmr = TimerClass()
if rec:
print("start")
tmr.start()
else:
print("stop")
tmr.stop()
#return render_template_string('index.html')
return render_template_string('''
<img src="/video_feed"><br/>
<form method="POST">
<button type="submit" name="click" value="Capture">Capture</button>
<button type="submit" name="rec" value="Start/Stop Recording">Start/Stop Recording</button>
</form>
''')
if __name__ == '__main__':
thread_cam = threading.Thread(target=gen_frames)
thread_cam.start()
app.run(host='0.0.0.0', threaded=True)
EDIT:
The same with thread record()
def record(seconds):
now = datetime.datetime.now()
filename = "vid_{}.avi".format(str(now).replace(":", ''))
path = os.path.sep.join(['clips', filename])
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(path, fourcc, 25.0, size)
end = now + datetime.timedelta(seconds=seconds)
print('[DEBUG] end:', end)
while now < end and rec:
if img is not None: # `img` can be `numpy.array` so it can't check `if img:`
out.write(img)
time.sleep(0.03) # 1s / 25fps = 0.04 # it needs some time for code.
now = datetime.datetime.now()
out.release()
# define class only once
class TimerClass(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.event = threading.Event()
def run(self):
length = 10
while rec and not self.event.is_set():
t = threading.Thread(target=record, args=(length,))
t.start()
self.event.wait(length)
def stop(self):
self.event.set()

Trying to display multiple streams with Opencv and Flask

I'm trying to capture two rtsp streams with opencv and then present them with a simple flask server. I can show the two streams together when just using opencv, but when I try to display it through flask it just picks either stream and shows it twice.
Here's the original creators blog
Here is my flask code:
#!/usr/bin/env python
from importlib import import_module
import os
from flask import Flask, render_template, Response
# import camera driver
'''
if os.environ.get('CAMERA'):
Camera = import_module('camera_' + os.environ['CAMERA']).Camera
else:
from camera import Camera
'''
#
from camera_opencv import Camera1, Camera2
app = Flask(__name__)
#app.route('/')
def index():
"""Video streaming home page."""
return render_template('index.html')
def gen(camera):
"""Video streaming generator function."""
while True:
frame = camera.get_frame()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
#app.route('/video_feed1')
def video_feed1():
"""Video streaming route. Put this in the src attribute of an img tag."""
return Response(gen(Camera1()),
mimetype='multipart/x-mixed-replace; boundary=frame')
#app.route('/video_feed2')
def video_feed2():
"""Video streaming route. Put this in the src attribute of an img tag."""
return Response(gen(Camera2()),
mimetype='multipart/x-mixed-replace; boundary=frame')
if __name__ == '__main__':
app.run(host='0.0.0.0', threaded=True, port=8888)
Here's the camera_opencv file
import cv2
from base_camera import BaseCamera
class Camera1(BaseCamera):
video_source = 0
#staticmethod
def set_video_source(source):
Camera1.video_source = source
#staticmethod
def frames():
camera = cv2.VideoCapture(Camera1.video_source)
if not camera.isOpened():
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img = camera.read()
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()
class Camera2(BaseCamera):
video_source = 1
#staticmethod
def set_video_source(source):
Camera2.video_source = source
#staticmethod
def frames():
camera = cv2.VideoCapture(Camera2.video_source)
if not camera.isOpened():
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img = camera.read()
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()
Base camera file
import time
import threading
try:
from greenlet import getcurrent as get_ident
except ImportError:
try:
from thread import get_ident
except ImportError:
from _thread import get_ident
class CameraEvent(object):
"""An Event-like class that signals all active clients when a new frame is
available.
"""
def __init__(self):
self.events = {}
def wait(self):
"""Invoked from each client's thread to wait for the next frame."""
ident = get_ident()
if ident not in self.events:
# this is a new client
# add an entry for it in the self.events dict
# each entry has two elements, a threading.Event() and a timestamp
self.events[ident] = [threading.Event(), time.time()]
return self.events[ident][0].wait()
def set(self):
"""Invoked by the camera thread when a new frame is available."""
now = time.time()
remove = None
for ident, event in self.events.items():
if not event[0].isSet():
# if this client's event is not set, then set it
# also update the last set timestamp to now
event[0].set()
event[1] = now
else:
# if the client's event is already set, it means the client
# did not process a previous frame
# if the event stays set for more than 5 seconds, then assume
# the client is gone and remove it
if now - event[1] > 5:
remove = ident
if remove:
del self.events[remove]
def clear(self):
"""Invoked from each client's thread after a frame was processed."""
self.events[get_ident()][0].clear()
class BaseCamera(object):
thread = None # background thread that reads frames from camera
frame = None # current frame is stored here by background thread
last_access = 0 # time of last client access to the camera
event = CameraEvent()
def __init__(self):
"""Start the background camera thread if it isn't running yet."""
if BaseCamera.thread is None:
BaseCamera.last_access = time.time()
# start background frame thread
BaseCamera.thread = threading.Thread(target=self._thread)
BaseCamera.thread.start()
# wait until frames are available
while self.get_frame() is None:
time.sleep(0)
def get_frame(self):
"""Return the current camera frame."""
BaseCamera.last_access = time.time()
# wait for a signal from the camera thread
BaseCamera.event.wait()
BaseCamera.event.clear()
return BaseCamera.frame
#staticmethod
def frames():
""""Generator that returns frames from the camera."""
raise RuntimeError('Must be implemented by subclasses.')
#classmethod
def _thread(cls):
"""Camera background thread."""
print('Starting camera thread.')
frames_iterator = cls.frames()
for frame in frames_iterator:
BaseCamera.frame = frame
BaseCamera.event.set() # send signal to clients
time.sleep(0)
# if there hasn't been any clients asking for frames in
# the last 10 seconds then stop the thread
if time.time() - BaseCamera.last_access > 10:
frames_iterator.close()
print('Stopping camera thread due to inactivity.')
break
BaseCamera.thread = None
Index.html
<html>
<head>
<title>Video Streaming Demonstration</title>
</head>
<body>
<h1>Video Streaming Demonstration</h1>
<img src="{{ url_for('video_feed1') }}">
<img src="{{ url_for('video_feed2') }}">
</body>
</html>

So I kind of managed to make a hackey workaround. For whatever reason I could not resolve, the app just couldn't handle multiple streams individually.
So I changed the camera class and added multiple sources to it and used numpy.hstack(()) to merge both the frames together thus returning one unique stream.
Will be very grateful if someone could help out here as my method is not at all scalable.
import cv2
from base_camera import BaseCamera
import numpy as np
class Camera(BaseCamera):
video_source1 = 0
video_source2 = 1
#staticmethod
def set_video_source(sources):
Camera.video_source1 = sources[0]
Camera.video_source2 = sources[1]
#staticmethod
def frames():
camera1 = cv2.VideoCapture(Camera.video_source1)
camera2 = cv2.VideoCapture(Camera.video_source2)
if not (camera1.isOpened() or camera2.isOpened()):
raise RuntimeError('Could not start camera.')
while True:
# read current frame
_, img1 = camera1.read()
_, img2 = camera2.read()
img1 = cv2.resize(img1, (704, 396))
img2 = cv2.resize(img2, (704, 396))
img = np.hstack((img1, img2))
# encode as a jpeg image and return it
yield cv2.imencode('.jpg', img)[1].tobytes()

According to the blog, I use two generator and send the image to the index.html. And I can see tow streaming.
def generate2():
# it is a generator
global outputFrame2, lock
while True:
with lock:
if outputFrame is None:
continue
(flag, encodedImage2) = cv2.imencode(".jpg", outputFrame2)
if not flag:
continue
yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' +
bytearray(encodedImage2) + b'\r\n')
And here is videofeed2
#app.route("/video_feed2")
def video_feed2():
return Response(generate2(),
mimetype = "multipart/x-mixed-replace; boundary=frame")

How do I use multiple threads with PySide and Twython?

I'm trying to write a small python app, using PySide for the GUI and Twython as a Twitter API library, to catch a stream from Twitter.
The problem that I am having is that when I click "Start Monitoring Twitter" button, the UI freezes until the stream is complete, at which point the code continues to execute and disables the Start button and enables the Stop button. Here's the UI:
Everything else seems to work -- if I leave it, then the CSV file is created as I suspect -- the Twython components seem to be working as expected.
Line 151 is where the streaming from Twitter is engaged when I click start:
self.stream.statuses.filter(track=self.search_term)
How can I move the streaming to a separate thread and then use the Stop button on the UI to tell Twython to complete capturing the stream and exit?
I need to be able to send the MyStreamer instance to another thread and then send it the .disconnect() signal to have it terminate capturing the stream.
Here's the full code:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import platform
import PySide
from PySide.QtGui import QApplication, QMainWindow, QPushButton, QCheckBox, QTextEdit
from time import sleep
from ui_tweetstream import Ui_MainWindow
from twython import Twython
from twython import TwythonStreamer
import csv
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
# Set up Variables
self.tweet_fav_count = True
self.tweet_geocoordinates = True
self.tweet_id = True
self.tweet_language = True
self.tweet_orig_tweet_id = True
self.tweet_orig_username = True
self.tweet_retweeted = True
self.tweet_sensitive = True
self.tweet_source_app = True
self.tweet_timestamp = True
self.tweet_user_name = True
self.search_term = "#bigdata"
self.tweets_to_get = 1000
# Bind the interface
self.check_tweet_fav_count.clicked.connect(self.setTweetFavCount)
self.check_tweet_geocoordinates.clicked.connect(self.setTweetGeocoordinates)
self.check_tweet_id.clicked.connect(self.setTweetID)
self.check_tweet_language.clicked.connect(self.setTweetLanguage)
self.check_tweet_orig_tweet_id.clicked.connect(self.setTweetOrigTweetID)
self.check_tweet_orig_username.clicked.connect(self.setTweetOrigUsername)
self.check_tweet_retweeted.clicked.connect(self.setTweetRetweeted)
self.check_tweet_sensitive.clicked.connect(self.setTweetSensitive)
self.check_tweet_source_app.clicked.connect(self.setTweetSourceApp)
self.check_tweet_timestamp.clicked.connect(self.setTweetTimestamp)
self.check_tweet_user_name.clicked.connect(self.setTweetUsername)
self.button_start.clicked.connect(self.streamStart)
self.button_stop.clicked.connect(self.streamStop)
# Set the initial states
self.button_stop.setEnabled(False)
APP_KEY = ''
APP_SECRET = ''
OAUTH_TOKEN = ''
OAUTH_TOKEN_SECRET = ''
self.t = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
self.stream = MyStreamer(APP_KEY,APP_SECRET,OAUTH_TOKEN,OAUTH_TOKEN_SECRET)
self.stream.init_mainWindow(self)
def streamStop(self):
print "Stopping stream"
# Enable other controls here
self.button_stop.setEnabled(False)
self.button_start.setEnabled(True)
self.setControlStates(True)
self.stream.stopStream()
def setControlStates(self, state):
self.check_tweet_fav_count.setEnabled(state)
self.check_tweet_geocoordinates.setEnabled(state)
self.check_tweet_id.setEnabled(state)
self.check_tweet_language.setEnabled(state)
self.check_tweet_orig_tweet_id.setEnabled(state)
self.check_tweet_orig_username.setEnabled(state)
self.check_tweet_retweeted.setEnabled(state)
self.check_tweet_sensitive.setEnabled(state)
self.check_tweet_source_app.setEnabled(state)
self.check_tweet_timestamp.setEnabled(state)
self.check_tweet_user_name.setEnabled(state)
self.search_box.setEnabled(state)
self.num_tweets_box.setEnabled(state)
# Functions for determining what to track
def setTweetFavCount(self):
self.tweet_fav_count = not self.tweet_fav_count
print "tweet_fav_count:", self.tweet_fav_count
def setTweetGeocoordinates(self):
self.tweet_geocoordinates = not self.tweet_geocoordinates
print "tweet_geocoordinates:", self.tweet_geocoordinates
def setTweetID(self):
self.tweet_id = not self.tweet_id
print "tweet_id:", self.tweet_id
def setTweetLanguage(self):
self.tweet_language = not self.tweet_language
print "tweet_language:", self.tweet_language
def setTweetOrigTweetID(self):
self.tweet_orig_tweet_id = not self.tweet_orig_tweet_id
print "tweet_orig_tweet_id:", self.tweet_orig_tweet_id
def setTweetOrigUsername(self):
self.tweet_orig_username = not self.tweet_orig_tweet_id
print "tweet_orig_username:", self. tweet_orig_username
def setTweetRetweeted(self):
self.tweet_retweeted = not self.tweet_retweeted
print "tweet_retweeted:", self.tweet_retweeted
def setTweetSensitive(self):
self.tweet_sensitive = not self.tweet_sensitive
print "tweet_sensitive:", self.tweet_sensitive
def setTweetSourceApp(self):
self.tweet_source_app = not self.tweet_source_app
print "tweet_source_app:", self.tweet_source_app
def setTweetTimestamp(self):
self.tweet_timestamp = not self.tweet_timestamp
print "tweet_timestamp:", self.tweet_timestamp
def setTweetUsername(self):
self.tweet_user_name = not self.tweet_user_name
print "tweet_user_name:", self.tweet_user_name
# Functions for starting and stopping the stream
def streamStart(self):
print "Starting stream"
self.setControlStates(False)
# Disable other controls here
self.button_start.setEnabled(False)
self.button_stop.setEnabled(True)
# Hack to try to disable the UI
# sleep(0.25)
# Get the active search term
self.search_term = self.search_box.text()
# Get the number of tweets
self.tweets_to_get = int(self.num_tweets_box.text())
# Set the streamer
self.stream.set_start_criteria(self.tweets_to_get)
self.stream.statuses.filter(track=self.search_term)
class MyStreamer(TwythonStreamer):
def init_mainWindow(self, the_main_window):
self.main_window = the_main_window
self.stop = False
self.header_done = False
def set_start_criteria(self, numTweets):
self.maxTweets = numTweets
self.tweetCount = 0
print "Number of tweets to get:", self.maxTweets
def stopStream(self):
self.stop = True
def on_success(self, data):
if 'text' in data:
self.tweetCount += 1
print "tweetCount:", self.tweetCount
#tweet = data['text'].encode('utf-8')
theTweet = data
writer = TweetMonkey()
writer.assignMainWindow(self.main_window, self.header_done)
self.header_done = True
writer.process(theTweet)
# Want to disconnect after the first result?
if self.stop is True or self.tweetCount >= self.maxTweets:
self.disconnect()
def on_error(self, status_code, data):
print status_code, data
class TweetMonkey:
def assignMainWindow(self,the_main_window, is_header_done):
self.main_window = the_main_window
self.header_done = is_header_done
def clean(self,text):
text = text.replace("\n","; ")
text = text.replace('"', "'")
text = text.replace(','," ")
return text
def create_header(self):
header = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
header.append("id")
if self.main_window.tweet_language is True:
header.append("lang")
if self.main_window.tweet_user_name is True:
header.append("user_name")
header.append("tweet")
if self.main_window.tweet_retweeted is True:
header.append("retweeted")
if self.main_window.tweet_fav_count is True:
header.append("favorite_count")
if self.main_window.tweet_source_app is True:
header.append("source")
if self.main_window.tweet_orig_tweet_id is True:
header.append("in_reply_to_status_id")
if self.main_window.tweet_orig_username is True:
header.append("in_reply_to_screen_name")
# header.append("in_reply_to_user_id")
if self.main_window.tweet_sensitive is True:
header.append("possibly_sensitive")
if self.main_window.tweet_geocoordinates is True:
header.append("geo")
if self.main_window.tweet_timestamp is True:
header.append("created_at")
wr.writerow(header)
tweets.close()
def process(self, tweet):
if not self.header_done:
self.create_header()
self.header_done = True
# Create the file or append to the existing
theOutput = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
theOutput.append(tweet['id'])
if self.main_window.tweet_language is True:
theOutput.append(tweet['lang'].encode('utf-8'))
if self.main_window.tweet_user_name is True:
theOutput.append(tweet['user']['name'].encode('utf-8', 'replace'))
theOutput.append(self.clean(tweet['text']).encode('utf-8', 'replace'))
if self.main_window.tweet_retweeted is True:
theOutput.append(tweet['retweeted'])
if self.main_window.tweet_fav_count is True:
theOutput.append(tweet['favorite_count'])
if self.main_window.tweet_source_app is True:
theOutput.append(self.clean(tweet['source']).encode('utf-8', 'replace'))
if self.main_window.tweet_orig_tweet_id is True:
theOutput.append(tweet['in_reply_to_status_id'])
if self.main_window.tweet_orig_username is True:
theOutput.append(tweet['in_reply_to_screen_name'])
#theOutput.append(tweet['in_reply_to_user_id'])
if self.main_window.tweet_sensitive is True:
if tweet.get('possibly_sensitive'):
theOutput.append(tweet['possibly_sensitive'])
else:
theOutput.append("False")
if self.main_window.tweet_geocoordinates is True:
if tweet['geo'] is not None:
if tweet['geo']['type'] == 'Point':
lat = str(tweet['geo']['coordinates'][0]) + " "
lon = str(tweet['geo']['coordinates'][1])
theOutput.append(lat + lon)
else:
theOutput.append(tweet['geo'])
else:
theOutput.append(tweet['geo'])
if self.main_window.tweet_timestamp is True:
theOutput.append(tweet['created_at'])
wr.writerow(theOutput)
tweets.close()
if __name__ == '__main__':
app = QApplication(sys.argv)
frame = MainWindow()
frame.show()
app.exec_()

I know this is an old post but I ran into a similar problem in a simple app I recently wrote, my solution was to use threading.
I used the worker from:
https://pymotw.com/2/threading/
and the method described in:
http://aadrake.com/using-twitter-as-a-stream-processing-source.html
Basically running the Twython stream as a separate thread feeding text to a queue then I run the rest of the program in a separate loop reading from the queue.