I Am New In wxpython.
I am Making Program using the pytube library.
I can download youtube videos just fine on its own, but when I put it in the WXPython method, the UI suddenly hangs and becomes unresponsive. But When The Video finnished downloading, the UI is responsive again.
This Is My Code.
import wx
import pytube
from pytube import YouTube
from pytube import Playlist
import os
import time
from threading import Timer as wait
from accessible_output2 import outputs
say=outputs.auto.Auto()
speak=say.speak
app = wx.App()
class abhishek(wx.Frame):
def __init__(self):
super().__init__(None, -1, title = 'ar-developers YouTube Downloader, 1.2.')
self.p = wx.Panel(self)
self.Center()
vid = wx.Button(self.p, -1, 'Download Video...')
vid.Bind(wx.EVT_BUTTON, self.video_download)
aud = wx.Button(self.p, -1, 'Download Audio...')
aud.Bind(wx.EVT_BUTTON, self.audio_download)
pv = wx.Button(self.p, -1, 'Download Playlist...')
pv.Bind(wx.EVT_BUTTON, self.playlist_download)
pa = wx.Button(self.p, -1, 'Download Playlist in Audio Format...')
pa.Bind(wx.EVT_BUTTON, self.playlist_audio_download)
self.Show()
def on_progress(self, stream=None, chunk=None, bytes_remaining=None):
self.x = '%'
self.total_size = self.video.filesize
self.bytes_downloaded = self.total_size - bytes_remaining
self.liveprogress = int(self.bytes_downloaded / self.total_size * 100)
speak(f"downloading, {self.liveprogress} {self.x}")
time.sleep(2)
def video_download(self, event):
url = wx.GetTextFromUser('Type the URL of your video', 'URL')
try:
youtube = pytube.YouTube(url,on_progress_callback=self.on_progress)
except:
wx.MessageBox('Please enter a valid URL', 'Alert')
exit()
speak(f'downloading, {youtube.title}')
self.video = youtube.streams.get_highest_resolution()
self.video.download("Downloads")
wx.MessageBox('Download Complete!', 'Complete')
def audio_download(self, event):
url = wx.GetTextFromUser("Enter URL of Your Video", "URL")
try:
divi = YouTube(url, on_progress_callback=self.on_progress)
except:
wx.MessageBox('Please Enter a Valid URL','Error')
exit()
speak(f'downloading, {divi.title}')
self.video = divi.streams.filter(only_audio=True).first()
self.out_file = self.video.download("Downloads")
base, ext = os.path.splitext(self.out_file)
self.new_file = base + '.mp3'
os.rename(self.out_file, self.new_file)
wx.MessageBox('Download Complete!', 'Complete')
def playlist_download(self, event):
url = wx.GetTextFromUser("Enter the URL of your playlist. ", "URL")
try:
playlist = Playlist(url)
except:
wx.MessageBox('Please Enter a Valid URL','Error')
exit()
speak('Please Wait, While We Are Downloading Your Play List')
for p in playlist.video_urls:
f = pytube.YouTube(p,on_progress_callback=self.on_progress)
speak(f'downloading, {f.title}')
self.video = f.streams.get_highest_resolution()
self.video.download("Downloads")
def playlist_audio_download(self, event):
url = wx.GetTextFromUser("Enter URL of your playlist", "URL")
try:
playlist = Playlist(url)
except:
wx.MessageBox('Please Enter a Valid URL','Error')
exit()
speak('Please Wait, While We Are Downloading Your Play List')
for p in playlist.video_urls:
f = YouTube(p, on_progress_callback=self.on_progress)
speak(f'downloading, {f.title}')
self.video = f.streams.filter(only_audio=True).first()
self.out_file = self.video.download("Downloads")
base, ext = os.path.splitext(self.out_file)
self.new_file = base + '.mp3'
os.rename(self.out_file, self.new_file)
abhishek()
app.MainLoop()
It is a general issue. Not only in wx the not responding issue is happening in tkinter too. But it happens when you run the code after compiling to an exe file.
Related
I have made a Desktop Application using Python and used PyQt5 and Pytube which could download video from youtube. When download is in Progress, I want to show user an animation. In Fact I did it, but when the file is getting downloaded the PyQt window seems like freezing and everything just gets paused until the download is complete. So, Does anyone know why is this happening? How do I fix it?
Here's the code snippet:
def download_created(self, qual): # Used in 'selection' method
selected_stream = yt.streams.get_by_resolution(qual)
self.progress_func()
try:
self.download_btn.setCurrentIndex(-1)
selected_stream.download(self.askLocation() + "/")
except:
pass
# This gets the quality that the user chooses
def selection(self):
global quality
quality = self.download_btn.currentText()
try:
self.download_created(quality) # Calls a method called 'download'
except:
self.start_anime()
# Fetching the details about the Link from Youtube
def download_youtube(self):
global check
if check != self.get_input():
check = self.get_input()
self.download_btn.clear()
enter_url = self.get_input()
try:
global yt
yt = pytube.YouTube(
enter_url,
on_progress_callback = on_progress,
on_complete_callback = self.complete_func)
self.start_anime()
except:
self.input_error()
VIDEO_TITLE = (yt.title)
global VIDEO_ID
VIDEO_ID = (yt.video_id)
videos = yt.streams.filter(mime_type="video/mp4", progressive="True")
# Display all the available qualities
for i in videos:
self.download_btn.addItem(i.resolution)
self.download_btn.currentIndexChanged.connect(self.selection)
You have to execute the time consuming tasks in another thread, for example in your case the task of getting the streams and downloading.
import sys
import threading
from functools import cached_property
from PyQt5 import QtCore, QtWidgets
import pytube
class QPyTube(QtCore.QObject):
initialized = QtCore.pyqtSignal(bool, str)
download_started = QtCore.pyqtSignal()
download_progress_changed = QtCore.pyqtSignal(int)
download_finished = QtCore.pyqtSignal()
def __init__(self, url):
super().__init__()
self._url = url
self._yt = None
self._mutex = threading.Lock()
threading.Thread(target=self._init, daemon=True).start()
#property
def url(self):
return self._url
#cached_property
def resolutions(self):
return list()
def _init(self):
with self._mutex:
self.resolutions.clear()
try:
self._yt = pytube.YouTube(
self.url,
on_progress_callback=self._on_progress,
on_complete_callback=self._on_complete,
)
streams = self._yt.streams.filter(mime_type="video/mp4", progressive="True")
except Exception as e:
self.initialized.emit(False, str(e))
return
with self._mutex:
self.resolutions = [stream.resolution for stream in streams]
self.initialized.emit(True, "")
def download(self, resolution, directory):
threading.Thread(
target=self._download, args=(resolution, directory), daemon=True
).start()
def _download(self, resolution, directory):
stream = self._yt.streams.get_by_resolution(resolution)
self.download_started.emit()
stream.download(directory)
def _on_progress(self, stream, chunk, bytes_remaining):
self.download_progress_changed.emit(
100 * (stream.filesize - bytes_remaining) // stream.filesize
)
def _on_complete(self, stream, filepath):
self.download_finished.emit()
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.le_url = QtWidgets.QLineEdit("http://youtube.com/watch?v=2lAe1cqCOXo")
self.lbl_error = QtWidgets.QLabel()
self.btn_search = QtWidgets.QPushButton("Search")
self.cmb_resolutions = QtWidgets.QComboBox()
self.le_directory = QtWidgets.QLineEdit("")
self.btn_download = QtWidgets.QPushButton("Download")
self.pgb_download = QtWidgets.QProgressBar()
central_widget = QtWidgets.QWidget()
self.setCentralWidget(central_widget)
lay = QtWidgets.QGridLayout(central_widget)
lay.addWidget(self.le_url, 0, 0)
lay.addWidget(self.btn_search, 0, 1)
lay.addWidget(self.cmb_resolutions, 1, 0)
lay.addWidget(self.le_directory, 1, 1)
lay.addWidget(self.btn_download, 1, 2)
lay.addWidget(self.pgb_download, 2, 0, 1, 3)
self.btn_download.setEnabled(False)
self._qpytube = None
self.btn_search.clicked.connect(self.handle_search_clicked)
self.btn_download.clicked.connect(self.handle_download_clicked)
def handle_search_clicked(self):
self.cmb_resolutions.clear()
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.lbl_error.clear()
self._qpytube = QPyTube(self.le_url.text())
self._qpytube.initialized.connect(self.handle_initialized)
self._qpytube.download_progress_changed.connect(self.pgb_download.setValue)
self._qpytube.download_started.connect(self.handle_download_started)
self._qpytube.download_finished.connect(self.handle_download_finished)
#QtCore.pyqtSlot(bool, str)
def handle_initialized(self, status, error=""):
if status:
self.cmb_resolutions.addItems(self._qpytube.resolutions)
self.btn_download.setEnabled(True)
else:
self.lbl_error.setText(error)
self.btn_search.setEnabled(True)
def handle_download_clicked(self):
self._qpytube.download(
self.cmb_resolutions.currentText(), self.le_directory.text()
)
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.le_directory.setEnabled(False)
def handle_download_started(self):
self.lbl_error.clear()
print("started")
def handle_download_finished(self):
self.pgb_download.setValue(100)
self.btn_search.setEnabled(True)
self.btn_download.setEnabled(True)
self.le_directory.setEnabled(True)
print("finished")
def main(args):
app = QtWidgets.QApplication(args)
w = MainWindow()
w.show()
app.exec_()
if __name__ == "__main__":
main(sys.argv)
I have made a Desktop Application using Python and used PyQt5 and Pytube which could download video from youtube. When download is in Progress, I want to show user an animation. In Fact I did it, but when the file is getting downloaded the PyQt window seems like freezing and everything just gets paused until the download is complete. So, Does anyone know why is this happening? How do I fix it?
Here's the code snippet:
def download_created(self, qual): # Used in 'selection' method
selected_stream = yt.streams.get_by_resolution(qual)
self.progress_func()
try:
self.download_btn.setCurrentIndex(-1)
selected_stream.download(self.askLocation() + "/")
except:
pass
# This gets the quality that the user chooses
def selection(self):
global quality
quality = self.download_btn.currentText()
try:
self.download_created(quality) # Calls a method called 'download'
except:
self.start_anime()
# Fetching the details about the Link from Youtube
def download_youtube(self):
global check
if check != self.get_input():
check = self.get_input()
self.download_btn.clear()
enter_url = self.get_input()
try:
global yt
yt = pytube.YouTube(
enter_url,
on_progress_callback = on_progress,
on_complete_callback = self.complete_func)
self.start_anime()
except:
self.input_error()
VIDEO_TITLE = (yt.title)
global VIDEO_ID
VIDEO_ID = (yt.video_id)
videos = yt.streams.filter(mime_type="video/mp4", progressive="True")
# Display all the available qualities
for i in videos:
self.download_btn.addItem(i.resolution)
self.download_btn.currentIndexChanged.connect(self.selection)
You have to execute the time consuming tasks in another thread, for example in your case the task of getting the streams and downloading.
import sys
import threading
from functools import cached_property
from PyQt5 import QtCore, QtWidgets
import pytube
class QPyTube(QtCore.QObject):
initialized = QtCore.pyqtSignal(bool, str)
download_started = QtCore.pyqtSignal()
download_progress_changed = QtCore.pyqtSignal(int)
download_finished = QtCore.pyqtSignal()
def __init__(self, url):
super().__init__()
self._url = url
self._yt = None
self._mutex = threading.Lock()
threading.Thread(target=self._init, daemon=True).start()
#property
def url(self):
return self._url
#cached_property
def resolutions(self):
return list()
def _init(self):
with self._mutex:
self.resolutions.clear()
try:
self._yt = pytube.YouTube(
self.url,
on_progress_callback=self._on_progress,
on_complete_callback=self._on_complete,
)
streams = self._yt.streams.filter(mime_type="video/mp4", progressive="True")
except Exception as e:
self.initialized.emit(False, str(e))
return
with self._mutex:
self.resolutions = [stream.resolution for stream in streams]
self.initialized.emit(True, "")
def download(self, resolution, directory):
threading.Thread(
target=self._download, args=(resolution, directory), daemon=True
).start()
def _download(self, resolution, directory):
stream = self._yt.streams.get_by_resolution(resolution)
self.download_started.emit()
stream.download(directory)
def _on_progress(self, stream, chunk, bytes_remaining):
self.download_progress_changed.emit(
100 * (stream.filesize - bytes_remaining) // stream.filesize
)
def _on_complete(self, stream, filepath):
self.download_finished.emit()
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.le_url = QtWidgets.QLineEdit("http://youtube.com/watch?v=2lAe1cqCOXo")
self.lbl_error = QtWidgets.QLabel()
self.btn_search = QtWidgets.QPushButton("Search")
self.cmb_resolutions = QtWidgets.QComboBox()
self.le_directory = QtWidgets.QLineEdit("")
self.btn_download = QtWidgets.QPushButton("Download")
self.pgb_download = QtWidgets.QProgressBar()
central_widget = QtWidgets.QWidget()
self.setCentralWidget(central_widget)
lay = QtWidgets.QGridLayout(central_widget)
lay.addWidget(self.le_url, 0, 0)
lay.addWidget(self.btn_search, 0, 1)
lay.addWidget(self.cmb_resolutions, 1, 0)
lay.addWidget(self.le_directory, 1, 1)
lay.addWidget(self.btn_download, 1, 2)
lay.addWidget(self.pgb_download, 2, 0, 1, 3)
self.btn_download.setEnabled(False)
self._qpytube = None
self.btn_search.clicked.connect(self.handle_search_clicked)
self.btn_download.clicked.connect(self.handle_download_clicked)
def handle_search_clicked(self):
self.cmb_resolutions.clear()
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.lbl_error.clear()
self._qpytube = QPyTube(self.le_url.text())
self._qpytube.initialized.connect(self.handle_initialized)
self._qpytube.download_progress_changed.connect(self.pgb_download.setValue)
self._qpytube.download_started.connect(self.handle_download_started)
self._qpytube.download_finished.connect(self.handle_download_finished)
#QtCore.pyqtSlot(bool, str)
def handle_initialized(self, status, error=""):
if status:
self.cmb_resolutions.addItems(self._qpytube.resolutions)
self.btn_download.setEnabled(True)
else:
self.lbl_error.setText(error)
self.btn_search.setEnabled(True)
def handle_download_clicked(self):
self._qpytube.download(
self.cmb_resolutions.currentText(), self.le_directory.text()
)
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.le_directory.setEnabled(False)
def handle_download_started(self):
self.lbl_error.clear()
print("started")
def handle_download_finished(self):
self.pgb_download.setValue(100)
self.btn_search.setEnabled(True)
self.btn_download.setEnabled(True)
self.le_directory.setEnabled(True)
print("finished")
def main(args):
app = QtWidgets.QApplication(args)
w = MainWindow()
w.show()
app.exec_()
if __name__ == "__main__":
main(sys.argv)
I wrote this program with Python:
import os
from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.textinput import TextInput
from kivy.uix.button import Button
from time import sleep
import pytube
import scrapetube
from pathlib import Path
save_addr = f"{os.environ['USERPROFILE']}\Downloads\Youtube DL"
print(save_addr)
layout = BoxLayout
def youtube_dl(video,addr):
#yt = pytube.YouTube(f"{link.text}")
#ys = yt.streams.get_highest_resolution()
#ys.download()
video = str(video.text)
try:
yt = pytube.YouTube(video)
ys = yt.streams.get_highest_resolution()
ys.download(output_path=addr)
except pytube.exceptions.RegexMatchError:
print('The Regex pattern did not return any matches for the video: {}'.format(video))
except pytube.exceptions.ExtractError:
print('An extraction error occurred for the video: {}'.format(video))
except pytube.exceptions.VideoUnavailable:
print('The following video is unavailable: {}'.format(video))
class cd:
"""Context manager for changing the current working directory"""
def __init__(self, newPath):
self.newPath = os.path.expanduser(newPath)
def __enter__(self):
self.savedPath = os.getcwd()
os.chdir(self.newPath)
def __exit__(self, etype, value, traceback):
os.chdir(self.savedPath)
class MyGrid(layout):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.orientation = "vertical"
self.spacing = 20
self.padding = [10, 10]
self.cols = 1
self.label = Label(text="Enter link of Video You Want to Download!")
self.add_widget(self.label)
self.link = TextInput(hint_text="Example: youtube.com/watch?v=MXDHK_MqRsk", multiline=False)
self.add_widget(self.link)
self.btn = Button(text="OK!")
self.btn.bind(on_press=lambda x:self.callback())
self.add_widget(self.btn)
sleep(3)
def callback(self):
def dl(video,addr,DL=True):
try:
yt = pytube.YouTube(video)
if DL:
ys = yt.streams.get_highest_resolution()
cd(addr)
ys.download(output_path=addr)
except pytube.exceptions.RegexMatchError:
print('The Regex pattern did not return any matches for the video: {}'.format(video))
except pytube.exceptions.ExtractError:
print('An extraction error occurred for the video: {}'.format(video))
except pytube.exceptions.VideoUnavailable:
print('The following video is unavailable: {}'.format(video))
return yt
video = str(self.link.text)
title = str(dl(video,self.link,DL=False).title)
self.label.text = f"Download : \"{title}\" , Save in:"
self.link.text = f"{save_addr}"
self.btn.text = "Download video!"
self.btn.bind(on_press=lambda x:dl(video,self.link.text))
class MyKivy(App):
def build(self):
return MyGrid()
if __name__ == "__main__":
MyKivy().run()
My problem is that when I put the video link to the program and click on download, when downloading the video from YouTube, the message not responding is displayed, but after the download is complete, the program returns to normal:
image of this error
What can I do to prevent the message not responding from being displayed?
To prevent this, you have to use multithreading, that means you can do multiple tasks simulatiously, that will prevent the UI from freezing because it will running in a thread that is not in the same thread that the downloader is running on. you will make a function to run the downloader in threading like this:
import threading
def run_thread(self,*args):
thread_dl = thrading.Thread(target = self.downloader,args = (arg1,arg2,arg3))
thread_dl.start()
you will edit this function to be the button on_press function
I wrote a program to extract link pictures from webcomics, however, when I run it, it just extracts image links from the last link chapter, not all image links from all chapters. What is the issue with my program?
I have tried several ways but not things useful.
from PyQt5 import QtNetwork, QtCore
from requests_html import HTML
from functools import cached_property
from PyQt5.QtCore import QCoreApplication, QUrl
url1 = "https://saytruyen.net/truyen-su-tro-lai-cua-phap-su-hac-am-sau-66666-nam.html"
class Manager:
def __init__(self):
self.manager.finished.connect(self.handle_response)
#cached_property
def manager(self):
return QtNetwork.QNetworkAccessManager()
def start(self):
self.start_request(QtCore.QUrl(url1))
def start_request(self, url):
request = QtNetwork.QNetworkRequest(url)
self.manager.get(request)
def handle_response(self, reply):
err = reply.error()
if err == QtNetwork.QNetworkReply.NoError:
self.process(str(reply.readAll(), 'utf-8'))
else:
print("Error occured: ", err)
print(reply.errorString())
def process(self, data):
html = HTML(html=data)
rs = html.find("#list-chapter a", first=False)
for i in reversed(rs):
url2 = "https://saytruyen.net/" + i.attrs["href"]
#print(url2)
#self.start_request(QtCore.QUrl(url2))
req = QtNetwork.QNetworkRequest(QUrl(url2))
self.nam = QtNetwork.QNetworkAccessManager()
self.nam.finished.connect(self.handleResponse)
self.nam.get(req)
def handleResponse(self, reply):
er = reply.error()
if er == QtNetwork.QNetworkReply.NoError:
bytes_string = reply.readAll()
html2 = HTML(html = str(bytes_string, 'utf-8'))
rs_c = html2.find("#lst_content img")
for x in rs_c:
img ="https://saytruyen.net/" + x.attrs['src']
print(img)
else:
print("Error occured: ", er)
print(reply.errorString())
QCoreApplication.quit()
There are two problems:
the QNetworkAccessManager used for the download is being continuously recreated; since a network request is asynchronous, it isn't processed instantly, and it will be destroyed along with the network manager in the next cycle of the for loop since it's being overwritten; the result is that the previous request will be destroyed and only the last one will "survive";
the application is quit as soon as the first reply is received, preventing all other requests to be processed;
The solution is to create a single manager for the download process in the __init__, and quit as soon as all requests have been received.
class Manager:
def __init__(self):
self.manager.finished.connect(self.handle_response)
self.nam = QtNetwork.QNetworkAccessManager()
self.nam.finished.connect(self.handleResponse)
self.urls = set()
# ...
def process(self, data):
html = HTML(html=data)
rs = html.find("#list-chapter a", first=False)
for i in reversed(rs):
url2 = QUrl("https://saytruyen.net/" + i.attrs["href"])
if url2 in self.urls:
continue
self.urls.add(url2)
req = QtNetwork.QNetworkRequest(url2)
self.nam.get(req)
def handleResponse(self, reply):
self.urls.discard(reply.url())
er = reply.error()
if er == QtNetwork.QNetworkReply.NoError:
bytes_string = reply.readAll()
html2 = HTML(html = str(bytes_string, 'utf-8'))
rs_c = html2.find("#lst_content img")
for x in rs_c:
img ="https://saytruyen.net/" + x.attrs['src']
print(img)
else:
print("Error occured: ", er)
print(reply.errorString())
if not self.urls:
QCoreApplication.quit()
Note that it's usually enough (and better) to have a single network manager and properly handle responses based on queued requests, but for simple situations like this one having two managers doesn't represent a huge problem.
I'm trying to write a small python app, using PySide for the GUI and Twython as a Twitter API library, to catch a stream from Twitter.
The problem that I am having is that when I click "Start Monitoring Twitter" button, the UI freezes until the stream is complete, at which point the code continues to execute and disables the Start button and enables the Stop button. Here's the UI:
Everything else seems to work -- if I leave it, then the CSV file is created as I suspect -- the Twython components seem to be working as expected.
Line 151 is where the streaming from Twitter is engaged when I click start:
self.stream.statuses.filter(track=self.search_term)
How can I move the streaming to a separate thread and then use the Stop button on the UI to tell Twython to complete capturing the stream and exit?
I need to be able to send the MyStreamer instance to another thread and then send it the .disconnect() signal to have it terminate capturing the stream.
Here's the full code:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import platform
import PySide
from PySide.QtGui import QApplication, QMainWindow, QPushButton, QCheckBox, QTextEdit
from time import sleep
from ui_tweetstream import Ui_MainWindow
from twython import Twython
from twython import TwythonStreamer
import csv
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
# Set up Variables
self.tweet_fav_count = True
self.tweet_geocoordinates = True
self.tweet_id = True
self.tweet_language = True
self.tweet_orig_tweet_id = True
self.tweet_orig_username = True
self.tweet_retweeted = True
self.tweet_sensitive = True
self.tweet_source_app = True
self.tweet_timestamp = True
self.tweet_user_name = True
self.search_term = "#bigdata"
self.tweets_to_get = 1000
# Bind the interface
self.check_tweet_fav_count.clicked.connect(self.setTweetFavCount)
self.check_tweet_geocoordinates.clicked.connect(self.setTweetGeocoordinates)
self.check_tweet_id.clicked.connect(self.setTweetID)
self.check_tweet_language.clicked.connect(self.setTweetLanguage)
self.check_tweet_orig_tweet_id.clicked.connect(self.setTweetOrigTweetID)
self.check_tweet_orig_username.clicked.connect(self.setTweetOrigUsername)
self.check_tweet_retweeted.clicked.connect(self.setTweetRetweeted)
self.check_tweet_sensitive.clicked.connect(self.setTweetSensitive)
self.check_tweet_source_app.clicked.connect(self.setTweetSourceApp)
self.check_tweet_timestamp.clicked.connect(self.setTweetTimestamp)
self.check_tweet_user_name.clicked.connect(self.setTweetUsername)
self.button_start.clicked.connect(self.streamStart)
self.button_stop.clicked.connect(self.streamStop)
# Set the initial states
self.button_stop.setEnabled(False)
APP_KEY = ''
APP_SECRET = ''
OAUTH_TOKEN = ''
OAUTH_TOKEN_SECRET = ''
self.t = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
self.stream = MyStreamer(APP_KEY,APP_SECRET,OAUTH_TOKEN,OAUTH_TOKEN_SECRET)
self.stream.init_mainWindow(self)
def streamStop(self):
print "Stopping stream"
# Enable other controls here
self.button_stop.setEnabled(False)
self.button_start.setEnabled(True)
self.setControlStates(True)
self.stream.stopStream()
def setControlStates(self, state):
self.check_tweet_fav_count.setEnabled(state)
self.check_tweet_geocoordinates.setEnabled(state)
self.check_tweet_id.setEnabled(state)
self.check_tweet_language.setEnabled(state)
self.check_tweet_orig_tweet_id.setEnabled(state)
self.check_tweet_orig_username.setEnabled(state)
self.check_tweet_retweeted.setEnabled(state)
self.check_tweet_sensitive.setEnabled(state)
self.check_tweet_source_app.setEnabled(state)
self.check_tweet_timestamp.setEnabled(state)
self.check_tweet_user_name.setEnabled(state)
self.search_box.setEnabled(state)
self.num_tweets_box.setEnabled(state)
# Functions for determining what to track
def setTweetFavCount(self):
self.tweet_fav_count = not self.tweet_fav_count
print "tweet_fav_count:", self.tweet_fav_count
def setTweetGeocoordinates(self):
self.tweet_geocoordinates = not self.tweet_geocoordinates
print "tweet_geocoordinates:", self.tweet_geocoordinates
def setTweetID(self):
self.tweet_id = not self.tweet_id
print "tweet_id:", self.tweet_id
def setTweetLanguage(self):
self.tweet_language = not self.tweet_language
print "tweet_language:", self.tweet_language
def setTweetOrigTweetID(self):
self.tweet_orig_tweet_id = not self.tweet_orig_tweet_id
print "tweet_orig_tweet_id:", self.tweet_orig_tweet_id
def setTweetOrigUsername(self):
self.tweet_orig_username = not self.tweet_orig_tweet_id
print "tweet_orig_username:", self. tweet_orig_username
def setTweetRetweeted(self):
self.tweet_retweeted = not self.tweet_retweeted
print "tweet_retweeted:", self.tweet_retweeted
def setTweetSensitive(self):
self.tweet_sensitive = not self.tweet_sensitive
print "tweet_sensitive:", self.tweet_sensitive
def setTweetSourceApp(self):
self.tweet_source_app = not self.tweet_source_app
print "tweet_source_app:", self.tweet_source_app
def setTweetTimestamp(self):
self.tweet_timestamp = not self.tweet_timestamp
print "tweet_timestamp:", self.tweet_timestamp
def setTweetUsername(self):
self.tweet_user_name = not self.tweet_user_name
print "tweet_user_name:", self.tweet_user_name
# Functions for starting and stopping the stream
def streamStart(self):
print "Starting stream"
self.setControlStates(False)
# Disable other controls here
self.button_start.setEnabled(False)
self.button_stop.setEnabled(True)
# Hack to try to disable the UI
# sleep(0.25)
# Get the active search term
self.search_term = self.search_box.text()
# Get the number of tweets
self.tweets_to_get = int(self.num_tweets_box.text())
# Set the streamer
self.stream.set_start_criteria(self.tweets_to_get)
self.stream.statuses.filter(track=self.search_term)
class MyStreamer(TwythonStreamer):
def init_mainWindow(self, the_main_window):
self.main_window = the_main_window
self.stop = False
self.header_done = False
def set_start_criteria(self, numTweets):
self.maxTweets = numTweets
self.tweetCount = 0
print "Number of tweets to get:", self.maxTweets
def stopStream(self):
self.stop = True
def on_success(self, data):
if 'text' in data:
self.tweetCount += 1
print "tweetCount:", self.tweetCount
#tweet = data['text'].encode('utf-8')
theTweet = data
writer = TweetMonkey()
writer.assignMainWindow(self.main_window, self.header_done)
self.header_done = True
writer.process(theTweet)
# Want to disconnect after the first result?
if self.stop is True or self.tweetCount >= self.maxTweets:
self.disconnect()
def on_error(self, status_code, data):
print status_code, data
class TweetMonkey:
def assignMainWindow(self,the_main_window, is_header_done):
self.main_window = the_main_window
self.header_done = is_header_done
def clean(self,text):
text = text.replace("\n","; ")
text = text.replace('"', "'")
text = text.replace(','," ")
return text
def create_header(self):
header = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
header.append("id")
if self.main_window.tweet_language is True:
header.append("lang")
if self.main_window.tweet_user_name is True:
header.append("user_name")
header.append("tweet")
if self.main_window.tweet_retweeted is True:
header.append("retweeted")
if self.main_window.tweet_fav_count is True:
header.append("favorite_count")
if self.main_window.tweet_source_app is True:
header.append("source")
if self.main_window.tweet_orig_tweet_id is True:
header.append("in_reply_to_status_id")
if self.main_window.tweet_orig_username is True:
header.append("in_reply_to_screen_name")
# header.append("in_reply_to_user_id")
if self.main_window.tweet_sensitive is True:
header.append("possibly_sensitive")
if self.main_window.tweet_geocoordinates is True:
header.append("geo")
if self.main_window.tweet_timestamp is True:
header.append("created_at")
wr.writerow(header)
tweets.close()
def process(self, tweet):
if not self.header_done:
self.create_header()
self.header_done = True
# Create the file or append to the existing
theOutput = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
theOutput.append(tweet['id'])
if self.main_window.tweet_language is True:
theOutput.append(tweet['lang'].encode('utf-8'))
if self.main_window.tweet_user_name is True:
theOutput.append(tweet['user']['name'].encode('utf-8', 'replace'))
theOutput.append(self.clean(tweet['text']).encode('utf-8', 'replace'))
if self.main_window.tweet_retweeted is True:
theOutput.append(tweet['retweeted'])
if self.main_window.tweet_fav_count is True:
theOutput.append(tweet['favorite_count'])
if self.main_window.tweet_source_app is True:
theOutput.append(self.clean(tweet['source']).encode('utf-8', 'replace'))
if self.main_window.tweet_orig_tweet_id is True:
theOutput.append(tweet['in_reply_to_status_id'])
if self.main_window.tweet_orig_username is True:
theOutput.append(tweet['in_reply_to_screen_name'])
#theOutput.append(tweet['in_reply_to_user_id'])
if self.main_window.tweet_sensitive is True:
if tweet.get('possibly_sensitive'):
theOutput.append(tweet['possibly_sensitive'])
else:
theOutput.append("False")
if self.main_window.tweet_geocoordinates is True:
if tweet['geo'] is not None:
if tweet['geo']['type'] == 'Point':
lat = str(tweet['geo']['coordinates'][0]) + " "
lon = str(tweet['geo']['coordinates'][1])
theOutput.append(lat + lon)
else:
theOutput.append(tweet['geo'])
else:
theOutput.append(tweet['geo'])
if self.main_window.tweet_timestamp is True:
theOutput.append(tweet['created_at'])
wr.writerow(theOutput)
tweets.close()
if __name__ == '__main__':
app = QApplication(sys.argv)
frame = MainWindow()
frame.show()
app.exec_()
I know this is an old post but I ran into a similar problem in a simple app I recently wrote, my solution was to use threading.
I used the worker from:
https://pymotw.com/2/threading/
and the method described in:
http://aadrake.com/using-twitter-as-a-stream-processing-source.html
Basically running the Twython stream as a separate thread feeding text to a queue then I run the rest of the program in a separate loop reading from the queue.