Develop Reference

using select stdin for non blocking input python

I want to create a process that has 3 sub processes, 2 to handle websockets and one to get input from terminal to pass to the other two sockets.
import sys
import time
import select
import asyncio
import threading
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Process
def managment_api():
poller = select.poll()
poller.register(sys.stdin, select.POLLIN)
started = True
count = 0
while started:
print("management api [{:^6}]".format(count))
while sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
_line = sys.stdin.readline()
if _line:
line = _line.strip().lower()
if(line == "exit" or line == "quit"):
started = False
print("got exit message [{}]".format(started))
else:
pass
count += 1
time.sleep(1)
def process_main(loop):
print("process_main BEGIN")
loop.run_in_executor(None, managment_api())
print("process_main ENG ")
if __name__ == "__main__":
#this doesn't work
main = Process(target=managment_api, args=())
main.name = "management api"
main.start()
main.join()
"""
#asyncio.run(managment_api()) #need to add async to management_api
When I make management_api an async function and call asyncio.run(management_api); I can get input.
If I try to run the same code without async in a separate process it, I get stuck in the while sys.stdin in selec... section of the code. I've tried with threads but that doesn't work either.
How can I run this code from a separate process, to get the input in another process?

I was able to solve the problem by first, using fn = sys.stdin.fileno() to get the main process file descriptor, passing that as an argument to the subprocess. Then using sys.stdin = os.fdopen(fn)
import sys
import time
import select
import asyncio
import threading
import multiprocessing as mp
from multiprocessing import Queue
from multiprocessing import Process
def managment_api(fn):
sys.stdin = os.fdopen(fn)
poller = select.poll()
poller.register(sys.stdin, select.POLLIN)
started = True
count = 0
while started:
print("management api [{:^6}]".format(count))
while sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
_line = sys.stdin.readline()
if _line:
line = _line.strip().lower()
if(line == "exit" or line == "quit"):
started = False
print("got exit message [{}]".format(started))
else:
pass
count += 1
time.sleep(1)
def process_main(loop):
print("process_main BEGIN")
loop.run_in_executor(None, managment_api())
print("process_main ENG ")
if __name__ == "__main__":
fn = sys.stdin.fileno()
main = Process(target=managment_api, args=(fn, ))
main.name = "management api"
main.start()
main.join()
"""
#asyncio.run(managment_api()) #need to add async to management_api

Can't read input.txt file correctly using a multithreaded scraper in Python

I use the input file IG#Input.txt which contains:
qwdlab
qwdwe
The script would get all usernames from these Instagram hashtags but it's not working if there is more than 1 hashtag in the input.txt file. What do I need to change to make it work correctly along with the multithreading?
The script workflow would be like:
Reading hashtags/lines from input.txt file
Scraping them multithreaded
Printing the output into the outputIG.txt file
Code:
import threading
from instaloader import Instaloader
import time
#Classes for threading
class LockedIterator(object):
def __init__(self, it):
self.lock = threading.Lock()
self.it = it.__iter__()
def __iter__(self):
return self
def __next__(self):
self.lock.acquire()
try:
return self.it.__next__()
finally:
self.lock.release()
# input()
input = open("IG#Input.txt","r",encoding='utf-8')
HASHTAG = input.read()
p = HASHTAG.split('\n')
PROFILE = p[:]
posts = Instaloader(sleep=False).get_hashtag_posts(HASHTAG)
posts = LockedIterator(posts)
output = open("outputIG.txt","w+")
# Main
def worker():
for ind in range(len(PROFILE)):
pro = PROFILE[ind]
try:
for post in posts:
print(post.owner_username)
output.write(post.owner_username + '\n')
except Exception as e:
print(e)
raise
#Start Threading
threads = []
for i in range(4):
t = threading.Thread(target=worker)
threads.append(t)
t.start()
for t in threads:
t.join()
output.close()

Capture and display console output of function at the same time

Right now I use this to catch the output of a Python function and store it in a variable:
import io
from contextlib import redirect_stdout
def catch_output(func):
result = io.StringIO()
with redirect_stdout(result):
func()
return result.getvalue()
output = catch_output(my_func)
This works fine, but it also mutes the console until the func call finished.
Does anybody know if I can write/pipe the live output of the func to the console and store it in a variable at the same time?

You can redirect stdout to a custom file-like object that forwards writes to multiple files:
import contextlib
import io
import sys
class TeeIO:
def __init__(self, original, target):
self.original = original
self.target = target
def write(self, b):
self.original.write(b)
self.target.write(b)
#contextlib.contextmanager
def tee_stdout(target):
tee = TeeIO(sys.stdout, target)
with contextlib.redirect_stdout(tee):
yield
buf = io.StringIO()
with tee_stdout(buf):
print("foo")
print(buf.getvalue())

This is what I ended up using. I thought I leave this here for people who have a hard time with classes and oop, like me.
import sys
import io
from contextlib import redirect_stdout
def get_multi_writer(streams):
writer = type('obj', (object,), {})
writer.write = lambda s: [stream.write(s) for stream in streams]
return writer
def catch_output(func, args, kwargs):
streams = [sys.stdout, io.StringIO()]
with redirect_stdout(get_multi_writer(streams)):
func(*args, **kwargs)
return streams[1].getvalue()
print(catch_output(my_func, [], {}))

As per the suggestions from the comments I've made and example turning our function into a thread so we can simultaneously check for output from that function periodically and copy it to the real stdout.
import sys
import time
import threading
from cStringIO import StringIO
def foo(n):
for x in range(n):
time.sleep(1) #intense computation
print('test: {}'.format(n))
#i'm using python 2.7 so I don't have contextlib.redirect_stdout
realstdout = sys.stdout
sys.stdout = StringIO()
t = threading.Thread(target=foo, args=(10,))
t.start()
lastpos = 0 #last cursor position in file
while True:
t.join(.1) #wait .1 sec for thread to complete
if sys.stdout.tell() != lastpos: #data has been written to stdout
sys.stdout.seek(lastpos) #go back to our last position
realstdout.write(sys.stdout.read()) #read the data to the real stdout
lastpos = sys.stdout.tell() #update lastpos
if not t.is_alive(): #when we're done
break
sys.stdout.seek(0) #seek back to beginning of file
output = sys.stdout.read() #copy to a usable variable
sys.stdout = realstdout #reset stdout

How do I use multiple threads with PySide and Twython?

I'm trying to write a small python app, using PySide for the GUI and Twython as a Twitter API library, to catch a stream from Twitter.
The problem that I am having is that when I click "Start Monitoring Twitter" button, the UI freezes until the stream is complete, at which point the code continues to execute and disables the Start button and enables the Stop button. Here's the UI:
Everything else seems to work -- if I leave it, then the CSV file is created as I suspect -- the Twython components seem to be working as expected.
Line 151 is where the streaming from Twitter is engaged when I click start:
self.stream.statuses.filter(track=self.search_term)
How can I move the streaming to a separate thread and then use the Stop button on the UI to tell Twython to complete capturing the stream and exit?
I need to be able to send the MyStreamer instance to another thread and then send it the .disconnect() signal to have it terminate capturing the stream.
Here's the full code:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import platform
import PySide
from PySide.QtGui import QApplication, QMainWindow, QPushButton, QCheckBox, QTextEdit
from time import sleep
from ui_tweetstream import Ui_MainWindow
from twython import Twython
from twython import TwythonStreamer
import csv
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
# Set up Variables
self.tweet_fav_count = True
self.tweet_geocoordinates = True
self.tweet_id = True
self.tweet_language = True
self.tweet_orig_tweet_id = True
self.tweet_orig_username = True
self.tweet_retweeted = True
self.tweet_sensitive = True
self.tweet_source_app = True
self.tweet_timestamp = True
self.tweet_user_name = True
self.search_term = "#bigdata"
self.tweets_to_get = 1000
# Bind the interface
self.check_tweet_fav_count.clicked.connect(self.setTweetFavCount)
self.check_tweet_geocoordinates.clicked.connect(self.setTweetGeocoordinates)
self.check_tweet_id.clicked.connect(self.setTweetID)
self.check_tweet_language.clicked.connect(self.setTweetLanguage)
self.check_tweet_orig_tweet_id.clicked.connect(self.setTweetOrigTweetID)
self.check_tweet_orig_username.clicked.connect(self.setTweetOrigUsername)
self.check_tweet_retweeted.clicked.connect(self.setTweetRetweeted)
self.check_tweet_sensitive.clicked.connect(self.setTweetSensitive)
self.check_tweet_source_app.clicked.connect(self.setTweetSourceApp)
self.check_tweet_timestamp.clicked.connect(self.setTweetTimestamp)
self.check_tweet_user_name.clicked.connect(self.setTweetUsername)
self.button_start.clicked.connect(self.streamStart)
self.button_stop.clicked.connect(self.streamStop)
# Set the initial states
self.button_stop.setEnabled(False)
APP_KEY = ''
APP_SECRET = ''
OAUTH_TOKEN = ''
OAUTH_TOKEN_SECRET = ''
self.t = Twython(APP_KEY, APP_SECRET, OAUTH_TOKEN, OAUTH_TOKEN_SECRET)
self.stream = MyStreamer(APP_KEY,APP_SECRET,OAUTH_TOKEN,OAUTH_TOKEN_SECRET)
self.stream.init_mainWindow(self)
def streamStop(self):
print "Stopping stream"
# Enable other controls here
self.button_stop.setEnabled(False)
self.button_start.setEnabled(True)
self.setControlStates(True)
self.stream.stopStream()
def setControlStates(self, state):
self.check_tweet_fav_count.setEnabled(state)
self.check_tweet_geocoordinates.setEnabled(state)
self.check_tweet_id.setEnabled(state)
self.check_tweet_language.setEnabled(state)
self.check_tweet_orig_tweet_id.setEnabled(state)
self.check_tweet_orig_username.setEnabled(state)
self.check_tweet_retweeted.setEnabled(state)
self.check_tweet_sensitive.setEnabled(state)
self.check_tweet_source_app.setEnabled(state)
self.check_tweet_timestamp.setEnabled(state)
self.check_tweet_user_name.setEnabled(state)
self.search_box.setEnabled(state)
self.num_tweets_box.setEnabled(state)
# Functions for determining what to track
def setTweetFavCount(self):
self.tweet_fav_count = not self.tweet_fav_count
print "tweet_fav_count:", self.tweet_fav_count
def setTweetGeocoordinates(self):
self.tweet_geocoordinates = not self.tweet_geocoordinates
print "tweet_geocoordinates:", self.tweet_geocoordinates
def setTweetID(self):
self.tweet_id = not self.tweet_id
print "tweet_id:", self.tweet_id
def setTweetLanguage(self):
self.tweet_language = not self.tweet_language
print "tweet_language:", self.tweet_language
def setTweetOrigTweetID(self):
self.tweet_orig_tweet_id = not self.tweet_orig_tweet_id
print "tweet_orig_tweet_id:", self.tweet_orig_tweet_id
def setTweetOrigUsername(self):
self.tweet_orig_username = not self.tweet_orig_tweet_id
print "tweet_orig_username:", self. tweet_orig_username
def setTweetRetweeted(self):
self.tweet_retweeted = not self.tweet_retweeted
print "tweet_retweeted:", self.tweet_retweeted
def setTweetSensitive(self):
self.tweet_sensitive = not self.tweet_sensitive
print "tweet_sensitive:", self.tweet_sensitive
def setTweetSourceApp(self):
self.tweet_source_app = not self.tweet_source_app
print "tweet_source_app:", self.tweet_source_app
def setTweetTimestamp(self):
self.tweet_timestamp = not self.tweet_timestamp
print "tweet_timestamp:", self.tweet_timestamp
def setTweetUsername(self):
self.tweet_user_name = not self.tweet_user_name
print "tweet_user_name:", self.tweet_user_name
# Functions for starting and stopping the stream
def streamStart(self):
print "Starting stream"
self.setControlStates(False)
# Disable other controls here
self.button_start.setEnabled(False)
self.button_stop.setEnabled(True)
# Hack to try to disable the UI
# sleep(0.25)
# Get the active search term
self.search_term = self.search_box.text()
# Get the number of tweets
self.tweets_to_get = int(self.num_tweets_box.text())
# Set the streamer
self.stream.set_start_criteria(self.tweets_to_get)
self.stream.statuses.filter(track=self.search_term)
class MyStreamer(TwythonStreamer):
def init_mainWindow(self, the_main_window):
self.main_window = the_main_window
self.stop = False
self.header_done = False
def set_start_criteria(self, numTweets):
self.maxTweets = numTweets
self.tweetCount = 0
print "Number of tweets to get:", self.maxTweets
def stopStream(self):
self.stop = True
def on_success(self, data):
if 'text' in data:
self.tweetCount += 1
print "tweetCount:", self.tweetCount
#tweet = data['text'].encode('utf-8')
theTweet = data
writer = TweetMonkey()
writer.assignMainWindow(self.main_window, self.header_done)
self.header_done = True
writer.process(theTweet)
# Want to disconnect after the first result?
if self.stop is True or self.tweetCount >= self.maxTweets:
self.disconnect()
def on_error(self, status_code, data):
print status_code, data
class TweetMonkey:
def assignMainWindow(self,the_main_window, is_header_done):
self.main_window = the_main_window
self.header_done = is_header_done
def clean(self,text):
text = text.replace("\n","; ")
text = text.replace('"', "'")
text = text.replace(','," ")
return text
def create_header(self):
header = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
header.append("id")
if self.main_window.tweet_language is True:
header.append("lang")
if self.main_window.tweet_user_name is True:
header.append("user_name")
header.append("tweet")
if self.main_window.tweet_retweeted is True:
header.append("retweeted")
if self.main_window.tweet_fav_count is True:
header.append("favorite_count")
if self.main_window.tweet_source_app is True:
header.append("source")
if self.main_window.tweet_orig_tweet_id is True:
header.append("in_reply_to_status_id")
if self.main_window.tweet_orig_username is True:
header.append("in_reply_to_screen_name")
# header.append("in_reply_to_user_id")
if self.main_window.tweet_sensitive is True:
header.append("possibly_sensitive")
if self.main_window.tweet_geocoordinates is True:
header.append("geo")
if self.main_window.tweet_timestamp is True:
header.append("created_at")
wr.writerow(header)
tweets.close()
def process(self, tweet):
if not self.header_done:
self.create_header()
self.header_done = True
# Create the file or append to the existing
theOutput = []
tweets = open("tweets.csv", 'ab+')
wr = csv.writer(tweets, dialect='excel')
if self.main_window.tweet_id is True:
theOutput.append(tweet['id'])
if self.main_window.tweet_language is True:
theOutput.append(tweet['lang'].encode('utf-8'))
if self.main_window.tweet_user_name is True:
theOutput.append(tweet['user']['name'].encode('utf-8', 'replace'))
theOutput.append(self.clean(tweet['text']).encode('utf-8', 'replace'))
if self.main_window.tweet_retweeted is True:
theOutput.append(tweet['retweeted'])
if self.main_window.tweet_fav_count is True:
theOutput.append(tweet['favorite_count'])
if self.main_window.tweet_source_app is True:
theOutput.append(self.clean(tweet['source']).encode('utf-8', 'replace'))
if self.main_window.tweet_orig_tweet_id is True:
theOutput.append(tweet['in_reply_to_status_id'])
if self.main_window.tweet_orig_username is True:
theOutput.append(tweet['in_reply_to_screen_name'])
#theOutput.append(tweet['in_reply_to_user_id'])
if self.main_window.tweet_sensitive is True:
if tweet.get('possibly_sensitive'):
theOutput.append(tweet['possibly_sensitive'])
else:
theOutput.append("False")
if self.main_window.tweet_geocoordinates is True:
if tweet['geo'] is not None:
if tweet['geo']['type'] == 'Point':
lat = str(tweet['geo']['coordinates'][0]) + " "
lon = str(tweet['geo']['coordinates'][1])
theOutput.append(lat + lon)
else:
theOutput.append(tweet['geo'])
else:
theOutput.append(tweet['geo'])
if self.main_window.tweet_timestamp is True:
theOutput.append(tweet['created_at'])
wr.writerow(theOutput)
tweets.close()
if __name__ == '__main__':
app = QApplication(sys.argv)
frame = MainWindow()
frame.show()
app.exec_()

I know this is an old post but I ran into a similar problem in a simple app I recently wrote, my solution was to use threading.
I used the worker from:
https://pymotw.com/2/threading/
and the method described in:
http://aadrake.com/using-twitter-as-a-stream-processing-source.html
Basically running the Twython stream as a separate thread feeding text to a queue then I run the rest of the program in a separate loop reading from the queue.

Set attribute on a python thread from traits

I am new to python and I am implementing a simple serial adquisition in a thread.
I can adquire the data using a class by
class CaptureAngles(threading.Thread, port)
def __init__(self):
threading.Thread.__init__(self)
self.port_name = port
...
def run():
self.connect(self.port_name)
...
However, to better integrate with a graphical interface using the traits library I wrote the code as the following, which is no longer working. I am not able to define the attribute of a thread that is started from traits, what am I doing wrong?
This is the error reported
AttributeError: 'CaptureAngles' object has no attribute 'port_name'
And this the full code:
from threading import Thread
from time import sleep
from enthought.traits.api import *
from enthought.traits.ui.api import View, Item, ButtonEditor
from Queue import Queue
class TextDisplay(HasTraits):
string = String()
view= View( Item('string',show_label=False, springy=True, style='custom' ))
class CaptureAngles(Thread):
self.port_name = String('COM5')
def connect(self, port_name):
self.port = serial.Serial(
port = port_name,
baudrate = 9600,
)
self.display.string='Opening Serial Port...' + self.display.string
self.port.close()
self.port.open()
def run(self):
self.connect(self.port_name)
self.display.string = 'Arduino started\n' + self.display.string
self.port.flushInput()
self.port.flushOutput()
self.port.readline() # Discard first package (can be corrupt)
while not self.wants_abort:
rcv = self.port.readline() # Read the data and split into words
angle = int(rcv)
self.display.string = '%d angle captured\n' % n_img \
+ self.display.string
self.close()
def close(self):
self.port.close()
self.display.string='...Serial Port Closed!' + self.display.string
class Arduino(HasTraits):
start_stop_capture = Button()
display = Instance(TextDisplay)
capture_angles = Instance(CaptureAngles)
capture_angles.port_name = 'COM5'
view = View(Item('start_stop_capture', show_label=False ))
def _start_stop_capture_fired(self):
if self.capture_angles and self.capture_angles.isAlive():
self.capture_angles.wants_abort = True
else:
self.capture_angles = CaptureAngles()
self.capture_angles.wants_abort = False
self.capture_angles.display = self.display
self.capture_angles.start()
class MainWindow(HasTraits):
display = Instance(TextDisplay, ())
arduino = Instance(Arduino)
def _arduino_default(self):
return Arduino(display=self.display)
view = View('arduino','display', style="custom", resizable=True)
if __name__ == '__main__':
MainWindow().configure_traits()

Ok, I got it: I was adding the attribute port_name before creating the instance.
class Arduino(HasTraits):
start_stop_capture = Button()
display = Instance(TextDisplay)
capture_angles = Instance(CaptureAngles)
capture_angles.port_name = 'COM5' # <-- wrong: the object is not created yet
...
instead of:
def _start_stop_capture_fired(self):
if self.capture_angles and self.capture_angles.isAlive():
self.capture_angles.wants_abort = True
else:
self.capture_angles = CaptureAngles()
self.capture_angles.port_name = 'COM5' # <-- correct
...