I have a dummy example, I want to apply multiprocessing in it. Consider a scenario where you have a stream of numbers(which I call frame) incoming one by one. And I want to assign it to any single process that is available currently. So I am creating 4 processes that are running a while loop, seeing if any element in queue, than apply function on it.
The problem is that when I join it, it gets stuck in any while loop, even though I close the while loop before it. But somehow it gets stuck inside it.
Code:
# step 1, 4 processes
import multiprocessing as mp
import os
import time
class MpListOperations:
def __init__(self):
self.results_queue = mp.Manager().Queue()
self.frames_queue = mp.Manager().Queue()
self.flag = mp.Manager().Value(typecode='b',value=True)
self.list_nums = list(range(0,5000))
def process_list(self):
print(f"Process id {os.getpid()} started")
while self.flag.value:
# print(self.flag.value)
if self.frames_queue.qsize():
self.results_queue.put(self.frames_queue.get()**2)
def create_processes(self, no_of_processes = mp.cpu_count()):
print("Creating Processes")
self.processes = [mp.Process(target=self.process_list) for _ in range(no_of_processes)]
def start_processes(self):
print(f"starting processes")
for process in self.processes:
process.start()
def join_process(self):
print("Joining Processes")
while True:
if not self.frames_queue.qsize():
self.flag.value=False
print("JOININNG HERE")
for process in self.processes:
exit_code = process.join()
print(exit_code)
print("BREAKING DONE")
break
def stream_frames(self):
print("Streaming Frames")
for frame in self.list_nums:
self.frames_queue.put(frame)
if __name__=="__main__":
start = time.time()
mp_ops = MpListOperations()
mp_ops.create_processes()
mp_ops.start_processes()
mp_ops.stream_frames()
mp_ops.join_process()
print(time.time()-start)
Now if I add a timeout parameter in join, even 0, i.e exit_code = process.join(0) it works. I want to understand in this scenario, if this code is correct, what should be the value of timeout? Why is it working with timeout and not without it? What is the proper way to implement multiprocessing with it?
If you look at the documentation for a managed queue you will see that the qsize method only returns an approximate size. I would therefore not use it for testing when all the items have been taken of the frames queue. Presumably you want to let the processes run until all frames have been processed. The simplest way I know would be to put N sentinel items on the frames queue after the actual frames have been put where N is the number of processes getting from the queue. A sentinel item is a special value that cannot be mistaken for an actual frame and signals to the process that there are no more items for it to get from the queue (i.e. a quasi end-of-file item). In this case we can use None as the sentinel items. Each process then just continues to do get operations on the queue until it sees a sentinel item and then terminates. There is therefore no need for the self.flag attribute.
Here is the updated and simplified code. I have made some other minor changes that have been commented:
import multiprocessing as mp
import os
import time
class MpListOperations:
def __init__(self):
# Only create one manager process:
manager = mp.Manager()
self.results_queue = manager.Queue()
self.frames_queue = manager.Queue()
# No need to convert range to a list:
self.list_nums = range(0, 5000)
def process_list(self):
print(f"Process id {os.getpid()} started")
while True:
frame = self.frames_queue.get()
if frame is None: # Sentinel?
# Yes, we are done:
break
self.results_queue.put(frame ** 2)
def create_processes(self, no_of_processes = mp.cpu_count()):
print("Creating Processes")
self.no_of_processes = no_of_processes
self.processes = [mp.Process(target=self.process_list) for _ in range(no_of_processes)]
def start_processes(self):
print("Starting Processes")
for process in self.processes:
process.start()
def join_processes(self):
print("Joining Processes")
for process in self.processes:
# join returns None:
process.join()
def stream_frames(self):
print("Streaming Frames")
for frame in self.list_nums:
self.frames_queue.put(frame)
# Put sentinels:
for _ in range(self.no_of_processes):
self.frames_queue.put(None)
if __name__== "__main__":
start = time.time()
mp_ops = MpListOperations()
mp_ops.create_processes()
mp_ops.start_processes()
mp_ops.stream_frames()
mp_ops.join_processes()
print(time.time()-start)
Prints:
Creating Processes
Starting Processes
Process id 28 started
Process id 29 started
Streaming Frames
Process id 33 started
Process id 31 started
Process id 38 started
Process id 44 started
Process id 42 started
Process id 45 started
Joining Processes
2.3660173416137695
Note for Windows
I have modified method start_processes to temporarily set attribute self.processes to None:
def start_processes(self):
print("Starting Processes")
processes = self.processes
# Don't try to pickle list of processes:
self.processes = None
for process in processes:
process.start()
# Restore attribute:
self.processes = processes
Otherwise under Windows we get a pickle error trying to serialize/deserialize a list of processes containing two or more multiprocessing.Process instances. The error is "TypeError: cannot pickle 'weakref' object." This can be demonstrated with the following code where we first try to pickle a list of 1 process and then a list of 2 processes:
import multiprocessing as mp
import os
class Foo:
def __init__(self, number_of_processes):
self.processes = [mp.Process(target=self.worker) for _ in range(number_of_processes)]
self.start_processes()
self.join_processes()
def start_processes(self):
processes = self.processes
for process in self.processes:
process.start()
def join_processes(self):
for process in self.processes:
process.join()
def worker(self):
print(f"Process id {os.getpid()} started")
print(f"Process id {os.getpid()} ended")
if __name__== "__main__":
foo = Foo(1)
foo = Foo(2)
Prints:
Process id 7540 started
Process id 7540 ended
Traceback (most recent call last):
File "C:\Booboo\test\test.py", line 26, in <module>
foo = Foo(2)
File "C:\Booboo\test\test.py", line 7, in __init__
self.start_processes()
File "C:\Booboo\test\test.py", line 13, in start_processes
process.start()
File "C:\Program Files\Python38\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Program Files\Python38\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Program Files\Python38\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "C:\Program Files\Python38\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Program Files\Python38\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Process id 18152 started
Process id 18152 ended
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Program Files\Python38\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Program Files\Python38\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
The target loop is stuck in the get() method of your loop. This is because multiple processes could see that the queue wasn't empty, but only 1 of them was able to get the last item. The remaining processes are waiting for the next item to be available from the queue.
You might need to add a Lock when you are reading the size of the Queue object And getting the object of that queue.
Or alternatively, you avoid reading the size of the queue by simply using the queue.get() method with a timeout that allows us to check the flag regularly
import queue
TIMEOUT = 1 # seconds
class MpListOperations:
#[...]
def process_list(self):
print(f"Process id {os.getpid()} started")
previous = self.flag.value
while self.flag.value:
try:
got = self.frames_queue.get(timeout=TIMEOUT)
except queue.Empty:
pass
else:
print(f"Gotten {got}")
self.results_queue.put(got**2)
_next = self.flag.value
if previous != _next:
print(f"Flag change: {_next}")
$ python ./test_mp.py
Creating Processes
starting processes
Process id 36566 started
Streaming Frames
Process id 36565 started
Process id 36564 started
Process id 36570 started
Process id 36567 started
Gotten 0
Process id 36572 started
Gotten 1
Gotten 2
Gotten 3
Process id 36579 started
Gotten 4
Gotten 5
Gotten 6
Process id 36583 started
Gotten 7
# [...]
Gotten 4997
Joining Processes
Gotten 4998
Gotten 4999
JOININNG HERE
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
BREAKING DONE
1.4375360012054443
Alternatively, using a multiprocessing.Pool object:
def my_func(arg):
time.sleep(0.002)
return arg**2
def get_input():
for i in range(5000):
yield i
time.sleep(0.001)
if __name__=="__main__":
start = time.time()
mp_pool = mp.Pool()
result = mp_pool.map(my_func, get_input())
mp_pool.close()
mp_pool.join()
print(len(result))
print(f"Duration: {time.time()-start}")
Giving:
$ python ./test_mp.py
5000
Duration: 6.847279787063599
I have a pyglet program that runs a GUI in one thread and receives a message from another thread. It's supposed to update the GUI based on that message. The problem though is whenever I go to update a GUI element, I get an error message like this one:
Traceback (most recent call last):
File --- line 40, in run
pub.sendMessage("update", msg="Banana")
File --- line 77, in updateDisplay
self.label.text = msg
File "---\pyglet\text\__init__.py", line 289, in text
self.document.text = text
File "---\pyglet\text\document.py", line 294, in text
self.insert_text(0, text)
File "---\pyglet\text\document.py", line 425, in insert_text
self.dispatch_event('on_insert_text', start, text)
File "---\pyglet\event.py", line 408, in dispatch_event
if handler(*args):
File "---\pyglet\text\layout.py", line 1045, in on_insert_text
self._init_document()
File "---\pyglet\text\layout.py", line 1034, in _init_document
self._update()
File "---\pyglet\text\layout.py", line 957, in _update
lines = self._get_lines()
File "---\pyglet\text\layout.py", line 933, in _get_lines
glyphs = self._get_glyphs()
...
File "---\pyglet\font\base.py", line 246, in fit
region.blit_into(image, 0, 0, 0)
File "---\pyglet\image\__init__.py", line 1730, in blit_into
self.owner.blit_into(source, x + self.x, y + self.y, z + self.z)
File "---\pyglet\image\__init__.py", line 1622, in blit_into
glBindTexture(self.target, self.id)
File "---\pyglet\gl\lib.py", line 107, in errcheck
raise GLException(msg)
pyglet.gl.lib.GLException: b'invalid operation'
The operative error being pyglet.gl.lib.GLException: b'invalid operation'
I know self.label.text = Y works because it updates the text if I put it in the init() function. However, trying to do any kind of update from the update_display function (see code below) throws this error. I know the function gets called correctly because it successfully prints the message to console, but it breaks on the update element line. I've tried other elements than label and it's the same thing.
I have no idea why it's throwing this error or how to resolve it. Any ideas?
My code is:
import pyglet
from threading import Thread
from random import choice
from time import sleep
from pubsub import pub
RUN = True
class MessageThread(Thread):
def __init__(self):
"""Init Thread Class."""
Thread.__init__(self)
self.start() # start the thread
def run(self):
"""Run Thread."""
array = [0, 1, 2]
while True:
if RUN == False:
print("End runner")
break
value = choice(array)
sleep(1)
if value == 0:
print("event", value, ": Apple", flush=True)
pub.sendMessage("update", msg="Apple")
elif value == 1:
print("event", value, ": Banana", flush=True)
pub.sendMessage("update", msg="Banana")
else:
print("event", value, ": Carrot", flush=True)
pub.sendMessage("update", msg="Carrot")
class MyGUI(pyglet.window.Window):
# Note: With this structure, there is no "self.app", you just use
# self. This structure allows onDraw() to be called.
def __init__(self):
super(MyGUI, self).__init__(width=600,height=650,resizable=True)
self.label = pyglet.text.Label('Waiting...',
font_name='Times New Roman',
font_size=36,
x=self.width/2, y=self.height - 36,
anchor_x='center', anchor_y='center')
# create a pubsub receiver
# update is the topic subscribed to, updateDisplay the callable
pub.subscribe(self.updateDisplay, "update")
MessageThread()
def on_draw(self):
self.clear()
self.label.draw()
def updateDisplay(self, msg):
""" Receives data from thread and updates the display """
print ("Updating Display")
self.label.text = msg
def on_close(self):
""" Override the normal onClose behavior so that we can
kill the MessageThread, too
"""
print("Closing Application")
global RUN
RUN = False
self.close() # Need to include since defining on_close overrides it
if __name__ == "__main__":
window = MyGUI()
pyglet.app.run()
I am running a web server on a raspberry pi which is logging temperatures etc.
I am using websockets in Tornado to communicate with my client.
I want the client to be able to control when the server is sending data over the socket.
My thought was that when client connects and says it's ready, the server will start a loop where it logs temps every second. But I need this loop to run asynchronously. This is where I get in trouble. I tried to follow an example, but I cant get it to run properly.
class TemperatureSocketHandler(tornado.websocket.WebSocketHandler):
#gen.coroutine
def async_func(self):
num = 0
while(self.sending):
num = num + 1
temp = self.sense.get_temperature()
yield self.write_message(str(temp))
gen.sleep(1)
def open(self):
print("Temperature socket opened")
self.sense = SenseHat()
self.sense.clear()
self.sending = False
def on_message(self, message):
if(message == "START"):
self.sending = True
if(message == "STOP"):
self.sending = False
tornado.ioloop.IOLoop.current().spawn_callback(self.async_func(self))
But I get an error here when I run this:
ERROR:tornado.application:Exception in callback functools.partial(<function wrap.<locals>.null_wrapper at 0x75159858>)
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.5/site-packages/tornado/ioloop.py", line 605, in _run_callback
ret = callback()
File "/home/pi/.local/lib/python3.5/site-packages/tornado/stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
TypeError: 'Future' object is not callable
You have to use IOLoop.add_future()
since async_func() returns a Future (it is decorated as a coroutine!).
Also, you should add the future, when you receive the start message, not on any message:
def on_message(self, message):
if(message == "START"):
self.sending = True
tornado.ioloop.IOLoop.current().add_future(
self.async_func(self), lambda f: self.close())
if(message == "STOP"):
self.sending = False
Just to give some context: I'm currently learning Python and to do so I started a little Project.
I wrote two Python Scripts: a Host and a Client script.
I'm already at the point where multiple Clients can connect to one "Host" and the Host can send a random String to all Clients.
Now I wanted to solve the problem, that if a Client disconnects from the Host nobody knows until the CLIENTSOCKET is called again.
So I wrote the checkConnection method to ping all Clients every 5 sec, for now.
class Connection():
def __init__(self):
self.s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
def bound(self,RHOST,PORT):
self.s.bind((RHOST,PORT))
self.s.listen(1000)
def connect(self):
((CLIENTSOCKET,ADRESS)) = self.s.accept()
return (CLIENTSOCKET,ADRESS)
def newConnection(Connection,CLIENTS):
con = Connection
cli = CLIENTS
while True:
c = con.connect()
if c != None:
if len(cli) != 0:
for x in range (len(cli)):
if c[1][0] == cli[x][1][0]:
pass
else:
cli.append(c)
else:
cli.append(c)
def checkConnection(CLIENTS):
cli = CLIENTS
while True:
if(len(cli)!=0):
for x in range(len(cli)):
try:
cli[x][0].sendall(b'')
except:
del cli[x]
time.sleep(5)
I'm creating to Threads with each method as a target and give them the needed parameters. The first thread starts like it should, but the second doesn't.
I really don't know why?
t = threading.Thread(target = newConnection,name = "newConnection",args = (g,CLIENTS))
t2 = threading.Thread(target = checkConnection,name = "checkConnection",args = (CLIENTS))
t.start()
t2.start()
Exception in thread checkConnection:
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python35-32\lib\threading.py", line 914, in _bootstrap_inner
self.run()
File "C:\Users\User\AppData\Local\Programs\Python\Python35-32\lib\threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
TypeError: checkConnection() missing 1 required positional argument: 'CLIENTS'
I wrote a simple Tkinter based Python application that reads text from a serial connection and adds it to the window, specifically a text widged.
After a lot of tweaks and some very strange exceptions, this works. Then I added autoscrolling by doing this:
self.text.insert(END, str(parsed_line))
self.text.yview(END)
These lines run in a thread. The thread blocks on reading fromt the serial connection, splits lines and then adds all lines to the widget.
This works, too. Then I wanted to allow the user to scroll which should disable auto-scroll until the user scrolls back to the bottom.
I found this
Stop Text widget from scrolling when content is changed
which seems to be related. Especially, I tried the code from DuckAssasin's comment:
if self.myWidgetScrollbar.get() == 1.0:
self.myWidget.yview(END)
I also tried .get()[1] which is actually the element I want (bottom position). However, this crashes with the following exception:
Traceback (most recent call last):
File "transformer-gui.py", line 119, in run
pos = self.scrollbar.get()[1]
File "C:\Python26\lib\lib-tk\Tkinter.py", line 2809, in get
return self._getdoubles(self.tk.call(self._w, 'get'))
File "C:\Python26\lib\lib-tk\Tkinter.py", line 1028, in _getdoubles
return tuple(map(getdouble, self.tk.splitlist(string)))
ValueError: invalid literal for float(): None
It seems as if tkinter somewhere returns None which then is being parsed as a float. I read somewhere, that e.g. the index method of the text widged sometimes returnes None if the requested location is not visible.
Hopefully, anybody can help me out with this problem!
[EDIT]
Ok, I have assembled a demo script that can reproduce this issue on my Win XP machine:
import re,sys,time
from Tkinter import *
import Tkinter
import threading
import traceback
class ReaderThread(threading.Thread):
def __init__(self, text, scrollbar):
print "Thread init"
threading.Thread.__init__(self)
self.text = text
self.scrollbar = scrollbar
self.running = True
def stop(self):
print "Stopping thread"
running = False
def run(self):
print "Thread started"
time.sleep(5)
i = 1
try:
while(self.running):
# emulating delay when reading from serial interface
time.sleep(0.05)
line = "the quick brown fox jumps over the lazy dog\n"
curIndex = "1.0"
lowerEdge = 1.0
pos = 1.0
# get cur position
pos = self.scrollbar.get()[1]
# Disable scrollbar
self.text.configure(yscrollcommand=None, state=NORMAL)
# Add to text window
self.text.insert(END, str(line))
startIndex = repr(i) + ".0"
curIndex = repr(i) + ".end"
# Perform colorization
if i % 6 == 0:
self.text.tag_add("warn", startIndex, curIndex)
elif i % 6 == 1:
self.text.tag_add("debug", startIndex, curIndex)
elif i % 6 == 2:
self.text.tag_add("info", startIndex, curIndex)
elif i % 6 == 3:
self.text.tag_add("error", startIndex, curIndex)
elif i % 6 == 4:
self.text.tag_add("fatal", startIndex, curIndex)
i = i + 1
# Enable scrollbar
self.text.configure(yscrollcommand=self.scrollbar.set, state=DISABLED)
# Auto scroll down to the end if scroll bar was at the bottom before
# Otherwise allow customer scrolling
if pos == 1.0:
self.text.yview(END)
#if(lowerEdge == 1.0):
# print "is lower edge!"
#self.text.see(curIndex)
#else:
# print "Customer scrolling", lowerEdge
# Get current scrollbar position before inserting
#(upperEdge, lowerEdge) = self.scrollbar.get()
#print upperEdge, lowerEdge
#self.text.update_idletasks()
except Exception as e:
traceback.print_exc(file=sys.stdout)
print "Exception in receiver thread, stopping..."
pass
print "Thread stopped"
class Transformer:
def __init__(self):
pass
def start(self):
"""starts to read linewise from self.in_stream and parses the read lines"""
count = 1
root = Tk()
root.title("Tkinter Auto-Scrolling Test")
topPane = PanedWindow(root, orient=HORIZONTAL)
topPane.pack(side=TOP, fill=X)
lowerPane = PanedWindow(root, orient=VERTICAL)
scrollbar = Scrollbar(root)
scrollbar.pack(side=RIGHT, fill=Y)
text = Text(wrap=WORD, yscrollcommand=scrollbar.set)
scrollbar.config(command=text.yview)
# Color definition for log levels
text.tag_config("debug",foreground="gray50")
text.tag_config("info",foreground="green")
text.tag_config("warn",foreground="orange")
text.tag_config("error",foreground="red")
text.tag_config("fatal",foreground="#8B008B")
# set default color
text.config(background="black", foreground="gray");
text.pack(expand=YES, fill=BOTH)
lowerPane.add(text)
lowerPane.pack(expand=YES, fill=BOTH)
t = ReaderThread(text, scrollbar)
print "Starting thread"
t.start()
try:
root.mainloop()
except Exception as e:
print "Exception in window manager: ", e
t.stop()
t.join()
if __name__ == "__main__":
try:
trans = Transformer()
trans.start()
except Exception as e:
print "Error: ", e
sys.exit(1)
I let this scipt run and start to scroll up and down and after some time I get a lot of always different exceptions such as:
.\source\testtools\device-log-transformer>python tkinter-autoscroll.py
Thread init
Starting thread
Thread started
Traceback (most recent call last):
File "tkinter-autoscroll.py", line 59, in run
self.text.configure(yscrollcommand=self.scrollbar.set, state=DISABLED)
File "C:\Python26\lib\lib-tk\Tkinter.py", line 1202, in configure
Stopping thread
return self._configure('configure', cnf, kw)
File "C:\Python26\lib\lib-tk\Tkinter.py", line 1193, in _configure
self.tk.call(_flatten((self._w, cmd)) + self._options(cnf))
TclError: invalid command name ".14762592"
Exception in receiver thread, stopping...
Thread stopped
.\source\testtools\device-log-transformer>python tkinter-autoscroll.py
Thread init
Starting thread
Thread started
Stopping thread
Traceback (most recent call last):
File "tkinter-autoscroll.py", line 35, in run
pos = self.scrollbar.get()[1]
File "C:\Python26\lib\lib-tk\Tkinter.py", line 2809, in get
return self._getdoubles(self.tk.call(self._w, 'get'))
TclError: invalid command name ".14762512"
Exception in receiver thread, stopping...
Thread stopped
.\source\testtools\device-log-transformer>python tkinter-autoscroll.py
Thread init
Starting thread
Thread started
Traceback (most recent call last):
File "tkinter-autoscroll.py", line 65, in run
self.text.yview(END)
File "C:\Python26\lib\lib-tk\Tkinter.py", line 3156, in yview
self.tk.call((self._w, 'yview') + what)
Stopping threadTclError: invalid command name ".14762592"
Exception in receiver thread, stopping...
Thread stopped
.\source\testtools\device-log-transformer>python tkinter-autoscroll.py
Thread init
Starting thread
Thread started
Traceback (most recent call last):
File "tkinter-autoscroll.py", line 35, in run
pos = self.scrollbar.get()[1]
File "C:\Python26\lib\lib-tk\Tkinter.py", line 2809, in get
return self._getdoubles(self.tk.call(self._w, 'get'))
File "C:\Python26\lib\lib-tk\Tkinter.py", line 1028, in _getdoubles
return tuple(map(getdouble, self.tk.splitlist(string)))
ValueError: invalid literal for float(): None
Exception in receiver thread, stopping...
Thread stopped
Stopping thread
.\source\testtools\device-log-transformer>python tkinter-autoscroll.py
Thread init
Starting thread
Thread started
Traceback (most recent call last):
File "tkinter-autoscroll.py", line 53, in run
self.text.tag_add("error", startIndex, curIndex)
File "C:\Python26\lib\lib-tk\Tkinter.py", line 3057, in tag_add
(self._w, 'tag', 'add', tagName, index1) + args)
TclError: bad option "261.0": must be bbox, cget, compare, configure, count, debug, delete, dlineinfo, dump, edit, get, image, index, insert, mark, pe
er, replace, scan, search, see, tag, window, xview, or yview
Exception in receiver thread, stopping...
Thread stopped
I hope this helps you to help me :)
Thanks,
/J
It's hard to tell what's really going on but have you considered using a Queue?
from Tkinter import *
import time, Queue, thread
def simulate_input(queue):
for i in range(100):
info = time.time()
queue.put(info)
time.sleep(0.5)
class Demo:
def __init__(self, root, dataQueue):
self.root = root
self.dataQueue = dataQueue
self.text = Text(self.root, height=10)
self.scroller = Scrollbar(self.root, command=self.text.yview)
self.text.config(yscrollcommand=self.scroller.set)
self.text.tag_config('newline', background='green')
self.scroller.pack(side='right', fill='y')
self.text.pack(fill='both', expand=1)
self.root.after_idle(self.poll)
def poll(self):
try:
data = self.dataQueue.get_nowait()
except Queue.Empty:
pass
else:
self.text.tag_remove('newline', '1.0', 'end')
position = self.scroller.get()
self.text.insert('end', '%s\n' %(data), 'newline')
if (position[1] == 1.0):
self.text.see('end')
self.root.after(1000, self.poll)
q = Queue.Queue()
root = Tk()
app = Demo(root, q)
worker = thread.start_new_thread(simulate_input, (q,))
root.mainloop()
Regarding your demo script.
You're doing GUI stuff from the non-GUI thread. That tends to cause problems.
see: http://www.effbot.org/zone/tkinter-threads.htm
OK,
based on the valuable suggestions by noob oddy I was able to rewrite the example script by using the Tkinter.generate_event() method to generate asynchronous event and a queue to pass the information.
Every time a line is read from the stream (which is simulated by a constant string and a delay), I append the line to a queue (because passing objects to the event method is not supported AFAIK) and then create a new event.
The event callback method retrieves the message from the queue and adds it to the Text widged. This works because this method is called from the Tkinter mainloop an thus it cannot interfere with the other jobs.
Here is the script:
import re,sys,time
from Tkinter import *
import Tkinter
import threading
import traceback
import Queue
class ReaderThread(threading.Thread):
def __init__(self, root, queue):
print "Thread init"
threading.Thread.__init__(self)
self.root = root
self.running = True
self.q = queue
def stop(self):
print "Stopping thread"
running = False
def run(self):
print "Thread started"
time.sleep(5)
try:
while(self.running):
# emulating delay when reading from serial interface
time.sleep(0.05)
curline = "the quick brown fox jumps over the lazy dog\n"
try:
self.q.put(curline)
self.root.event_generate('<<AppendLine>>', when='tail')
# If it failed, the window has been destoyed: over
except TclError as e:
print e
break
except Exception as e:
traceback.print_exc(file=sys.stdout)
print "Exception in receiver thread, stopping..."
pass
print "Thread stopped"
class Transformer:
def __init__(self):
self.q = Queue.Queue()
self.lineIndex = 1
pass
def appendLine(self, event):
line = self.q.get_nowait()
if line == None:
return
i = self.lineIndex
curIndex = "1.0"
lowerEdge = 1.0
pos = 1.0
# get cur position
pos = self.scrollbar.get()[1]
# Disable scrollbar
self.text.configure(yscrollcommand=None, state=NORMAL)
# Add to text window
self.text.insert(END, str(line))
startIndex = repr(i) + ".0"
curIndex = repr(i) + ".end"
# Perform colorization
if i % 6 == 0:
self.text.tag_add("warn", startIndex, curIndex)
elif i % 6 == 1:
self.text.tag_add("debug", startIndex, curIndex)
elif i % 6 == 2:
self.text.tag_add("info", startIndex, curIndex)
elif i % 6 == 3:
self.text.tag_add("error", startIndex, curIndex)
elif i % 6 == 4:
self.text.tag_add("fatal", startIndex, curIndex)
i = i + 1
# Enable scrollbar
self.text.configure(yscrollcommand=self.scrollbar.set, state=DISABLED)
# Auto scroll down to the end if scroll bar was at the bottom before
# Otherwise allow customer scrolling
if pos == 1.0:
self.text.yview(END)
self.lineIndex = i
def start(self):
"""starts to read linewise from self.in_stream and parses the read lines"""
count = 1
self.root = Tk()
self.root.title("Tkinter Auto-Scrolling Test")#
self.root.bind('<<AppendLine>>', self.appendLine)
self.topPane = PanedWindow(self.root, orient=HORIZONTAL)
self.topPane.pack(side=TOP, fill=X)
self.lowerPane = PanedWindow(self.root, orient=VERTICAL)
self.scrollbar = Scrollbar(self.root)
self.scrollbar.pack(side=RIGHT, fill=Y)
self.text = Text(wrap=WORD, yscrollcommand=self.scrollbar.set)
self.scrollbar.config(command=self.text.yview)
# Color definition for log levels
self.text.tag_config("debug",foreground="gray50")
self.text.tag_config("info",foreground="green")
self.text.tag_config("warn",foreground="orange")
self.text.tag_config("error",foreground="red")
self.text.tag_config("fatal",foreground="#8B008B")
# set default color
self.text.config(background="black", foreground="gray");
self.text.pack(expand=YES, fill=BOTH)
self.lowerPane.add(self.text)
self.lowerPane.pack(expand=YES, fill=BOTH)
t = ReaderThread(self.root, self.q)
print "Starting thread"
t.start()
try:
self.root.mainloop()
except Exception as e:
print "Exception in window manager: ", e
t.stop()
t.join()
if __name__ == "__main__":
try:
trans = Transformer()
trans.start()
except Exception as e:
print "Error: ", e
sys.exit(1)
Thanks again to everybody who contributed for your help!