Develop Reference

python multithreaded application hanging

I have written a program that I am using to benchmark a mongodb database performing under multithreaded bulk write conditions.
The problem is that the program hangs and does not finish executing.
I am quite sure that the problem is due to writing 530838 records to the database and using 10 threads to bulk write 50 records at a time. This leaves a modulo value of 38 records, however the run method fetches 50 records from the queue so the process hangs when 530800 records have been written and never writes the final 38 records as the following code never finishes executing
for object in range(50):
objects.append(self.queue.get())
I would like the program to write 50 records at a time until fewer than 50 remain at which point it should write the remaining records in the queue and then exit the thread when no records remain in the queue.
Thanks in advance :)
import threading
import Queue
import json
from pymongo import MongoClient, InsertOne
import datetime
#Set the number of threads
n_thread = 10
#Create the queue
queue = Queue.Queue()
#Connect to the database
client = MongoClient("mongodb://mydatabase.com")
db = client.threads
class ThreadClass(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
#Assign thread working with queue
self.queue = queue
def run(self):
while True:
objects = []
#Get next 50 objects from queue
for object in range(50):
objects.append(self.queue.get())
#Insert the queued objects into the database
db.threads.insert_many(objects)
#signals to queue job is done
self.queue.task_done()
#Create number of processes
threads = []
for i in range(n_thread):
t = ThreadClass(queue)
t.setDaemon(True)
#Start thread
t.start()
#Start timer
starttime = datetime.datetime.now()
#Read json object by object
content = json.load(open("data.txt","r"))
for jsonobj in content:
#Put object into queue
queue.put(jsonobj)
#wait on the queue until everything has been processed
queue.join()
for t in threads:
t.join()
#Print the total execution time
endtime = datetime.datetime.now()
duration = endtime-starttime
print(divmod(duration.days * 86400 + duration.seconds, 60))

From the docs on Queue.get you can see that the default settings are block=True and timeout=None, which results in blocked waiting on an empty queue to have a next item that can be taken.
You could use get_nowait or get(False) to ensure you're not blocking. If you want the blocking to be conditional on whether the queue has 50 items, whether it is empty, or other conditions, you can use Queue.empty and Queue.qsize, but note that they do not provide race-condition-proof guarantees of non-blocking behavior... they would merely be heuristics for whether to use block=False with get.
Something like this:
def run(self):
while True:
objects = []
#Get next 50 objects from queue
block = self.queue.qsize >= 50
for i in range(50):
try:
item = self.queue.get(block=block)
except Queue.Empty:
break
objects.append(item)
#Insert the queued objects into the database
db.threads.insert_many(objects)
#signals to queue job is done
self.queue.task_done()
Another approach would be to set timeout and use a try ... except block to catch any Empty exceptions that are raised. This has the advantage that you can decide how long to wait, rather than heuristically guessing when to immediately return, but they are similar.
Also note that I changed your loop variable from object to i ... you should most likely avoid having your loop variable ghost the global object class.

Creating a threaded python queue?

How would I go and create a queue to run tasks in the background in Python?
I have tried via asyncio.Queue() but whenever I use Queue.put(task) it immediately starts the task.
It is for an application which receives an unknown amount of entries (filenames) from a database on a specified time interval. What I wish to accomplish with this backgroundqueue would be that the python application keeps running and keeps returning new filenames. Everytime the application finds new filenames it should handle them by creating a task, which would contain (method(variables)). These tasks should all be thrown into an ever expanding queue which runs the tasks on its own. Here's the code.
class DatabaseHandler:
def __init__(self):
try:
self.cnx = mysql.connector.connect(user='root', password='', host='127.0.0.1', database='mydb')
self.cnx.autocommit = True
self.q = asyncio.Queue()
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
print("Something is wrong with your user name or password")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
print("Database does not exist")
else:
print(err)
self.get_new_entries(30.0)
def get_new_entries(self, delay):
start_time = t.time()
while True:
current_time = datetime.datetime.now() - datetime.timedelta(seconds=delay)
current_time = current_time.strftime("%Y-%m-%d %H:%M:%S")
data = current_time
print(current_time)
self.select_latest_entries(data)
print("###################")
t.sleep(delay - ((t.time() - start_time) % delay))
def select_latest_entries(self, input_data):
query = """SELECT FILE_NAME FROM `added_files` WHERE CREATION_TIME > %s"""
cursor = self.cnx.cursor()
cursor.execute(query, (input_data,))
for file_name in cursor.fetchall():
file_name_string = ''.join(file_name)
self.q.put(self.handle_new_file_names(file_name_string))
cursor.close()
def handle_new_file_names(self, filename):
create_new_npy_files(filename)
self.update_entry(filename)
def update_entry(self, filename):
print(filename)
query = """UPDATE `added_files` SET NPY_CREATED_AT=NOW(), DELETED=1 WHERE FILE_NAME=%s"""
update_cursor = self.cnx.cursor()
self.cnx.commit()
update_cursor.execute(query, (filename,))
update_cursor.close()
As I said, this will instantly run the task.
create_new_npy_files is a pretty time consuming method in a static class.

There are two problems with this expression:
self.q.put(self.handle_new_file_names(file_name_string))
First, it is actually calling the handle_new_file_names method and is enqueueing its result. This is not specific to asyncio.Queue, it is how function calls work in Python (and most mainstream languages). The above is equivalent to:
_tmp = self.handle_new_file_names(file_name_string)
self.q.put(_tmp)
The second problem is that asyncio.Queue operations like get and put are coroutines, so you must await them.
If you want to enqueue a callable, you can use a lambda:
await self.q.put(lambda: self.handle_new_file_names(file_name_string))
But since the consumer of the queue is under your control, you can simply enqueue the file names, as suggested by #dirn:
await self.q.put(file_name_string)
The consumer of the queue would use await self.q.get() to read the file names and call self.handle_new_file_names() on each.
If you plan to use asyncio, consider reading a tutorial that covers the basics, and switching to an asyncio compliant database connector, so that the database queries play along with the asyncio event loop.

For people who see this in the future. The answer I marked as accepted is the explanation of how to solve the problem. I'll write down some code which I used to create what I wanted. That is, tasks that should run in the background. Here you go.
from multiprocessing import Queue
import threading
class ThisClass
def __init__(self):
self.q = Queue()
self.worker = threading.Thread(target=self._consume_queue)
self.worker.start()
self.run()
The queue created is not a queue for tasks, but for the variables you want to handle.
def run(self):
for i in range(100):
self.q.put(i)
Then for the _consume_queue(), which consumes the items in the queue when there are items:
def _consume_queue(self):
while True:
number = self.q.get()
# the logic you want to use per number.
It seems the self.q.get() waits for new entries, even when there are none.
The -simplified- code above works for me, I hope it will also work for others.

Writing an object oriented multithreaded job\result queue in python

long time lurker here.
I have a thread controller object. This object takes in other objects called "Checks". These Checks pull in DB rows that match their criteria. The thread manager polls each check (asking it for it's DB rows aka work units) and then enqueues each row along with a reference to that check object. The thought is that N many threads will come in and pull off an item from the queue and execute the corresponding Check's do_work method. The do_work method will return Pass\Fail and all passes will be enqueued for further processing.
The main script (not shown) instantiates the checks and adds them to the thread manager using add_check and then calls kick_off_work.
So far I am testing and it simply locks up:
import Queue
from threading import Thread
class ThreadMan:
def __init__(self, reporter):
print "Initializing thread manager..."
self.workQueue = Queue.Queue()
self.resultQueue = Queue.Queue()
self.checks = []
def add_check(self, check):
self.checks.append(check)
def kick_off_work(self):
for check in self.checks:
for work_unit in check.populate_work():
#work unit is a DB row
self.workQueue.put({"object" : check, "work" : work_unit})
threads = Thread(target=self.execute_work_unit)
threads = Thread(target=self.execute_work_unit)
threads.start()
self.workQueue.join();
def execute_work_unit(self):
unit = self.workQueue.get()
check_object = unit['object'] #Check object
work_row = unit['work'] # DB ROW
check_object.do_work(work_row)
self.workQueue.task_done();
print "Done with work!!"
The output is simply:
Initializing thread manager...
In check1's do_work method... Doing work
Done with work!!
(locked up)
I would like to run through the entire queue

you should only add a "while" in your execute_work_unit otherwise it stops at first iteration:
def execute_work_unit(self):
while True:
unit = self.workQueue.get()
check_object = unit['object'] #Check object
work_row = unit['work'] # DB ROW
check_object.do_work(work_row)
self.workQueue.task_done();
print "Done with work!!"
have a look there:
http://docs.python.org/2/library/queue.html#module-Queue
EDIT: to get it finish just add threads.join() after your self.workQueue.join() in
def kick_off_work(self):

Python Multiprocessing: Handling Child Errors in Parent

I am currently playing around with multiprocessing and queues.
I have written a piece of code to export data from mongoDB, map it into a relational (flat) structure, convert all values to string and insert them into mysql.
Each of these steps is submitted as a process and given import/export queues, safe for the mongoDB export which is handled in the parent.
As you will see below, I use queues and child processes terminate themselves when they read "None" from the queue. The problem I currently have is that, if a child process runs into an unhandled Exception, this is not recognized by the parent and the rest just Keeps running. What I want to happen is that the whole shebang quits and at best reraise the child error.
I have two questions:
How do I detect the child error in the parent?
How do I kill my child processes after detecting the error (best practice)? I realize that putting "None" to the queue to kill the child is pretty dirty.
I am using python 2.7.
Here are the essential parts of my code:
# Establish communication queues
mongo_input_result_q = multiprocessing.Queue()
mapper_result_q = multiprocessing.Queue()
converter_result_q = multiprocessing.Queue()
[...]
# create child processes
# all processes generated here are subclasses of "multiprocessing.Process"
# create mapper
mappers = [mongo_relational_mapper.MongoRelationalMapper(mongo_input_result_q, mapper_result_q, columns, 1000)
for i in range(10)]
# create datatype converter, converts everything to str
converters = [datatype_converter.DatatypeConverter(mapper_result_q, converter_result_q, 'str', 1000)
for i in range(10)]
# create mysql writer
# I create a list of writers. currently only one,
# but I have the option to parallellize it further
writers = [mysql_inserter.MySqlWriter(mysql_host, mysql_user, mysql_passwd, mysql_schema, converter_result_q
, columns, 'w_'+mysql_table, 1000) for i in range(1)]
# starting mapper
for mapper in mappers:
mapper.start()
time.sleep(1)
# starting converter
for converter in converters:
converter.start()
# starting writer
for writer in writers:
writer.start()
[... initializing mongo db connection ...]
# put each dataset read to queue for the mapper
for row in mongo_collection.find({inc_column: {"$gte": start}}):
mongo_input_result_q.put(row)
count += 1
if count % log_counter == 0:
print 'Mongo Reader' + " " + str(count)
print "MongoReader done"
# Processes are terminated when they read "None" object from queue
# now that reading is finished, put None for each mapper in the queue so they terminate themselves
# the same for all followup processes
for mapper in mappers:
mongo_input_result_q.put(None)
for mapper in mappers:
mapper.join()
for converter in converters:
mapper_result_q.put(None)
for converter in converters:
converter.join()
for writer in writers:
converter_result_q.put(None)
for writer in writers:
writer.join()

Why not to let the Process to take care of its own exceptions, like this:
from __future__ import print_function
import multiprocessing as mp
import traceback
class Process(mp.Process):
def __init__(self, *args, **kwargs):
mp.Process.__init__(self, *args, **kwargs)
self._pconn, self._cconn = mp.Pipe()
self._exception = None
def run(self):
try:
mp.Process.run(self)
self._cconn.send(None)
except Exception as e:
tb = traceback.format_exc()
self._cconn.send((e, tb))
# raise e # You can still rise this exception if you need to
#property
def exception(self):
if self._pconn.poll():
self._exception = self._pconn.recv()
return self._exception
Now you have, both error and traceback at your hands:
def target():
raise ValueError('Something went wrong...')
p = Process(target = target)
p.start()
p.join()
if p.exception:
error, traceback = p.exception
print(traceback)
Regards,
Marek

I don't know standard practice but what I've found is that to have reliable multiprocessing I design the methods/class/etc. specifically to work with multiprocessing. Otherwise you never really know what's going on on the other side (unless I've missed some mechanism for this).
Specifically what I do is:
Subclass multiprocessing.Process or make functions that specifically support multiprocessing (wrapping functions that you don't have control over if necessary)
always provide a shared error multiprocessing.Queue from the main process to each worker process
enclose the entire run code in a try: ... except Exception as e. Then when something unexpected happens send an error package with:
the process id that died
the exception with it's original context (check here). The original context is really important if you want to log useful information in the main process.
of course handle expected issues as normal within the normal operation of the worker
(similar to what you said already) assuming a long-running process, wrap the running code (inside the try/catch-all) with a loop
define a stop token in the class or for functions.
When the main process wants the worker(s) to stop, just send the stop token. to stop everyone, send enough for all the processes.
the wrapping loop checks the input q for the token or whatever other input you want
The end result is worker processes that can survive for a long time and that can let you know what's happening when something goes wrong. They will die quietly since you can handle whatever you need to do after the catch-all exception and you will also know when you need to restart a worker.
Again, I've just come to this pattern through trial and error so I don't know how standard it is. Does that help with what you are asking for?

#mrkwjc 's solution is simple, so easy to understand and implement, but there is one disadvantage of this solution. When we have few processes and we want to stop all processes if any single process has error, we need to wait until all processes are finished in order to check if p.exception. Below is the code which fixes this problem (ie when one child has error, we terminate also another child):
import multiprocessing
import traceback
from time import sleep
class Process(multiprocessing.Process):
"""
Class which returns child Exceptions to Parent.
https://stackoverflow.com/a/33599967/4992248
"""
def __init__(self, *args, **kwargs):
multiprocessing.Process.__init__(self, *args, **kwargs)
self._parent_conn, self._child_conn = multiprocessing.Pipe()
self._exception = None
def run(self):
try:
multiprocessing.Process.run(self)
self._child_conn.send(None)
except Exception as e:
tb = traceback.format_exc()
self._child_conn.send((e, tb))
# raise e # You can still rise this exception if you need to
#property
def exception(self):
if self._parent_conn.poll():
self._exception = self._parent_conn.recv()
return self._exception
class Task_1:
def do_something(self, queue):
queue.put(dict(users=2))
class Task_2:
def do_something(self, queue):
queue.put(dict(users=5))
def main():
try:
task_1 = Task_1()
task_2 = Task_2()
# Example of multiprocessing which is used:
# https://eli.thegreenplace.net/2012/01/16/python-parallelizing-cpu-bound-tasks-with-multiprocessing/
task_1_queue = multiprocessing.Queue()
task_2_queue = multiprocessing.Queue()
task_1_process = Process(
target=task_1.do_something,
kwargs=dict(queue=task_1_queue))
task_2_process = Process(
target=task_2.do_something,
kwargs=dict(queue=task_2_queue))
task_1_process.start()
task_2_process.start()
while task_1_process.is_alive() or task_2_process.is_alive():
sleep(10)
if task_1_process.exception:
error, task_1_traceback = task_1_process.exception
# Do not wait until task_2 is finished
task_2_process.terminate()
raise ChildProcessError(task_1_traceback)
if task_2_process.exception:
error, task_2_traceback = task_2_process.exception
# Do not wait until task_1 is finished
task_1_process.terminate()
raise ChildProcessError(task_2_traceback)
task_1_process.join()
task_2_process.join()
task_1_results = task_1_queue.get()
task_2_results = task_2_queue.get()
task_1_users = task_1_results['users']
task_2_users = task_2_results['users']
except Exception:
# Here usually I send email notification with error.
print('traceback:', traceback.format_exc())
if __name__ == "__main__":
main()

Thanks to kobejohn i have found a solution which is nice and stable.
I have created a subclass of multiprocessing.Process which implements some functions and overwrites the run() method to wrap a new saferun method into a try-catch block. This Class requires a feedback_queue to initialize which is used to report info, debug, error messages back to the parent. The log methods in the class are wrappers for the globally defined log functions of the package:
class EtlStepProcess(multiprocessing.Process):
def __init__(self, feedback_queue):
multiprocessing.Process.__init__(self)
self.feedback_queue = feedback_queue
def log_info(self, message):
log_info(self.feedback_queue, message, self.name)
def log_debug(self, message):
log_debug(self.feedback_queue, message, self.name)
def log_error(self, err):
log_error(self.feedback_queue, err, self.name)
def saferun(self):
"""Method to be run in sub-process; can be overridden in sub-class"""
if self._target:
self._target(*self._args, **self._kwargs)
def run(self):
try:
self.saferun()
except Exception as e:
self.log_error(e)
raise e
return
I have subclassed all my other process steps from EtlStepProcess. The code to be run is implemented in the saferun() method rather than run. This ways i do not have to add a try catch block around it, since this is already done by the run() method.
Example:
class MySqlWriter(EtlStepProcess):
def __init__(self, mysql_host, mysql_user, mysql_passwd, mysql_schema, mysql_table, columns, commit_count,
input_queue, feedback_queue):
EtlStepProcess.__init__(self, feedback_queue)
self.mysql_host = mysql_host
self.mysql_user = mysql_user
self.mysql_passwd = mysql_passwd
self.mysql_schema = mysql_schema
self.mysql_table = mysql_table
self.columns = columns
self.commit_count = commit_count
self.input_queue = input_queue
def saferun(self):
self.log_info(self.name + " started")
#create mysql connection
engine = sqlalchemy.create_engine('mysql://' + self.mysql_user + ':' + self.mysql_passwd + '#' + self.mysql_host + '/' + self.mysql_schema)
meta = sqlalchemy.MetaData()
table = sqlalchemy.Table(self.mysql_table, meta, autoload=True, autoload_with=engine)
connection = engine.connect()
try:
self.log_info("start MySQL insert")
counter = 0
row_list = []
while True:
next_row = self.input_queue.get()
if isinstance(next_row, Terminator):
if counter % self.commit_count != 0:
connection.execute(table.insert(), row_list)
# Poison pill means we should exit
break
row_list.append(next_row)
counter += 1
if counter % self.commit_count == 0:
connection.execute(table.insert(), row_list)
del row_list[:]
self.log_debug(self.name + ' ' + str(counter))
finally:
connection.close()
return
In my main file, I submit a Process that does all the work and give it a feedback_queue. This process starts all the steps and thenreads from mongoDB and puts values to the initial queue. My main process listens to the feedback queue and prints all log messages. If it receives an error log, it print the error and terminate its child, which in return also terminates all its children before dying.
if __name__ == '__main__':
feedback_q = multiprocessing.Queue()
p = multiprocessing.Process(target=mongo_python_export, args=(feedback_q,))
p.start()
while p.is_alive():
fb = feedback_q.get()
if fb["type"] == "error":
p.terminate()
print "ERROR in " + fb["process"] + "\n"
for child in multiprocessing.active_children():
child.terminate()
else:
print datetime.datetime.fromtimestamp(fb["timestamp"]).strftime('%Y-%m-%d %H:%M:%S') + " " + \
fb["process"] + ": " + fb["message"]
p.join()
I think about making a module out of it and putting it up on github, but I have to do some cleaning up and commenting first.

Python Queues memory leaks when called inside thread

I have python TCP client and need to send media(.mpg) file in a loop to a 'C' TCP server.
I have following code, where in separate thread I am reading the 10K blocks of file and sending it and doing it all over again in loop, I think it is because of my implementation of thread module, or tcp send. I am using Queues to print the logs on my GUI ( Tkinter ) but after some times it goes out of memory..
UPDATE 1 - Added more code as requested
Thread class "Sendmpgthread" used to create thread to send data
.
.
def __init__ ( self, otherparams,MainGUI):
.
.
self.MainGUI = MainGUI
self.lock = threading.Lock()
Thread.__init__(self)
#This is the one causing leak, this is called inside loop
def pushlog(self,msg):
self.MainGUI.queuelog.put(msg)
def send(self, mysocket, block):
size = len(block)
pos = 0;
while size > 0:
try:
curpos = mysocket.send(block[pos:])
except socket.timeout, msg:
if self.over:
self.pushlog(Exit Send)
return False
except socket.error, msg:
print 'Exception'
return False
pos = pos + curpos
size = size - curpos
return True
def run(self):
media_file = None
mysocket = None
try:
mysocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mysocket.connect((self.ip, string.atoi(self.port)))
media_file = open(self.file, 'rb')
while not self.over:
chunk = media_file.read(10000)
if not chunk: # EOF Reset it
print 'resetting stream'
media_file.seek(0, 0)
continue
if not self.send(mysocket, chunk): # If some error or thread is killed
break;
#disabling this solves the issue
self.pushlog('print how much data sent')
except socket.error, msg:
print 'print exception'
except Exception, msg:
print 'print exception'
try:
if media_file is not None:
media_file.close()
media_file = None
if mysocket is not None:
mysocket.close()
mysocket = None
finally:
print 'some cleaning'
def kill(self):
self.over = True
I figured out that it is because of wrong implementation of Queue as commenting that piece resolves the issue
UPDATE 2 - MainGUI class which is called from above Thread class
class MainGUI(Frame):
def __init__(self, other args):
#some code
.
.
#from the above thread class used to send data
self.send_mpg_status = Sendmpgthread(params)
self.send_mpg_status.start()
self.after(100, self.updatelog)
self.queuelog = Queue.Queue()
def updatelog(self):
try:
msg = self.queuelog.get_nowait()
while msg is not None:
self.printlog(msg)
msg = self.queuelog.get_nowait()
except Queue.Empty:
pass
if self.send_mpg_status: # only continue when sending
self.after(100, self.updatelog)
def printlog(self,msg):
#print in GUI

Since printlog is adding to a tkinter text control, the memory occupied by that control will grow with each message (it has to store all the log messages in order to display them).
Unless storing all the logs is critical, a common solution is to limit the maximum number of log lines displayed.
A naive implementation is to eliminate extra lines from the begining after the control reaches a maximum number of messages. Add a function to get the number of lines in the control and then, in printlog something similar to:
while getnumlines(self.edit) > self.maxloglines:
self.edit.delete('1.0', '1.end')
(above code not tested)
update: some general guidelines
Keep in mind that what might look like a memory leak does not always mean that a function is wrong, or that the memory is no longer accessible. Many times there is missing cleanup code for a container that is accumulating elements.
A basic general approach for this kind of problems:
form an opinion on what part of the code might be causing the problem
check it by commenting that code out (or keep commenting code until you find a candidate)
look for containers in the responsible code, add code to print their size
decide what elements can be safely removed from that container, and when to do it
test the result

I can't see anything obviously wrong with your code snippet.
To reduce memory usage a bit under Python 2.7, I'd use buffer(block, pos) instead of block[pos:]. Also I'd use mysocket.sendall(block) instead of your send method.
If the ideas above don't solve your problem, then the bug is most probably elsewhere in your code. Could you please post the shortest possible version of the full Python script which still grows out-of-memory (http://sscce.org/)? That increases your change of getting useful help.

Out of memory errors are indicative of data being generated but not consumed or released. Looking through your code I would guess these two areas:
Messages are being pushed onto a Queue.Queue() instance in the pushlog method. Are they being consumed?
The MainGui printlog method may be writing text somewhere. eg. Is it continually writing to some kind of GUI widget without any pruning of messages?
From the code you've posted, here's what I would try:
Put a print statement in updatelog. If this is not being continually called for some reason such as a failed after() call, then the queuelog will continue to grow without bound.
If updatelog is continually being called, then turn your focus to printlog. Comment the contents of this function to see if out of memory errors still occur. If they don't, then something in printlog may be holding on to the logged data, you'll need to dig deeper to find out what.
Apart from this, the code could be cleaned up a bit. self.queuelog is not created until after the thread is started which gives rise to a race condition where the thread may try to write into the queue before it has been created. Creation of queuelog should be moved to somewhere before the thread is started.
updatelog could also be refactored to remove redundancy:
def updatelog(self):
try:
while True:
msg = self.queuelog.get_nowait()
self.printlog(msg)
except Queue.Empty:
pass
And I assume the the kill function is called from the GUI thread. To avoid thread race conditions, the self.over should be a thread safe variable such as a threading.Event object.
def __init__(...):
self.over = threading.Event()
def kill(self):
self.over.set()

There is no data piling up in your TCP sending loop.
Memory error is probably caused by logging queue, as you have not posted complete code try using following class for logging:
from threading import Thread, Event, Lock
from time import sleep, time as now
class LogRecord(object):
__slots__ = ["txt", "params"]
def __init__(self, txt, params):
self.txt, self.params = txt, params
class AsyncLog(Thread):
DEBUGGING_EMULATE_SLOW_IO = True
def __init__(self, queue_max_size=15, queue_min_size=5):
Thread.__init__(self)
self.queue_max_size, self.queue_min_size = queue_max_size, queue_min_size
self._queuelock = Lock()
self._queue = [] # protected by _queuelock
self._discarded_count = 0 # protected by _queuelock
self._pushed_event = Event()
self.setDaemon(True)
self.start()
def log(self, message, **params):
with self._queuelock:
self._queue.append(LogRecord(message, params))
if len(self._queue) > self.queue_max_size:
# empty the queue:
self._discarded_count += len(self._queue) - self.queue_min_size
del self._queue[self.queue_min_size:] # empty the queue instead of creating new list (= [])
self._pushed_event.set()
def run(self):
while 1: # no reason for exit condition here
logs, discarded_count = None, 0
with self._queuelock:
if len(self._queue) > 0:
# select buffered messages for printing, releasing lock ASAP
logs = self._queue[:]
del self._queue[:]
self._pushed_event.clear()
discarded_count = self._discarded_count
self._discarded_count = 0
if not logs:
self._pushed_event.wait()
self._pushed_event.clear()
continue
else:
# print logs
if discarded_count:
print ".. {0} log records missing ..".format(discarded_count)
for log_record in logs:
self.write_line(log_record)
if self.DEBUGGING_EMULATE_SLOW_IO:
sleep(0.5)
def write_line(self, log_record):
print log_record.txt, " ".join(["{0}={1}".format(name, value) for name, value in log_record.params.items()])
if __name__ == "__main__":
class MainGUI:
def __init__(self):
self._async_log = AsyncLog()
self.log = self._async_log.log # stored as bound method
def do_this_test(self):
print "I am about to log 100 times per sec, while text output frequency is 2Hz (twice per second)"
def log_100_records_in_one_second(itteration_index):
for i in xrange(100):
self.log("something happened", timestamp=now(), session=3.1415, itteration=itteration_index)
sleep(0.01)
for iter_index in range(3):
log_100_records_in_one_second(iter_index)
test = MainGUI()
test.do_this_test()
I have noticed that you do not sleep() anywhere in the sending loop, this means data is read as fast as it can and is sent as fast as it can. Note that this is not desirable behavior when playing media files - container time-stamps are there to dictate data-rate.