I've got a daemon that runs a number of child processes intended to maintain a telnet connection to collect data from a bunch of weather stations. I've set it up so that these child processes read from that telnet connection forever, passing the weather readings back to the parent process via a multiprocessing.Queue. I can't seem to get these child processes to exit cleanly when I stop the daemon with ./test.py stop. Is there an easy way to close the child processes on exit? A quick google mentioned someone using multiprocessing.Event, what's the best way to set this event on exit to ensure the processes exit? Here's our current code:
from daemon import runner
from multiprocessing import Process, Queue
import telnetlib
from django.utils.encoding import force_text
from observations.weather.models import WeatherStation
import os
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
def read_weather_data(name, ip_address, port, queue):
print "Started process to get data for", name
client = telnetlib.Telnet(ip_address, port)
while True:
response = client.read_until('\r\n'.encode('utf8'))
queue.put((name, force_text(response)))
client.close()
class App(object):
def __init__(self):
self.stdin_path = '/dev/null'
self.stdout_path = '/dev/tty'
self.stderr_path = '/dev/tty'
self.pidfile_path = '/tmp/process_weather.pid'
self.pidfile_timeout = 5
def run(self):
queue = Queue()
for station in WeatherStation.objects.filter(active=True):
p = Process(target=read_weather_data,
args=(station.name, station.ip_address, station.port,
queue,))
p.start()
while True:
name, data = queue.get()
print "Received data from ", name
print data
app = App()
daemon_runner = runner.DaemonRunner(app)
daemon_runner.do_action()
Seem to have found a way to do this, but am unsure about whether this is the best approach to take.
from daemon import runner
from multiprocessing import Process, Queue, Event
import telnetlib
from django.utils.encoding import force_text
from observations.weather.models import WeatherStation
import os
import signal
import errno
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
def read_weather_data(name, ip_address, port, queue, exit):
print "Started process to get data for", name
client = telnetlib.Telnet(ip_address, port)
while not exit.is_set():
response = client.read_until('\r\n'.encode('utf8'))
queue.put((name, force_text(response)))
print "exit called for", name
client.close()
def exit_handler(signum, frame):
print "exiting..."
class App(object):
def __init__(self):
self.stdin_path = '/dev/null'
self.stdout_path = '/dev/tty'
self.stderr_path = '/dev/tty'
self.pidfile_path = '/tmp/process_weather.pid'
self.pidfile_timeout = 5
def run(self):
exit = Event()
def exit_handler(signum, frame):
print "exiting..."
exit.set()
signal.signal(signal.SIGTERM, exit_handler)
queue = Queue()
workers = []
for station in WeatherStation.objects.filter(active=True):
p = Process(target=read_weather_data,
args=(station.name, station.ip_address, station.port,
queue, exit))
workers.append(p)
for worker in workers:
worker.start()
while True:
try:
name, data = queue.get()
except IOError as e:
# we received a signal whilst waiting for I/O
if e.errno != errno.EINTR:
raise
else:
break
print "Received data from ", name
print data
for worker in workers:
worker.join()
app = App()
daemon_runner = runner.DaemonRunner(app)
daemon_runner.do_action()
Related
I have a queue set up like this. I want to run through all the items in a DB query and pass them into my Downloader class. My DB connection keeps going away and eventually my program just dies because I think too many threads are open?
I get an error: Segmentation fault: 11
There are 100K+ items.
How can I properly fix this to process a few items at a time and speed up the process?
class Downloader(threading.Thread):
"""Threaded File Downloader"""
def __init__(self, queue, db):
threading.Thread.__init__(self)
self.queue = queue
self.db = db
def remove_unicode(self, title):
try:
return unicodedata.normalize('NFKD', title).encode('ascii','ignore')
except:
return title
def run(self):
while True:
# gets the url from the queue
row = self.queue.get()
title = row[0]
etc...
def main(urls):
queue = Queue.Queue()
# create a thread pool and give them a queue
for i in range(5):
t = Downloader(queue, db)
t.setDaemon(True)
t.start()
# give the queue some data
i = 1
for url in urls:
print i
queue.put(url)
i+=1
# wait for the queue to finish
queue.join()
if __name__ == "__main__":
db = DatabaseUtil()
sql = 'SELECT `Title`, `Site` from `XYZ`'
titles = db.query(sql)
main(titles)
I have an app with some subprocess running and I have successfully set the sys.excepthook exception handling for the main process. Now, I want to set it for the same hook on the subprocesses. I would expect it to be as simple copying the exact lines of code I used on the main process and that's it but it didn't work.
Next is my code:
class Consumer(multiprocessing.Process):
def __init__(self, codec_status_queue, logger_queue):
multiprocessing.Process.__init__(self)
self.codec_status_queue = codec_status_queue
self.logger_queue = logger_queue
def run(self):
# Set default unhandled exceptions handler
uncaughtErrorHandler = UncaughtErrorHandler(self.logger_queue)
sys.excepthook = uncaughtErrorHandler.error_handler
1/0
class UncaughtErrorHandler(object):
def __init__(self, logger_queue, child_processes=None):
self.logger_queue = logger_queue
self.child_processes = child_processes
def error_handler(self, type, value, trace_back):
trace_formatted = "".join(traceback.format_tb(trace_back))
exeption_message = "Unhandled Exception:\n Type: %s\n Value: %s\n Line: %s\n Traceback:\n %s" % (type, value.message, trace_back.tb_lineno, trace_formatted)
logger_queue.put((LoggerThread.CRITICAL, exeption_message))
if self.child_processes:
self.stop_children()
# Stopping this process
sys.exit()
def stop_children(self):
num_children = len(self.child_processes)
logger_queue.put((LoggerThread.DEBUG, "Terminating child processes (%s)" % num_children))
for process in self.child_processes:
log_message = "Terminating %s with PID %s" % (process.name, process.pid)
logger_queue.put((LoggerThread.DEBUG, log_message))
process.terminate()
if __name__ == '__main__':
...
# Create processes and communication queues
codec_status_queue = multiprocessing.Queue()
num_consumers = multiprocessing.cpu_count() * 2
print 'Creating %d consumers' % num_consumers
consumers = [ Consumer(codec_status_queue, logger_queue)
for i in xrange(num_consumers) ]
# Set default unhandled exceptions handler
uncaughtErrorHandler = UncaughtErrorHandler(logger_queue, consumers)
sys.excepthook = uncaughtErrorHandler.error_handler
# Start processes
for consumer in consumers:
consumer.daemon = True
consumer.start()
If I put the 1/0 on the __main__ part the UncaughtErrorHandler catches the exception but when the 1/0 is put as shown above, it doesn't.
Maybe someone can tell me what am I doing wrong?
The following code was written for Python 3.x but can be adapted to work with Python 3.x instead. It provides an alternative solution to overriding sys.excepthook in child processes. A simple fix involves catching all exceptions and handing the data from sys.exc_info over to the exception handler. The main process could use a similar pattern for exceptions but retains the original design from your program. The example shown below should be a full working demonstration you can play around with and adapt to your needs.
#! /usr/bin/env python3
import logging
import multiprocessing
import queue
import sys
import threading
import time
import traceback
def main():
"""Demonstrate exception handling and logging in several processes."""
logger_queue = multiprocessing.Queue()
logger_thread = LoggerThread(logger_queue)
logger_thread.start()
try:
# Create processes and communication queues
codec_status_queue = multiprocessing.Queue()
num_consumers = multiprocessing.cpu_count() * 2
print('Creating {} consumers'.format(num_consumers))
consumers = [Consumer(codec_status_queue, logger_queue)
for _ in range(num_consumers)]
# Set default unhandled exceptions handler
uncaught_error_handler = UncaughtErrorHandler(logger_queue, consumers)
sys.excepthook = uncaught_error_handler.error_handler
# Start processes
for consumer in consumers:
consumer.start()
time.sleep(2)
finally:
logger_thread.shutdown()
def get_message(value):
"""Retrieve an exception's error message and return it."""
if hasattr(value, 'message'):
return value.message
if hasattr(value, 'args') and value.args:
return value.args[0]
class LoggerThread(threading.Thread):
"""Handle logging messages coming from various sources via a queue."""
CRITICAL = logging.CRITICAL
DEBUG = logging.DEBUG
def __init__(self, logger_queue):
"""Initialize an instance of the LoggerThread class."""
super().__init__()
self.logger_queue = logger_queue
self.mutex = threading.Lock()
self.running = False
def run(self):
"""Process messages coming through the queue until shutdown."""
self.running = True
while self.running:
try:
while True:
self.handle_message(*self.logger_queue.get(True, 0.1))
except queue.Empty:
pass
def handle_message(self, level, message):
"""Show the message while ensuring a guaranteed order on screen."""
with self.mutex:
print('Level:', level)
print('Message:', message)
print('=' * 80, flush=True)
def shutdown(self):
"""Signal the thread to exit once it runs out of messages."""
self.running = False
class Consumer(multiprocessing.Process):
"""Simulate a consumer process that handles data from a queue."""
def __init__(self, codec_status_queue, logger_queue):
"""Initialize an instance of the Consumer class."""
super().__init__()
self.codec_status_queue = codec_status_queue
self.logger_queue = logger_queue
self.daemon = True
def run(self):
"""Begin working as a consumer while handling any exceptions."""
# Set default unhandled exceptions handler
uncaught_error_handler = UncaughtErrorHandler(self.logger_queue)
try:
self.do_consumer_work()
except:
uncaught_error_handler.error_handler(*sys.exc_info())
def do_consumer_work(self):
"""Pretend to be doing the work of a consumer."""
junk = 1 / 0
print('Process', self.ident, 'calculated', junk)
class UncaughtErrorHandler:
"""Organize error handling to automatically terminate child processes."""
def __init__(self, logger_queue, child_processes=None):
"""Initialize an instance of the UncaughtErrorHandler class."""
self.logger_queue = logger_queue
self.child_processes = child_processes
def error_handler(self, kind, value, trace_back):
"""Record errors as they happen and terminate the process tree."""
trace_formatted = ''.join(traceback.format_tb(trace_back))
exception_message = ('Unhandled Exception:\n'
' Type: {}\n'
' Value: {}\n'
' Line: {}\n'
' Traceback:\n{}').format(
kind, get_message(value), trace_back.tb_lineno, trace_formatted)
self.logger_queue.put((LoggerThread.CRITICAL, exception_message))
if self.child_processes:
self.stop_children()
# Stopping this process
sys.exit()
def stop_children(self):
"""Terminate all children associated with this error handler."""
num_children = len(self.child_processes)
log_message = 'Terminating child processes({})'.format(num_children)
self.logger_queue.put((LoggerThread.DEBUG, log_message))
for process in self.child_processes:
log_message = 'Terminating {} with PID {}'.format(
process.name, process.pid)
self.logger_queue.put((LoggerThread.DEBUG, log_message))
process.terminate()
if __name__ == '__main__':
main()
I wrote a Threading class which tests whether a webserver is up or not.
import urllib
import threading
import time
import Queue
class Thread_CheckDeviceState(threading.Thread):
def __init__(self, device_ip, queue, inter=0.1):
self._run = True
self._codes = {}
self._queue = queue
self._device_ip = device_ip
self._inter = inter
self._elapsed = 0
threading.Thread.__init__(self)
def stop(self):
self._run = False
def run(self):
start = time.time()
while self._run:
try:
code = urllib.urlopen(self._device_ip).getcode()
except Exception:
code = "nope"
finally:
measure = time.time()
self._elapsed += measure-start
print self._elapsed, code
self._codes.update(
{self._elapsed:code}
)
time.sleep(self._inter)
self._queue.put(self._codes)
q = Queue.Queue()
thread = Thread_CheckDeviceState("http://192.168.1.3", q)
thread.start()
time.sleep(10)
thread.stop()
print q.get()
It works fine - until I disconnect my pc from the network. From that moment on the thread just does nothing until it is stopped. I would expect it to just continue and set the code to "nope", like I wrote it in the exception handler. Why doesn't it work
You need to use urllib2 instead, and specify a timeout parameter when you call urlopen().
I have simple script for watchdog on network device. Script monitors response from PING command. If there is no answer then second thread executes and first thread is stopped. If second thread is finished then first thread is resumed (checking ping). If there is no answer then following message appears:
RuntimeError: threads can only be started once
Here is my code:
#!/usr/bin/python
import os
import time
import sqlite3
from ablib import Pin
import threading
led=Pin('W9','OUTPUT')
class threadout1(threading.Thread):
def run(self):
while True:
conn = sqlite3.connect('database/database.db')
cur = conn.cursor()
cur.execute("SELECT * FROM watchdog")
rows_output = cur.fetchall()
time.sleep(1)
if rows_output[0][1] == "ping":
response = os.system("ping -c 1 " + rows_output[0][2])
if response != 0:
print "bad"
rest.start()
rest.join()
class restart(threading.Thread):
def run(self):
led.on()
time.sleep(15)
led.off()
thr = threadout1()
rest = restart()
thr.start()
You can either create the restart thread every time you need it
if response != 0:
print "bad"
restart_thread = restart()
restart_thread.start()
restart_thread.join()
or use Events
class restart_thread(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
self.evt.wait()
# do stuff
self.evt.clear()
class threadout(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
if #other thread needs to run once
self.evt.set()
evt = threading.Event()
restart_thread = restart(evt)
restart_thread.start()
pinging_thread = threadout(evt)
pinging_thread.start()
To make the pinging_thread wait for the restart_thread to finish, you could use another Event.
One process is receiving data from a socket and then putting it in a queue, and the other one is processing the queued data. How to make them both run at the same time?
This socket is serve_forever while the processing of data will only run if the queue is not empty.
i have it working. i don't know if this is THE ANSWER. but it's working, well, so far. maybe there are or there will be bugs (hopefully none). any suggestions for improvements is welcome.
import multiprocessing
import socket
from multiprocessing import Process, Queue
import time
def handle(connection, address):
try:
while True:
data = connection.recv(1024)
if data == "":
break
else :
print "RECEIVE DATA : " + str(data)
xdata = data.strip()
xdata = data.split(" ")
for xd in xdata :
print "PUT Task : " + str(xd)
QueueTask.put((xd), block=True, timeout=5)
connection.sendall(data)
except:
print "Problem handling request"
finally:
connection.close()
class Server(object):
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def start(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.bind((self.hostname, self.port))
self.socket.listen(1)
while True:
conn, address = self.socket.accept()
process = multiprocessing.Process(target=handle, args=(conn, address))
process.daemon = True
process.start()
def f_Processor():
time.sleep(10)
print 'PROCESSOR Starting'
while 1:
try :
job = QueueTask.get(True,1)
print "GET Task : " + str(job)
time.sleep(5)
except Exception as err :
pass
print 'PROCESSOR Exiting'
if __name__ == "__main__":
server = Server("localhost", 9999)
QueueTask = Queue()
try:
p = multiprocessing.Process(name='Processing', target=f_Processor)
p.start()
server.start()
p.join()
except:
print "Unexpected exception"
finally:
for process in multiprocessing.active_children():
process.terminate()
process.join()
print "All done"
Also it's depend, if you have a server or client application. If it is a server than you can use
SocketServer.TCPServer.allow_reuse_address = True
self.server = TCPFactory( ( HOST, PORT ), TCPRequestHandler, params )
# Start a thread with the server
self.server_thread = threading.Thread( target = self.server.serve_forever )
self.server_thread.setDaemon( True )
self.server_thread.start()
class TCPFactory( SocketServer.ThreadingTCPServer ):
def __init__( self, server_address, RequestHandlerClass, params ):
"""
"""
SocketServer.ThreadingTCPServer.__init__( self, server_address, RequestHandlerClass )
self.patrams = params
class TCPRequestHandler( SocketServer.BaseRequestHandler ):
def setup( self ):
print self.server.params
pass
def handle( self ):
pass
So if a client connected to the server it will start a new thread. The setup and
handler function will be called automatically.
For the other thread you can use a timer, or an other thread
myt = Timer( 2, chackque, () )
myt.start()
def chackque():
if not myq.empty():
#Do what you want
or just start an other thread:
mythread = threading.Thread( target = chackque, args = ( myargs, ) )
mythread.setDaemon( True )
mythread.start()
def chackque():
while True:
if not myq.empty():
#Do what you want