I am implementing a WebSockets server in Tornado 3.2. The client connecting to the server won't be a browser.
For cases in which there is back-and-forth communication between server and client, I would like to add a max. time the server will wait for a client response before closing the connection.
This is roughly what I've been trying:
import datetime
import tornado
class WSHandler(WebSocketHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.timeout = None
def _close_on_timeout(self):
if self.ws_connection:
self.close()
def open(self):
initialize()
def on_message(self, message):
# Remove previous timeout, if one exists.
if self.timeout:
tornado.ioloop.IOLoop.instance().remove_timeout(self.timeout)
self.timeout = None
if is_last_message:
self.write_message(message)
self.close()
else:
# Add a new timeout.
self.timeout = tornado.ioloop.IOLoop.instance().add_timeout(
datetime.timedelta(milliseconds=1000), self._close_on_timeout)
self.write_message(message)
Am I being a klutz and is there a much simpler way of doing this? I can't even seem to schedule a simple print statement via add_timeout above.
I also need some help testing this. This is what I have so far:
from tornado.websocket import websocket_connect
from tornado.testing import AsyncHTTPTestCase, gen_test
import time
class WSTests(AsyncHTTPTestCase):
#gen_test
def test_long_response(self):
ws = yield websocket_connect('ws://address', io_loop=self.io_loop)
# First round trip.
ws.write_message('First message.')
result = yield ws.read_message()
self.assertEqual(result, 'First response.')
# Wait longer than the timeout.
# The test is in its own IOLoop, so a blocking sleep should be okay?
time.sleep(1.1)
# Expect either write or read to fail because of a closed socket.
ws.write_message('Second message.')
result = yield ws.read_message()
self.assertNotEqual(result, 'Second response.')
The client has no problem writing to and reading from the socket. This is presumably because the add_timeout isn't firing.
Does the test need to yield somehow to allow the timeout callback on the server to run? I would have thought not since the docs say the tests run in their own IOLoop.
Edit
This is the working version, per Ben's suggestions.
import datetime
import tornado
class WSHandler(WebSocketHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.timeout = None
def _close_on_timeout(self):
if self.ws_connection:
self.close()
def open(self):
initialize()
def on_message(self, message):
# Remove previous timeout, if one exists.
if self.timeout:
tornado.ioloop.IOLoop.current().remove_timeout(self.timeout)
self.timeout = None
if is_last_message:
self.write_message(message)
self.close()
else:
# Add a new timeout.
self.timeout = tornado.ioloop.IOLoop.current().add_timeout(
datetime.timedelta(milliseconds=1000), self._close_on_timeout)
self.write_message(message)
The test:
from tornado.websocket import websocket_connect
from tornado.testing import AsyncHTTPTestCase, gen_test
import time
class WSTests(AsyncHTTPTestCase):
#gen_test
def test_long_response(self):
ws = yield websocket_connect('ws://address', io_loop=self.io_loop)
# First round trip.
ws.write_message('First message.')
result = yield ws.read_message()
self.assertEqual(result, 'First response.')
# Wait a little more than the timeout.
yield gen.Task(self.io_loop.add_timeout, datetime.timedelta(seconds=1.1))
# Expect either write or read to fail because of a closed socket.
ws.write_message('Second message.')
result = yield ws.read_message()
self.assertEqual(result, None)
The timeout-handling code in your first example looks correct to me.
For testing, each test case gets its own IOLoop, but there is only one IOLoop for both the test and anything else it runs, so you must use add_timeout instead of time.sleep() here as well to avoid blocking the server.
Ey Ben, I know this question was long ago resolved, but I wanted to share with any user reading this the solution I made for this.
It's basically based on yours, but it solves the problem from an external Service that can be easily integrated within any websocket using composition instead of inheritance:
class TimeoutWebSocketService():
_default_timeout_delta_ms = 10 * 60 * 1000 # 10 min
def __init__(self, websocket, ioloop=None, timeout=None):
# Timeout
self.ioloop = ioloop or tornado.ioloop.IOLoop.current()
self.websocket = websocket
self._timeout = None
self._timeout_delta_ms = timeout or TimeoutWebSocketService._default_timeout_delta_ms
def _close_on_timeout(self):
self._timeout = None
if self.websocket.ws_connection:
self.websocket.close()
def refresh_timeout(self, timeout=None):
timeout = timeout or self._timeout_delta_ms
if timeout > 0:
# Clean last timeout, if one exists
self.clean_timeout()
# Add a new timeout (must be None from clean).
self._timeout = self.ioloop.add_timeout(
datetime.timedelta(milliseconds=timeout), self._close_on_timeout)
def clean_timeout(self):
if self._timeout is not None:
# Remove previous timeout, if one exists.
self.ioloop.remove_timeout(self._timeout)
self._timeout = None
In order to use the service, itś as easy as create a new TimeoutWebService instance (optionally with the timeout in ms, as well as the ioloop where it should be executed) and call the method ''refresh_timeout'' to either set the timeout for the first time or reset an already existing timeout, or ''clean_timeout'' to stop the timeout service.
class BaseWebSocketHandler(WebSocketHandler):
def prepare(self):
self.timeout_service = TimeoutWebSocketService(timeout=(1000*60))
## Optionally starts the service here
self.timeout_service.refresh_timeout()
## rest of prepare method
def on_message(self):
self.timeout_service.refresh_timeout()
def on_close(self):
self.timeout_service.clean_timeout()
Thanks to this approach, you can control when exactly and under which conditions you want to restart the timeout which might differ from app to app. As an example you might only want to refresh the timeout if user acomplish X conditions, or if the message is the expected one.
I hope ppl enjoy this solution !
Related
I'm trying to interrupt multiprocessing.connection.Listener.accept(), but have thus far been unsuccessful. Since it doesn't provide a timeout parameter, I thought perhaps I could use socket.setdefaulttimeout() to interrupt it, as suggested in post I cannot find anymore, here on SO.
This didnt work. I then tried calling close() on the Listener() object. according to this post's answer, this should have worked.
It appears, however, that these objects to not play along with the usual socket-related solutions.
I can confirm that that the Listener is closed by the Timer object as expected, but the accept() call isn't interrupted.
The Code:
import logging
import socket
import os
from multiprocessing.connection import Listener
from queue import Queue, Empty
from threading import Thread, Event, Timer
class Node(Thread):
"""Base Class providing a AF_INET, AF_UNIX or AF_PIPE connection to its
data queue. It offers put() and get() method wrappers, and therefore
behaves like a Queue as well as a Thread.
Data from the internal queue is automatically fed to any connecting client.
"""
def __init__(self, sock_name, max_q_size=None, timeout=None,
*thread_args, **thread_kwargs):
"""Initialize class.
:param sock_name: UDS, TCP socket or pipe name
:param max_q_size: maximum queue size for self.q, default infinite
"""
self._sock_name = sock_name
self.connector = Listener(sock_name)
max_q_size = max_q_size if max_q_size else 0
self.q = Queue(maxsize=max_q_size)
self._running = Event()
self.connection_timer = Timer(timeout, self.connection_timed_out)
super(Node, self).__init__(*thread_args, **thread_kwargs)
def connection_timed_out(self):
"""Closes the Listener and shuts down Node if no Client connected.
:return:
"""
self.connector.close()
self.join()
def _start_connection_timer(self):
self.connection_timer.start()
def start(self):
self._running.set()
super(Node, self).start()
def join(self, timeout=None):
print("clearing..")
self._running.clear()
print("internal join")
super(Node, self).join(timeout=timeout)
print("Done")
def run(self):
while self._running.is_set():
print("Accepting connections..")
self._start_connection_timer()
try:
client = self.connector.accept()
self.connection_timer.cancel()
self.feed_data(client)
except (TimeoutError, socket.timeout):
continue
except Exception as e:
raise
print("Run() Terminated!")
def feed_data(self, client):
try:
while self._running.is_set():
try:
client.send(self.q.get())
except Empty:
continue
except EOFError:
return
if __name__ == '__main__':
import time
n = Node('/home/nils/git/spab2/test.uds', timeout=10)
n.start()
print("Sleeping")
time.sleep(15)
print("Manual join")
n.join()
I realize my question is a duplicate of this question - however, it is almost one year old and has not even received a comment. In addition, I'm using Unix Domain Sockets, as opposed to the linked post's TCP connection.
I managed to set the timeout in the following way in Python 2.7:
self.listener = mpc.Listener((address, port))
self.listener._listener._socket.settimeout(3)
With this, the accept() call is interrupted.
Result:
conn = self.listener.accept()
File "/usr/lib/python2.7/multiprocessing/connection.py", line 145, in accept
c = self._listener.accept()
File "/usr/lib/python2.7/multiprocessing/connection.py", line 275, in accept
s, self._last_accepted = self._socket.accept()
File "/usr/lib/python2.7/socket.py", line 202, in accept
sock, addr = self._sock.accept()
timeout: timed out
Regards,
Henri
I managed to code a rather silly bug that would make one of my request handlers run a very slow DB query.
Interesting bit is that I noticed that even long-after siege completed Tornado was still churning through requests (sometimes 90s later). (Comment --> I'm not 100% sure of the workings of Siege, but I'm fairly sure it closed the connection..)
My question in two parts:
- Does Tornado cancel request handlers when client closes the connection?
- Is there a way to timeout request handlers in Tornado?
I read through the code and can't seem to find anything. Even though my request handlers are running asynchronously in the above bug the number of pending requests piled up to a level where it was slowing down the app and it would have been better to close out the connections.
Tornado does not automatically close the request handler when the client drops the connection. However, you can override on_connection_close to be alerted when the client drops, which would allow you to cancel the connection on your end. A context manager (or a decorator) could be used to handle setting a timeout for handling the request; use tornado.ioloop.IOLoop.add_timeout to schedule some method that times out the request to run after timeout as part of the __enter__ of the context manager, and then cancel that callback in the __exit__ block of the context manager. Here's an example demonstrating both of those ideas:
import time
import contextlib
from tornado.ioloop import IOLoop
import tornado.web
from tornado import gen
#gen.coroutine
def async_sleep(timeout):
yield gen.Task(IOLoop.instance().add_timeout, time.time() + timeout)
#contextlib.contextmanager
def auto_timeout(self, timeout=2): # Seconds
handle = IOLoop.instance().add_timeout(time.time() + timeout, self.timed_out)
try:
yield handle
except Exception as e:
print("Caught %s" % e)
finally:
IOLoop.instance().remove_timeout(handle)
if not self._timed_out:
self.finish()
else:
raise Exception("Request timed out") # Don't continue on passed this point
class TimeoutableHandler(tornado.web.RequestHandler):
def initialize(self):
self._timed_out = False
def timed_out(self):
self._timed_out = True
self.write("Request timed out!\n")
self.finish() # Connection to client closes here.
# You might want to do other clean up here.
class MainHandler(TimeoutableHandler):
#gen.coroutine
def get(self):
with auto_timeout(self): # We'll timeout after 2 seconds spent in this block.
self.sleeper = async_sleep(5)
yield self.sleeper
print("writing") # get will abort before we reach here if we timed out.
self.write("hey\n")
def on_connection_close(self):
# This isn't the greatest way to cancel a future, since it will not actually
# stop the work being done asynchronously. You'll need to cancel that some
# other way. Should be pretty straightforward with a DB connection (close
# the cursor/connection, maybe?)
self.sleeper.set_exception(Exception("cancelled"))
application = tornado.web.Application([
(r"/test", MainHandler),
])
application.listen(8888)
IOLoop.instance().start()
Another solution to this problem is to use gen.with_timeout:
import time
from tornado import gen
from tornado.util import TimeoutError
class MainHandler
#gen.coroutine
def get(self):
try:
# I'm using gen.sleep here but you can use any future in this place
yield gen.with_timeout(time.time() + 2, gen.sleep(5))
self.write("This will never be reached!!")
except TimeoutError as te:
logger.warning(te.__repr__())
self.timed_out()
def timed_out(self):
self.write("Request timed out!\n")
I liked the way handled by the contextlib solution but I'm was always getting logging leftovers.
The native coroutine solution would be:
async def get(self):
try:
await gen.with_timeout(time.time() + 2, gen.sleep(5))
self.write("This will never be reached!!")
except TimeoutError as te:
logger.warning(te.__repr__())
self.timed_out()
Maybe I'm missing something here in the asynchronous designs of Twisted, but I can't seem to find a way to call the sendMessage() method "externaly". By this I mean, sending messages without being solely at the callback methods of Twisted/AutobahnWebsockets (like at onOpen or when receiving data from server at onMessage())
Of course I could launch a thread and call my_protocol_instance.sendMessage("hello") but that would defeat every purpose of the asynchronous design right?
In a concrete example, I need to have a top wrapper class which opens the connection and manages it, and whenever I need I call my_class.send_my_toplevel_message(msg). How can I implement this?
Hope I've been clear on my explanation.
Thanks
Why do you need a thread to launch protocolInstance.sendMessage() ?
This can be done in a normal reactor loop.
The core of a twisted is reactor and it gives a much easier look at things when you consider twisted itself reactive - meaning it does something as a reaction (response) to something else.
Now I assume that the thread you are talking about, also gets created and made in calling sendMessage because of certain events or activity or status. I can hardly imagine a case where you would just need to send a message out of the blue without any reason to react.
If however there is an event which should trigger sendMessage, there is no need to invoke that in thread: just use twisted mechanisms for catching that event and then calling sendMessage from that particular event's callback.
Now on to your concrete example: can you specify what "whenever I need" means exactly in the context of this question? An input from another connection? An input from the user? Looping activity?
I managed to implement what I needed by running Twisted in another thread, keeping my program free to run and allowing it to trigger send data in Twisted with reactor.callFromThread().
What do you think?
# ----- twisted ----------
class _WebSocketClientProtocol(WebSocketClientProtocol):
def __init__(self, factory):
self.factory = factory
def onOpen(self):
log.debug("Client connected")
self.factory.protocol_instance = self
self.factory.base_client._connected_event.set()
class _WebSocketClientFactory(WebSocketClientFactory):
def __init__(self, *args, **kwargs):
WebSocketClientFactory.__init__(self, *args, **kwargs)
self.protocol_instance = None
self.base_client = None
def buildProtocol(self, addr):
return _WebSocketClientProtocol(self)
# ------ end twisted -------
class BaseWBClient(object):
def __init__(self, websocket_settings):
self.settings = websocket_settings
# instance to be set by the own factory
self.factory = None
# this event will be triggered on onOpen()
self._connected_event = threading.Event()
# queue to hold not yet dispatched messages
self._send_queue = Queue.Queue()
self._reactor_thread = None
def connect(self):
log.debug("Connecting to %(host)s:%(port)d" % self.settings)
self.factory = _WebSocketClientFactory(
"ws://%(host)s:%(port)d" % self.settings,
debug=True)
self.factory.base_client = self
c = connectWS(self.factory)
self._reactor_thread = threading.Thread(target=reactor.run,
args=(False,))
self._reactor_thread.daemon = True
self._reactor_thread.start()
def send_message(self, body):
if not self._check_connection():
return
log.debug("Queing send")
self._send_queue.put(body)
reactor.callFromThread(self._dispatch)
def _check_connection(self):
if not self._connected_event.wait(timeout=10):
log.error("Unable to connect to server")
self.close()
return False
return True
def _dispatch(self):
log.debug("Dispatching")
while True:
try:
body = self._send_queue.get(block=False)
except Queue.Empty:
break
self.factory.protocol_instance.sendMessage(body)
def close(self):
reactor.callFromThread(reactor.stop)
I'm trying to write unittests for my application that uses Autobahn.
I want to test my controllers which gets received data from protocol, parses it and reacts to it.
But when my test comes to a point when protocol should be disconnected (self.sendClose) then it raises error
exceptions.AttributeError: 'MyProtocol' object has no attribute 'state'.
I was trying to makeConnection using proto_helpers.StringTransport but then I have errors too
exceptions.AttributeError: StringTransport instance has no attribute 'setTcpNoDelay'`
I'm using trial and I don't want to run dummy server/client for testing purposes only, because it's not recommended.
How should I write my tests so I can test functions that sends data, read data, disconnects etc. using fake connection and trial ?
It is difficult to say exactly what is going on without having a peek at MyProtocol class. The problem sounds a lot like it is caused by the fact that you are directly messing round with low level functions and therefore also the state attribute of WebSocket class, which is, well, a representation of the internal state of the WebSocket connection.
According to the autobahn reference doc, the APIs from the WebSicketProtocol that you could directly use and override are:
onOpen
onMessage
onClose
sendMessage
sendClose
Your approach of using the StringTransport to test your protocol is not ideal. The problem lays in the fact that MyProtocol is a tiny layer on top of the WebSocketProtocol framework provided by autobahn which, for better or worse, hides the details about managing the connection, the transport and the internal protocol state.
If you think about it, you want to test your stuff, not WebSocketProtocol and therefore if you do not want to embed a dummy server or client, your best bet is to test directly the methods that MyProtocol overrides.
An example of what I am saying is the following
class MyPublisher(object):
cbk=None
def publish(self, msg):
if self.cbk:
self.cbk(msg)
class MyProtocol(WebSocketServerProtocol):
def __init__(self, publisher):
WebSocketServerProtocol.__init__(self)
#Defining callback for publisher
publisher.cbk = self.sendMessage
def onMessage(self, msg, binary)
#Stupid echo
self.sendMessage(msg)
class NotificationTest(unittest.TestCase):
class MyProtocolFactory(WebSocketServerFactory):
def __init__(self, publisher):
WebSocketServerFactory.__init__(self, "ws://127.0.0.1:8081")
self.publisher = publisher
self.openHandshakeTimeout = None
def buildProtocol(self, addr):
protocol = MyProtocol(self.listener)
protocol.factory = self
protocol.websocket_version = 13 #Hybi version 13 is supported by pretty much everyone (apart from IE <8 and android browsers)
return protocol
def setUp(self):
publisher = task.LoopingCall(self.send_stuff, "Hi there")
factory = NotificationTest.MyProtocolFactory(listener)
protocol = factory.buildProtocol(None)
transport = proto_helpers.StringTransport()
def play_dumb(*args): pass
setattr(transport, "setTcpNoDelay", play_dumb)
protocol.makeConnection(transport)
self.protocol, self.transport, self.publisher, self.fingerprint_handler = protocol, transport, publisher, fingerprint_handler
def test_onMessage(self):
#Following 2 lines are the problematic part. Here you are manipulating explicitly a hidden state which your implementation should not be concerned with!
self.protocol.state = WebSocketProtocol.STATE_OPEN
self.protocol.websocket_version = 13
self.protocol.onMessage("Whatever")
self.assertEqual(self.transport.value()[2:], 'Whatever')
def test_push(self):
#Following 2 lines are the problematic part. Here you are manipulating explicitly a hidden state which your implementation should not be concerned with!
self.protocol.state = WebSocketProtocol.STATE_OPEN
self.protocol.websocket_version = 13
self.publisher.publish("Hi there")
self.assertEqual(self.transport.value()[2:], 'Hi There')
As you might have noticed, using the StringTransport here is very cumbersome. You must have knowledge of the underline framework and bypass its state management, something you don't really want to do. Unfortunately autobahn does not provide a ready-to-use test object that would permit easy state manipulation and therefore my suggestion of using dummy servers and clients is still valid
Testing your server WITH network
The test provided shows how you can test server push, asserting that what your are getting is what you expect, and using also a hook on how to determine when to finish.
The server protocol
from twisted.trial.unittest import TestCase as TrialTest
from autobahn.websocket import WebSocketServerProtocol, WebSocketServerFactory, WebSocketClientProtocol, WebSocketClientFactory, connectWS, listenWS
from twisted.internet.defer import Deferred
from twisted.internet import task
START="START"
class TestServerProtocol(WebSocketServerProtocol):
def __init__(self):
#The publisher task simulates an event that triggers a message push
self.publisher = task.LoopingCall(self.send_stuff, "Hi there")
def send_stuff(self, msg):
#this method sends a message to the client
self.sendMessage(msg)
def _on_start(self):
#here we trigger the task to execute every second
self.publisher.start(1.0)
def onMessage(self, message, binary):
#According to this stupid protocol, the server starts sending stuff when the client sends a "START" message
#You can plug other commands in here
{
START : self._on_start
#Put other keys here
}[message]()
def onClose(self, wasClean, code, reason):
#After closing the connection, we tell the task to stop sending messages
self.publisher.stop()
The client protocol and factory
Next class is the client protocol. It basically tells the server to start pushing messages. It calls the close_condition on them to see if it is time to close the connection and as a last thing, it calls the assertion function on the messages it received to see if the test was successful or not
class TestClientProtocol(WebSocketClientProtocol):
def __init__(self, assertion, close_condition, timeout, *args, **kwargs):
self.assertion = assertion
self.close_condition = close_condition
self._received_msgs = []
from twisted.internet import reactor
#This is a way to set a timeout for your test
#in case you never meet the conditions dictated by close_condition
self.damocle_sword = reactor.callLater(timeout, self.sendClose)
def onOpen(self):
#After the connection has been established,
#you can tell the server to send its stuff
self.sendMessage(START)
def onMessage(self, msg, binary):
#Here you get the messages pushed from the server
self._received_msgs.append(msg)
#If it is time to close the connection
if self.close_condition(msg):
self.damocle_sword.cancel()
self.sendClose()
def onClose(self, wasClean, code, reason):
#Now it is the right time to check our test assertions
self.assertion.callback(self._received_msgs)
class TestClientProtocolFactory(WebSocketClientFactory):
def __init__(self, assertion, close_condition, timeout, **kwargs):
WebSocketClientFactory.__init__(self, **kwargs)
self.assertion = assertion
self.close_condition = close_condition
self.timeout = timeout
#This parameter needs to be forced to None to not leave the reactor dirty
self.openHandshakeTimeout = None
def buildProtocol(self, addr):
protocol = TestClientProtocol(self.assertion, self.close_condition, self.timeout)
protocol.factory = self
return protocol
The trial based test
class WebSocketTest(TrialTest):
def setUp(self):
port = 8088
factory = WebSocketServerFactory("ws://localhost:{}".format(port))
factory.protocol = TestServerProtocol
self.listening_port = listenWS(factory)
self.factory, self.port = factory, port
def tearDown(self):
#cleaning up stuff otherwise the reactor complains
self.listening_port.stopListening()
def test_message_reception(self):
#This is the test assertion, we are testing that the messages received were 3
def assertion(msgs):
self.assertEquals(len(msgs), 3)
#This class says when the connection with the server should be finalized.
#In this case the condition to close the connectionis for the client to get 3 messages
class CommunicationHandler(object):
msg_count = 0
def close_condition(self, msg):
self.msg_count += 1
return self.msg_count == 3
d = Deferred()
d.addCallback(assertion)
#Here we create the client...
client_factory = TestClientProtocolFactory(d, CommunicationHandler().close_condition, 5, url="ws://localhost:{}".format(self.port))
#...and we connect it to the server
connectWS(client_factory)
#returning the assertion as a deferred purely for demonstration
return d
This is obviously just an example, but as you can see I did not have to mess around with makeConnection or any transport explicitly
I'm running a WebSocketHandler with Tornado, and I have a while loop inside the Handler. This loop blocks everything - which is very bad. How can I make the tailstream() function asynchronous (a.k.a. non-blocking)? (As it is now, tailstream blocks everything, and makes even new websocket connections impossible. I need it to run for each websocket connection.)
(...)
class WSHandler(tornado.websocket.WebSocketHandler):
connections = []
filters = {}
def allow_draft76(self):
# for iOS 5.0 Safari
return True
def open(self):
self.write_message('open')
self.count = db.my_collection.count() - 1
self.cursor = coll.find(tailable=True, await_data=True, skip=self.count)
self.tailstream()
def on_message(self, message):
print message
def on_close(self):
self.connections.remove(self)
self.cb.stop()
print 'connection closed'
#tornado.web.asynchronous
def tailstream(self):
while self.cursor.alive:
try:
doc = self.cursor.next()
self.print2web(doc)
except StopIteration:
time.sleep(1)
(...)
I think while not blocking it. But time.sleep do!
Replace it with yield gen.Task(IOLoop.instance().add_timeout, time.time() + 5) from this answer.
If it will not help - we can think about whole structure of solution.