Develop Reference

Twisted Send Data to all Clients from Outside the Factory sessions with task Looping Call

I'm trying to solve a simple Twisted problem.
I'm using the Python 2.7 Twisted Chat.py example for simplicity.
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
class Chat(LineReceiver):
def __init__(self, users):
self.users = users
self.name = None
self.state = "GETNAME"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.users.has_key(self.name):
del self.users[self.name]
def lineReceived(self, line):
if self.state == "GETNAME":
self.handle_GETNAME(line)
else:
self.handle_CHAT(line)
def handle_GETNAME(self, name):
if self.users.has_key(name):
self.sendLine("Name taken, please choose another.")
return
self.sendLine("Welcome, %s!" % (name,))
self.name = name
self.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
message = "<%s> %s" % (self.name, message)
for name, protocol in self.users.iteritems():
if protocol != self:
protocol.sendLine(message)
class ChatFactory(Factory):
def __init__(self):
self.users = {} # maps user names to Chat instances
def buildProtocol(self, addr):
return Chat(self.users)
reactor.listenTCP(8123, ChatFactory())
reactor.run()
What I'm trying to do is use twisted.internet task to create a Task that runs every 60 seconds and sends data to all connected sessions.
semi Pseudocode
def broadcastmsg():
for client in factory:
client.protocol.transport.write("I am a Test\n\r")
event = task.LoopingCall(broadcastmsg)
event.start(60)
The issue is I can't get Twisted to behave correctly. I can make it happen on a per session basis. But then for every use that connects it spams twice as much etc.

how are you setting up looping call?
As you know protocols are created and managed by factories. Sending some message to all users periodically looks like a task that should be put into factory. You can create loopingCall in factory __init__ method and start it immediately after init. Since you have one factory per all connections this should send notifications only once per 60 seconds
class ChatFactory(Factory):
def __init__(self):
self.users = {} # maps user names to Chat instances
def broadcast_msg():
for name in self.users:
self.users[name].sendLine("looping call send to users: {}".format(self.users.keys()))
self.looping_call = task.LoopingCall(broadcast_msg)
self.looping_call.start(60)
this should work ok, at least it does for me.

Twisted Trial: How to test a MultiService with a client in it -- Reactor was unclean

I've inherited a Twisted MultiService that I'm trying to add tests to, but whatever I do, I end up with a DirtyReactorAggregateError. The service connects to a server with a twisted.application.internet.TCPClient. I think the error is because the TCPClient is not disconnecting, but I'm not sure how I'm supposed to disconnect it. What is the proper way to test a Twisted Service with a client in it?
Here's the test case:
from labrad.node import *
from twisted.trial import unittest
import os
from socket import gethostname
class NodeTestCase(unittest.TestCase):
def setUp(self):
import labrad
name = os.environ.get('LABRADNODE', gethostname()) + '_test'
self.node = Node(name,
labrad.constants.MANAGER_HOST,
labrad.constants.MANAGER_PORT)
self.node.startService()
#self.addCleanup(self.node.stopService)
def test_nothing(self):
self.assertEqual(3, 3)
def tearDown(self):
return self.node.stopService()
Here's the Node service itself:
class Node(MultiService):
"""Parent Service that keeps the node running.
If the manager is stopped or we lose the network connection,
this service attempts to restart it so that we will come
back online when the manager is back up.
"""
reconnectDelay = 10
def __init__(self, name, host, port):
MultiService.__init__(self)
self.name = name
self.host = host
self.port = port
def startService(self):
MultiService.startService(self)
self.startConnection()
def startConnection(self):
"""Attempt to start the node and connect to LabRAD."""
print 'Connecting to %s:%d...' % (self.host, self.port)
self.node = NodeServer(self.name, self.host, self.port)
self.node.onStartup().addErrback(self._error)
self.node.onShutdown().addCallbacks(self._disconnected, self._error)
self.cxn = TCPClient(self.host, self.port, self.node)
self.addService(self.cxn)
def stopService(self):
if hasattr(self, 'cxn'):
d = defer.maybeDeferred(self.cxn.stopService)
self.removeService(self.cxn)
del self.cxn
return defer.gatherResults([MultiService.stopService(self), d])
else:
return MultiService.stopService(self)
def _disconnected(self, data):
print 'Node disconnected from manager.'
return self._reconnect()
def _error(self, failure):
r = failure.trap(UserError)
if r == UserError:
print "UserError found!"
return None
print failure.getErrorMessage()
return self._reconnect()
def _reconnect(self):
"""Clean up from the last run and reconnect."""
## hack: manually clearing the dispatcher...
dispatcher.connections.clear()
dispatcher.senders.clear()
dispatcher._boundMethods.clear()
## end hack
if hasattr(self, 'cxn'):
self.removeService(self.cxn)
del self.cxn
reactor.callLater(self.reconnectDelay, self.startConnection)
print 'Will try to reconnect in %d seconds...' % self.reconnectDelay

You should refactor your service so that it can use something like MemoryReactor from twisted.test.proto_helpers (the one public module in the twisted.test package, although hopefully it will move out of twisted.test eventually).
The way that you use MemoryReactor is to pass it into your code as the reactor to use. If you then want to see what happens when the connection succeeds, look at some of its public attributes - tcpClients for connectTCP, tcpServers for listenTCP, etc. Your tests can then pull out the Factory instances which were passed to connectTCP/listenTCP, etc, and call buildProtocol on them and makeConnection on the result. To get an ITransport implementation you can use twisted.test.proto_helpers.StringTransportWithDisconnection. You might even look at the (private API! be careful! it'll break without warning! although it really should be public) twisted.test.iosim.IOPump to relay traffic between clients and servers.
If you really actually need to do whole-system non-deterministic real-world testing, with all the complexity and random unrelated failures that implies, here's an article on actually shutting down a client and server all the way.

I had a similar issue with trying to test a instance of an application. I ended up creating a Python base class that used setUp and tearDown methods to start/stop the application.
from twisted.application.app import startApplication
from twisted.application.service import IServiceCollection
from twisted.internet.defer import inlineCallbacks
from twisted.trial import unittest
class MyTest(unittest.TestCase):
def setUp(self):
startApplication(self.app, save=False)
#inlineCallbacks
def tearDown(self):
sc = self.app.getComponent(IServiceCollection)
yield sc.stopService()

Why does my tornado tcpserver (using tornado.gen interface) hang in read_until()?

I'm trying to write a TCP server in Tornado using the tornado.gen interface. It's a simple echo server which will read from a socket and echo back every line. My implementation is hanging in read_until().
If you're aware of any Tornado tcpserver (not HTTP!) code in the wild -- either examples or working projects -- that use tornado.gen interface (not callbacks!) I would appreciate any links. I haven't been able to find any.
I'm using Tornado 3.1.1 (current pip version as of this writing). The test code that's breaking is:
echo -e 'hello\nworld\n' | nc 127.0.0.1 1234
I'm specifically asking about this code (copy-pasted from my github at https://github.com/picomancer/echoserver/blob/d590b375304ced9bc0609b85270491f58eb6d788/echoserver.py).
"""
A simple echo server for telnet, implemented using tornado.gen
"""
import itertools
import socket
import tornado.gen
import tornado.ioloop
import tornado.iostream
import tornado.tcpserver
class SimpleEcho(object):
"""
Per-connection object.
"""
def __init__(self, client_id):
self.client_id = client_id
self.message_id_alloc = itertools.count(1)
return
#tornado.gen.coroutine
def on_disconnect(self):
self.log("on_disconnect")
yield []
self.log("on_disconnect done")
return
#tornado.gen.coroutine
def dispatch(self):
try:
while True:
self.log("waiting for line on stream {}", self.stream)
line = yield self.stream.read_until("\n")
message_id = next(self.message_id_alloc)
self.log("got line {}: {}", message_id, repr(line))
yield self.stream.write(line)
except tornado.iostream.StreamClosedError:
pass
return
#tornado.gen.coroutine
def on_connect(self):
self.log("on_connect")
yield self.dispatch()
self.log("on_connect done")
return
def log(self, msg, *args, **kwargs):
print "[{}]: {}".format(self.client_id, msg.format(*args, **kwargs))
return
class SimpleEchoServer(tornado.tcpserver.TCPServer):
"""
Server listener object.
"""
def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None):
tornado.tcpserver.TCPServer.__init__(self,
io_loop=io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size)
self.client_id_alloc = itertools.count(1)
return
#tornado.gen.coroutine
def handle_stream(self, stream, address):
print "begin handle_stream"
client_id = next(self.client_id_alloc)
print "got id", client_id
print "creating conn"
conn = SimpleEcho(client_id)
stream.set_close_callback(conn.on_disconnect)
stream.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
stream.socket.setsockopt(socket.IPPROTO_TCP, socket.SO_KEEPALIVE, 1)
conn.stream = stream
print "yielding to conn.on_connect()"
yield conn.on_connect()
print "exiting handle_stream"
return
if __name__ == "__main__":
server = SimpleEchoServer()
server.listen(1234)
tornado.ioloop.IOLoop.instance().start()

My code's usage of read_until and write is not supported by Tornado. I think this is a problem in Tornado, and have filed a bug report as Tornado issue 953. As a workaround, I have written wrappers for the Tornado methods that support this usage pattern.

Twisted deferred not callable after connection failure

Having my client factory here:
import logging, traceback
from twisted.internet.protocol import ClientFactory
from twisted.internet import defer, reactor, ssl
from twisted.application import service
from protocols.smpp.protocol import SMPPClientProtocol
class SMPPClientFactory(ClientFactory):
protocol = SMPPClientProtocol
def __init__(self, config):
self.config = config
def getConfig(self):
return self.config
def clientConnectionFailed(self, connector, reason):
print "clientConnectionFailed"
self.connectDeferred.errback(reason)
def clientConnectionLost(self, connector, reason):
print "clientConnectionLost"
def connect(self):
self.connectDeferred = defer.Deferred()
factory = SMPPClientFactory(self.config, self.msgHandler)
self.log.warning('Establishing TCP connection to %s:%d' % (self.config.host, self.config.port))
reactor.connectTCP(self.config.host, self.config.port, factory)
return self.connectDeferred
And it's launching code here:
import logging, traceback
from twisted.internet import reactor, defer
from protocols.smpp.configs import SMPPClientConfig
from protocols.smpp.smpp_operations import SMPPOperationFactory
from testbed.client import SMPPClientFactory
class SMPP(object):
def __init__(self, config=None):
if config is None:
config = SMPPClientConfig()
self.config = config
self.opFactory = SMPPOperationFactory(config)
def run(self):
try:
#Bind
SMPPClientFactory(self.config, self.handleMsg).connect().addErrback(self.connectFailed)
except Exception, e:
print "ERROR: %s" % str(e)
def connectFailed(self, reason):
print "Connection failed %s" % str(reason)
def handleMsg(self, smpp, pdu):
pass
if __name__ == '__main__':
config = SMPPClientConfig(host='127.0.0.1', port=2775, username='smppclient1', password='password',
log_level=logging.DEBUG)
logging.basicConfig(level=config.log_level, filename=config.log_file, format=config.log_format,datefmt=config.log_dateformat)
SMPP(config).run()
reactor.run()
When the connection is failing (the remote server is down), i get factory's clientConnectionFailed called but it is strangely getting a "exceptions.AttributeError: SMPPClientFactory instance has no attribute 'connectDeferred'".
I need to call the errback when the connection fails, it seems there's something missing when dealing with deferreds ..

On your launch code, you instantiated an SMPPClientFactory and called connect() on it. This particular instance will have the connectDeferred attribute. However, connect also instantiated another SMPPClientFactory: factory = SMPPClientFactory(self.config, self.msgHandler) and this is the instance you used to create the actual connection. This doesn't have the connectDeferred attribute because with this instance connect has never been called.

Good way to write a lightweight client function to be imported Twisted Python

I have the following server running:
class ThasherProtocol(basic.LineReceiver):
def lineReceived(self, line):
dic = simplejson.loads( line)
ret = self.factory.d[ dic['method'] ]( dic['args'] )
self.transport.write( simplejson.dumps( ret) )
self.transport.loseConnection()
class ThasherFactory(ServerFactory):
protocol = ThasherProtocol
def __init__(self):
self.thasher = Thasher()
self.d= {
'getHash': self.thasher.getHash,
'sellHash' : self.thasher.sellHash
}
reactor.listenUNIX( c.LOCATION_THASHER, ThasherFactory() )
reactor.run()
I have multiple files importing a special function called "getHash" from a particular file.
Note that getHash's arguments are only gonna be a dictionary of texts (strings).
How do I write a client function (getHash) that can be simply:
from particular file import getHash
i = getHash( { 'type':'url', 'url':'http://www.stackoverflow.com' } )
Note that ALL I WANT TO DO is:
1) dump a dict into json,
2) dump that json into the particular socket,
3) wait for that to come back and unpack the json

You want getHash to return a Deferred, not a synchronous value.
The way to do this is to create a Deferred and associate it with the connection that performs a particular request.
The following is untested and probably won't work, but it should give you a rough idea:
import simplejson
from twisted.python.protocol import ClientFactory
from twisted.internet.defer import Deferred
from twisted.internet import reactor
from twisted.protocols.basic import LineReceiver
class BufferingJSONRequest(LineReceiver):
buf = ''
def connectionMade(self):
self.sendLine(simplejson.dumps(self.factory.params))
def dataReceived(self, data):
self.buf += data
def connectionLost(self, reason):
deferred = self.factory.deferred
try:
result = simplejson.load(self.buf)
except:
deferred.errback()
else:
deferred.callback(result)
class BufferingRequestFactory(ClientFactory):
protocol = BufferingJSONRequest
def __init__(self, params, deferred):
self.params = params
self.deferred = deferred
def clientConnectionFailed(self, connector, reason):
self.deferred.errback(reason)
def getHash(params):
result = Deferred()
reactor.connectUNIX(LOCATION_THASHER,
BufferingRequestFactory(params, result))
return result
Now, in order to use this function, you will already need to be familiar with Deferreds, and you will need to write a callback function to run when the result eventually arrives. But an explanation of those belongs on a separate question ;).

I managed to solve my own problem.
Use sockets (Unix sockets in particular) it speed up my app 4x and it's not difficult to use at all.
so now my solution is simplejson + socket