I'm trying to build a server in Twisted which would let clients connect using Server Sent Events. I would like this server also to listen to Redis and if a message comes then push it to the connected SSE clients.
I have the SSE server working. I know how to subscribe to Redis. I can't figure out how to have both pieces running without blocking each other.
I'm aware of https://github.com/leporo/tornado-redis and https://github.com/fiorix/txredisapi, which were recommended in related questions. No idea how this helps :/
How to solve this? Could you help with both: conceptual tips and code snippets?
My Twisted SSE server code:
# coding: utf-8
from twisted.web import server, resource
from twisted.internet import reactor
class Subscribe(resource.Resource):
isLeaf = True
sse_conns = set()
def render_GET(self, request):
request.setHeader('Content-Type', 'text/event-stream; charset=utf-8')
request.write("")
self.add_conn(request)
return server.NOT_DONE_YET
def add_conn(self, conn):
self.sse_conns.add(conn)
finished = conn.notifyFinish()
finished.addBoth(self.rm_conn)
def rm_conn(self, conn):
self.sse_conns.remove(conn)
def broadcast(self, event):
for conn in self.sse_conns:
event_line = "data: {}'\r\n'".format(event)
conn.write(event_line + '\r\n')
if __name__ == "__main__":
sub = Subscribe()
reactor.listenTCP(9000, server.Site(sub))
reactor.run()
My Redis subscribe code:
import redis
redis = redis.StrictRedis.from_url('redis://localhost:6379')
class RedisSub(object):
def __init__(self):
self.pubsub = redis.pubsub()
self.pubsub.subscribe('foobar-channel')
def listen(self):
for item in self.pubsub.listen():
print str(item)
This is what works for me.
I've ended up using txredis lib with a slight change to the RedisClient (added minimal subscribe capabilities).
# coding: utf-8
import os
import sys
import weakref
from txredis.client import RedisClient
from twisted.web import server, resource
from twisted.internet import reactor, protocol, defer
from twisted.python import log
from utils import cors, redis_conf_from_url
log.startLogging(sys.stdout)
PORT = int(os.environ.get('PORT', 9000))
REDIS_CONF = redis_conf_from_url(os.environ.get('REDISCLOUD_URL', 'redis://localhost:6379'))
REDIS_SUB_CHANNEL = 'votes'
class RedisBroadcaster(RedisClient):
def subscribe(self, *channels):
self._send('SUBSCRIBE', *channels)
def handleCompleteMultiBulkData(self, reply):
if reply[0] == u"message":
message = reply[1:][1]
self.sse_connector.broadcast(message)
else:
super(RedisClient, self).handleCompleteMultiBulkData(reply)
#defer.inlineCallbacks
def redis_sub():
clientCreator = protocol.ClientCreator(reactor, RedisBroadcaster, password=REDIS_CONF.get('password'))
redis = yield clientCreator.connectTCP(REDIS_CONF['host'], REDIS_CONF['port'])
redis.subscribe(REDIS_SUB_CHANNEL)
class Subscribe(resource.Resource):
isLeaf = True
sse_conns = weakref.WeakSet()
#cors
def render_GET(self, request):
request.setHeader('Content-Type', 'text/event-stream; charset=utf-8')
request.write("")
self.sse_conns.add(request)
return server.NOT_DONE_YET
def broadcast(self, event):
for conn in self.sse_conns:
event_line = "data: {}\r\n".format(event)
conn.write(event_line + '\r\n')
if __name__ == "__main__":
sub = Subscribe()
reactor.listenTCP(PORT, server.Site(sub))
RedisBroadcaster.sse_connector = sub
reactor.callLater(0, redis_sub)
reactor.run()
Related
Setup
I have a python application, which should consume messages from a RabbitMQ and act as a SocketIO server to a Vue2 APP. When it receives messages from RabbitMQ it should send out a message over SocketIO to the Vue2 APP. Therefore I wrote 2 classes RabbitMQHandler and SocketIOHandler. I am starting the RabbitMQHandler in a separate thread so that both the RabbitMQ consume and the wsgi server can run in parallel.
Code
import random
import threading
import socketio
import eventlet
import sys
import os
import uuid
import pika
from dotenv import load_dotenv
import logging
class RabbitMQHandler():
def __init__(self, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
self.queue_name = 'myqueue'
self.exchange_name = 'myqueue'
credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PW)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(RABBITMQ_IP, 5672, '/', credentials))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name)
self.channel.exchange_declare(exchange=self.exchange_name, exchange_type='fanout')
self.channel.queue_bind(exchange=self.exchange_name, queue=self.queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
def run(self, callback):
logging.info('start consuming messages...')
self.channel.basic_consume(queue=self.queue_name,auto_ack=True, on_message_callback=callback)
self.channel.start_consuming()
class SocketIOHandler():
def __init__(self):
self.id = str(uuid.uuid4())
# create a Socket.IO server
self.sio = socketio.Server(async_mode='eventlet', cors_allowed_origins='*')
# wrap with a WSGI application
self.app = socketio.WSGIApp(self.sio)
self.sio.on('connect_to_backend', self.handle_connect)
self.sio.on('random_number', self.handle_random_number)
def handle_connect(self, sid, msg):
logging.info('new socket io message')
self.emit('connect_success', {
'success': True,
})
def handle_random_number(self, sid, msg):
logging.info('handle_random_number')
self.emit('response_random_number', { 'number': random.randint(0,10)})
def emit(self, event, msg):
logging.info('socket server: {}'.format(self.id))
logging.info('sending event: "{}"'.format(event))
self.sio.emit(event, msg)
logging.info('sent event: "{}"'.format(event))
def run(self):
logging.info('start web socket on port 8765...')
eventlet.wsgi.server(eventlet.listen(('', 8765)), self.app)
def start_rabbitmq_handler(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
def callback(ch, method, properties, body):
logging.info('rabbitmq handler')
socketio_handler.emit('response_random_number', { 'number': random.randint(0,10)})
with RabbitMQHandler(RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP) as rabbitmq_handler:
rabbitmq_handler.run(callback=callback)
threads = []
def main():
global threads
load_dotenv()
RABBITMQ_USER = os.getenv('RABBITMQ_USER')
RABBITMQ_PW = os.getenv('RABBITMQ_PW')
RABBITMQ_IP = os.getenv('RABBITMQ_IP')
socketio_handler = SocketIOHandler()
rabbitmq_thread = threading.Thread(target=start_rabbitmq_handler, args=(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP))
threads.append(rabbitmq_thread)
rabbitmq_thread.start()
socketio_handler.run()
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").propagate = False
main()
except KeyboardInterrupt:
try:
for t in threads:
t.exit()
sys.exit(0)
except SystemExit:
for t in threads:
t.exit()
os._exit(0)
Problem
The Problem is, that when the RabbitMQHandler receives a message the event response_random_number does not get through to the Vue2 APP. Even though it is emited in the callback function. When I send the random_number event from the Vue2 APP to the python application I do get the response_random_number event back from the python application in the Vue2 APP.
So all connections work on their own, but not together. My guess would be, that there is some sort of threading communication error. I added the id to the SocketIOHandler class to make sure it is the same instanced object and the prints are the same.
The logs 'socket server: ...', sending event: ... and sent event: ... tell me, that the function is being called correctly.
I'm very new to python and twisted. I tried to execute simple server client program. Am doing in localhost, both are in different trminal. Here what I need is, I want to call that server function "myfunc" from client terminal and also to print the value of "x" in client terminal. For that I don't know how to do and what commands to write in server and client. Please help, both codes are mentioned below. Thanks in advance.
Server.py
#!/usr/bin/env python
from twisted.internet import reactor, protocol
import time
PORT = 5000
abc='Hi, Your Message Is Received'
class Server(protocol.Protocol):
def dataReceived(self, data):
print ""
time.sleep(3)
print 'From Client:', data
self.transport.write(abc)
def myfunc(x):
print "",x
class ServerFactory(protocol.Factory):
protocol = Server
def main():
f = ServerFactory()
reactor.listenTCP(PORT, f)
reactor.callInThread(myfunc, 'Server is Ready....!')
reactor.run()
if '__main__'==__name__:
main()
Client.py
from twisted.internet.protocol import ClientFactory, Protocol
from twisted.internet import reactor
import time
HOST = 'localhost'
PORT = 5000
msg = 'Hello Server'
class Client(Protocol):
def connectionMade(self):
print ""
print 'Connection is successful'
print ""
print ""
self.transport.write(msg)
def dataReceived(self, abc):
print 'From Server:', abc
def func(nam, idn):
print'Client name:',nam
print'Client Id:', idn
class ClientFactory(ClientFactory):
protocol = Client
def main():
f = ClientFactory()
reactor.connectTCP(HOST, PORT, f)
reactor.callInThread(func,'ras','90')
reactor.run()
if '__main__'==__name__:
main()
Is it possible to share socket objects between 2 running processes?
To simplify the issue, assume that we have two files s1.py and s2.py. Both are tcp servers listening on different ports.
s1.py body
from twisted.internet import reactor, protocol
CLIENTS = []
class Echo(protocol.Protocol):
def connectionMade(self):
CLIENTS.append(self)
def dataReceived(self, data):
self.transport.write(data)
def connectionLost(self):
CLIENTS.remove(self)
def main():
factory = protocol.ServerFactory()
factory.protocol = Echo
reactor.listenTCP(8000, factory)
reactor.run()
main()
and s2.py
from twisted.internet import reactor, protocol
class Echo(protocol.Protocol):
def dataReceived(self, data):
for socket in list_of_serialized_redis_CLIENTS_socket_object:
socket.transport.write(data)
def main():
factory = protocol.ServerFactory()
factory.protocol = Echo
reactor.listenTCP(8001, factory)
reactor.run()
main()
Is it possible to share CLIENTS from s1.py with s2.py?
Maybe there is some way to serialize CLIENTS and store it in redis or so.
Using socket.share() might not be as straight forward as you might think.
First off, it's a Windows exclusive feature and Windows is the one platform that gets the least attention by Twisted.
Getting access to the low-level socket object in Twisted can be tricky because there are a lot of things happening behind the scenes and might cause strange outcomes if changed.
So in other words, it's certainly possible to go this route, but it's not recommended.
If you're open to new solutions, I think an RPC-style server will solve your issue.
My thinking is, since the connection objects live in s1, simply create a remote process that will "do stuff" to those connections.
Other processes can connect to that server and execute functions on objects local to s1. Twisted's Perspective Broker is a ready-made solution you could leverage.
s1.py
from twisted.internet import reactor, protocol, endpoints
from twisted.spread import pb
class Echo(protocol.Protocol):
def connectionMade(self):
self.factory.client_set.add(self)
def connectionLost(self, reason):
self.factory.client_set.remove(self)
class Remote(pb.Root):
def __init__(self, client_set):
self.client_set = client_set
def remote_s1_write(self, data):
msg_tmpl = 'From s1...{0}'
for client in self.client_set:
response = msg_tmpl.format(data).encode('utf-8')
client.transport.write(response)
def main():
client_set = set() # container will be shared between the 2 servers
# setup Echo server
factory = protocol.ServerFactory()
factory.protocol = Echo
factory.client_set = client_set
echo_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=8000)
echo_server.listen(factory)
# setup PB server
pb_root = Remote(client_set)
pb_factory = pb.PBServerFactory(pb_root)
pb_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=7999)
pb_server.listen(pb_factory)
reactor.run()
main()
s2.py
from twisted.internet import reactor, protocol, endpoints
from twisted.spread import pb
class Echo(protocol.Protocol):
def dataReceived(self, data):
remote = self.factory.pb_factory.getRootObject()
remote.addCallback(lambda obj: obj.callRemote('s1_write', data))
remote.addCallback(lambda echo: 's1 said: {0}'.format(data))
remote.addErrback(lambda reason: 'error: {0}'.format(str(reason.value)))
remote.addCallback(lambda response: print(response))
self.transport.write(data)
def main():
# connect to s1's PB server
pb_factory = pb.PBClientFactory()
pb_connection = endpoints.TCP4ClientEndpoint(reactor, host='localhost', port=7999)
pb_connection.connect(pb_factory)
# setup Echo server
factory = protocol.ServerFactory()
factory.protocol = Echo
factory.pb_factory = pb_factory
echo_server = endpoints.TCP4ServerEndpoint(reactor, interface='localhost', port=8001)
echo_server.listen(factory)
reactor.run()
main()
I've taken the liberty and updated the syntax. Try this code and see if it works and ask if you have any issues.
I'm trying to write a TCP server in Tornado using the tornado.gen interface. It's a simple echo server which will read from a socket and echo back every line. My implementation is hanging in read_until().
If you're aware of any Tornado tcpserver (not HTTP!) code in the wild -- either examples or working projects -- that use tornado.gen interface (not callbacks!) I would appreciate any links. I haven't been able to find any.
I'm using Tornado 3.1.1 (current pip version as of this writing). The test code that's breaking is:
echo -e 'hello\nworld\n' | nc 127.0.0.1 1234
I'm specifically asking about this code (copy-pasted from my github at https://github.com/picomancer/echoserver/blob/d590b375304ced9bc0609b85270491f58eb6d788/echoserver.py).
"""
A simple echo server for telnet, implemented using tornado.gen
"""
import itertools
import socket
import tornado.gen
import tornado.ioloop
import tornado.iostream
import tornado.tcpserver
class SimpleEcho(object):
"""
Per-connection object.
"""
def __init__(self, client_id):
self.client_id = client_id
self.message_id_alloc = itertools.count(1)
return
#tornado.gen.coroutine
def on_disconnect(self):
self.log("on_disconnect")
yield []
self.log("on_disconnect done")
return
#tornado.gen.coroutine
def dispatch(self):
try:
while True:
self.log("waiting for line on stream {}", self.stream)
line = yield self.stream.read_until("\n")
message_id = next(self.message_id_alloc)
self.log("got line {}: {}", message_id, repr(line))
yield self.stream.write(line)
except tornado.iostream.StreamClosedError:
pass
return
#tornado.gen.coroutine
def on_connect(self):
self.log("on_connect")
yield self.dispatch()
self.log("on_connect done")
return
def log(self, msg, *args, **kwargs):
print "[{}]: {}".format(self.client_id, msg.format(*args, **kwargs))
return
class SimpleEchoServer(tornado.tcpserver.TCPServer):
"""
Server listener object.
"""
def __init__(self, io_loop=None, ssl_options=None, max_buffer_size=None):
tornado.tcpserver.TCPServer.__init__(self,
io_loop=io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size)
self.client_id_alloc = itertools.count(1)
return
#tornado.gen.coroutine
def handle_stream(self, stream, address):
print "begin handle_stream"
client_id = next(self.client_id_alloc)
print "got id", client_id
print "creating conn"
conn = SimpleEcho(client_id)
stream.set_close_callback(conn.on_disconnect)
stream.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
stream.socket.setsockopt(socket.IPPROTO_TCP, socket.SO_KEEPALIVE, 1)
conn.stream = stream
print "yielding to conn.on_connect()"
yield conn.on_connect()
print "exiting handle_stream"
return
if __name__ == "__main__":
server = SimpleEchoServer()
server.listen(1234)
tornado.ioloop.IOLoop.instance().start()
My code's usage of read_until and write is not supported by Tornado. I think this is a problem in Tornado, and have filed a bug report as Tornado issue 953. As a workaround, I have written wrappers for the Tornado methods that support this usage pattern.
I have modified Jean-Paul Calderone's code from this question to achieve that two server are run asynchronously in one thread and the second server is the client of the first one (he's a broker). This is my current code:
from twisted.internet import protocol, reactor
from twisted.protocols import basic
class BaseServerProtocol(basic.LineReceiver):
def lineReceived(self, line):
self.sendLine("I can see that you said: %s" % (line,))
self.transport.loseConnection()
##############################################################
class BrokerServerProtocol(basic.LineReceiver):
def lineReceived(self, line):
factory = protocol.ClientFactory()
factory.protocol = BrokerClientProtocol
reactor.connectTCP('localhost', 12345, factory)
self.sendLine("I did connect but have nothing to return")
# TODO: return broker client's response
class BrokerClientProtocol(basic.LineReceiver):
def connectionMade(self):
self.sendLine("Hello!")
# TODO: receive the message from broker
self.transport.loseConnection()
##############################################################
def main():
import sys
from twisted.python import log
log.startLogging(sys.stdout)
factory = protocol.ServerFactory()
factory.protocol = BaseServerProtocol
reactor.listenTCP(12345, factory)
factory = protocol.ServerFactory()
factory.protocol = BrokerServerProtocol
reactor.listenTCP(12346, factory)
reactor.run()
if __name__ == '__main__':
main()
What I would like to modify here is to replace both # TODO lines - first, I'd like to fetch BaseServer's response by the BrokerClient (second todo) and then fetch it by the BrokerServer (which has just created the BrokerClient, first todo). Can anyone help?