I have a Tornado server with Tornadio2 handling the WebSocket and TornadoRedis by which I publish realtime messages to the clients (code was merged from several examples).
redis_client = tornadoredis.Client(config.get('cache', 'host'), int(config.get('cache', 'port')))
redis_client.connect()
class RealtimeHandler(tornadio2.conn.SocketConnection):
def __init__(self, *args, **kwargs):
super(RealtimeHandler, self).__init__(*args, **kwargs)
self.listen()
#tornado.gen.engine
def listen(self):
self.client = tornadoredis.Client(config.get('cache', 'host'), int(config.get('cache', 'port')))
self.client.connect()
yield tornado.gen.Task(self.client.subscribe, 'invalidation')
self.client.listen(self.on_message)
def on_event(self, name, *args, **kwargs):
if name == 'invalidation':
redis_client.publish('invalidation', kwargs['args'])
def on_message(self, msg):
if msg.kind == 'message':
self.send(msg.body)
def on_close(self):
if self.client.subscribed:
self.client.unsubscribe('invalidation')
self.client.disconnect()
This code works, and now, I want to store the connections in Redis to accomplish 2 things:
1. Support redundancy for my server
2. Be able to use the connections even if I restart Tornado
I saw several examples where a global list is used to save the connections but this of course won't solve my issues above.
So, how do I accomplish this?
On a different note, is this a good setup? How is it compared to node.js with socket.io and redis?
Related
I have a python Twisted server application interfacing with a legacy client application and each client has been assigned a specific port on which to connect to the server. So I have set up listeners on all those ports on the server and it is working beautifully, but I need to build in some safeguards to disallow more than one client connecting on the same server port. There are too many things on the client application that break when another client is connected to the same port, and I can't update that application right now. I gotta live with how it has been operating.
I know that I could build in some logic to the connectionMade() function to see if someone already exists on that port and if so, close this new connection. But I'd rather there be a way to reject it to begin with, so the client is not even allowed to connect. Then the client will know they made a mistake and they can alter which port they are trying to connect on.
Here is a stripped down version of my server code if that helps.
from twisted.internet.protocol import Factory
from twisted.internet.protocol import Protocol
from twisted.internet import reactor
from twisted.internet import task
import time
class MyServerTasks():
def someFunction(msg):
#Do stuff
def someOtherFunction(msg):
#Do other stuff
class MyServer(Protocol):
def __init__(self, users):
self.users = users
self.name = None
def connectionMade(self):
#Depending on which port is connected, go do stuff
def connectionLost(self, reason):
#Update dictionaries and other global info
def dataReceived(self, line):
t = time.strftime('%Y-%m-%d %H:%M:%S')
d = self.transport.getHost()
print("{} Received message from {}:{}...{}".format(t, d.host, d.port, line)) #debug
self.handle_GOTDATA(line)
def handle_GOTDATA(self, msg):
#Parse the received data string and do stuff based on the message.
#For example:
if "99" in msg:
MyServerTasks.someFunction(msg)
class MyServerFactory(Factory):
def __init__(self):
self.users = {} # maps user names to Chat instances
def buildProtocol(self, *args, **kwargs):
protocol = MyServer(self.users)
protocol.factory = self
protocol.factory.clients = []
return protocol
reactor.listenTCP(50010, MyServerFactory())
reactor.listenTCP(50011, MyServerFactory())
reactor.listenTCP(50012, MyServerFactory())
reactor.listenTCP(50013, MyServerFactory())
reactor.run()
When a client connected to the server, twisted use the factory to create
a protocol (by calling its buildProtocol method) instance to handle the client request.
therefore you can maintain a counter of connected client in your MyServerFactory,
if the counter has reached maximum allowed connected client you can return None
instead of creating new protocol for that client.
Twisted will close the client connection if the factory doesn't return a protocol from
its buildProtocol method.
you can see here
class MyServer(Protocol):
def __init__(self, users):
self.users = users
self.name = None
def connectionMade(self):
#Depending on which port is connected, go do stuff
def connectionLost(self, reason):
#Update dictionaries and other global info
self.factory.counter -= 1
def dataReceived(self, line):
t = time.strftime('%Y-%m-%d %H:%M:%S')
d = self.transport.getHost()
print("{} Received message from {}:{}...{}".format(t, d.host, d.port, line)) #debug
self.handle_GOTDATA(line)
def handle_GOTDATA(self, msg):
#Parse the received data string and do stuff based on the message.
#For example:
if "99" in msg:
MyServerTasks.someFunction(msg)
class MyServerFactory(Factory):
MAX_CLIENT = 2
def __init__(self):
self.users = {} # maps user names to Chat instances
self.counter = 0
def buildProtocol(self, *args, **kwargs):
if self.counter == self.MAX_CLIENT:
return None
self.counter += 1
protocol = MyServer(self.users)
protocol.factory = self
protocol.factory.clients = []
return protocol
I've a web-socket server which connects with the clients. Following is the code:-
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
class Chat(LineReceiver):
def __init__(self, users):
self.users = users
self.name = None
self.state = "GETNAME"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.users.has_key(self.name):
del self.users[self.name]
def lineReceived(self, line):
if self.state == "GETNAME":
self.handle_GETNAME(line)
else:
self.handle_CHAT(line)
def handle_GETNAME(self, name):
if self.users.has_key(name):
self.sendLine("Name taken, please choose another.")
return
self.sendLine("Welcome, %s!" % (name,))
self.name = name
self.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
# Need to send the message to the connected clients.
class ChatFactory(Factory):
def __init__(self):
self.users = {} # maps user names to Chat instances
def buildProtocol(self, addr):
return Chat(self.users)
reactor.listenTCP(8123, ChatFactory())
reactor.run()
Clients get connected to the above code(server), and sends the data to the server.
Now, I've another python script, basically a scraper which scrapes the web, processes it and finally need to send the data to the connected clients.
script.py
while True:
# call `send_message` function and send data to the connected clients.
How can I achieve it?? Any example would be of great help!!
UPDATE
After using Autobahn
I've a server that fetches data from 3rd party API. I want to send this data to all the connected web-socket clients. Here is my code:-
class MyServerProtocol(WebSocketServerProtocol):
def __init__(self):
self.connected_users = []
self.send_data()
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
self.connected_users.append(self) # adding users to the connected_list
def send_data(self):
# fetch data from the API and forward it to the connected_users.
for u in self.users:
print 1111
u.sendMessage('Hello, Some Data from API!', False)
def onClose(self, wasClean, code, reason):
connected_users.remove(self) # remove user from the connected list of users
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import sys
from twisted.python import log
from twisted.internet import reactor
factory = WebSocketServerFactory(u"ws://127.0.0.1:9000")
factory.protocol = MyServerProtocol
reactor.listenTCP(9000, factory)
reactor.run()
My Server will never receive a message or probably will receive, but as of right now there's no such use-case, hence no need for OnMessage event for this example).
How do I write my send_data function in order to send data to all my connected clients??
You need to avoid this pattern when writing software with Twisted:
while True:
# call `send_message` function and send data to the connected clients.
Twisted is a cooperative multitasking system. "Cooperative" means that you have to give up control of execution periodically so that other tasks get a chance to run.
twisted.internet.task.LoopingCall can be used to replace many while ... loops (particularly while True loops):
from twisted.internet.task import LoopingCall
LoopingCall(one_iteration).start(iteration_interval)
This will call one_iteration every iteration_interval seconds. In between, it will give up control of execution so other tasks can run.
Making one_iteration send a message to a client is just a matter of giving one_iteration a reference to that client (or those clients, if there are many).
This is a variation on the FAQ How do I make Input on One Connection Result in Output on Another.
If you have a ChatFactory with a dict containing all your clients, just pass that factory to one_iteration:
LoopingCall(one_iteration, that_factory)
or
LoopingCall(lambda: one_iteration(that_factory))
I'm not very experienced with Python and new to Tornado websockets, so I have a problem grasping the idea how it actually works.
Concretely, this is a usual example:
class EchoWebSocket(tornado.websocket.WebSocketHandler):
def open(self):
print("WebSocket opened")
def on_message(self, message):
self.write_message(u"You said: " + message)
def on_close(self):
print("WebSocket closed")
What confuses me is the way the requests are served. I read that Tornado is non-blocking and if understood well single-threaded, where each request is handled by a handler in a fast way and if the operation lasts longer then there are callbacks which are called when certain operation (e.g. database) is finished. This allows a large number of users to be served. But what confuses me with these websockets and examples, is what is self in this case?
I know that it is a similar thing like this in Java and C#, but I don't understand how this self represents different clients each time a client sends a message? Moreover, is there only a single instance of EchoWebSocket per application, or there is an instance per request?
Additionally, how should I keep the references to the clients? I tried two ways:
class EchoWebSocket(tornado.websocket.WebSocketHandler):
clients = []
def open(self):
print("WebSocket opened")
def on_message(self, message):
EchoWebSocket.clients.append(self)
self.write_message(u"You said: " + message)
def on_close(self):
print("WebSocket closed")
And
class EchoWebSocket(tornado.websocket.WebSocketHandler):
def __init__(self, application, request):
super(EchoWebSocket,self).__init__(application, request)
self.clients = []
def open(self):
print("WebSocket opened")
def on_message(self, message):
self.clients.append(self)
self.write_message(u"You said: " + message)
def on_close(self):
print("WebSocket closed")
But the second approach doesn't seem to be working. What is the proper way to do it and why?
In python, by convention self is a reference to the current class' instance object.
AFAIK, the tornado framework creates an instance of the class WebSocketHandler for each client. So you don't really have to keep a reference to clients. Each client is naturally handled by a different instance of the handler. Practically it means that self will be different objects for two distinct clients.
To keep references to connected clients, something like that would work:
class EchoWebSocket(tornado.websocket.WebSocketHandler):
all_clients = set()
def open(self):
self.all_clients.add(self)
def on_message(self, message):
pass
def on_close(self):
self.all_clients.remove(self)
I need to send system logs to the browser and so I have a tornado-based websocket server running like so.
class WSHandler(tornado.websocket.WebSocketHandler):
def check_origin(self, origin):
return True
def get(self, *args, **kwargs):
self.ip = self.get_argument('ip', None)
self.action = self.get_argument('action', None)
super(WSHandler, self).get(self, *args, **kwargs)
def open(self, *args, **kwargs):
clients.append(self)
def on_message(self, message):
ll = eval(message)
for cl in clients:
if cl.ip and cl.ip != ll.user:
continue
if cl.action and cl.action != ll.action:
continue
message = '%s %s' % (ll.action, ll.url)
cl.write_message(message)
def on_close(self):
try:
clients.remove(self)
except ValueError:
pass
The examples I've encountered so far revolve around Tornado-based servers and js-based clients.
What I need, however, is an easy way to connect to this websocket from a Python client, preferably powered by Tornado. The client does not need to receive messages - only send them. I thought I had my answer with this SO post,
How to run functions outside websocket loop in python (tornado)
...but I need to send a message whenever a log event occurs, and preferably from my code that's parsing the events. The examples I've encountered so far revolve around Tornado-based servers and js-based clients. Is there a short & sweet tornado-based client that only sends messages, that can be called from a for-loop?
Also, I developed a complete Tornado WebSocket Client/Server example.
https://github.com/ilkerkesen/tornado-websocket-client-example
If you want WebSocket Authentication/Authorization, look at my other projects trebol and sugar.
There is tornad-websocket-client project. Pay attention on it.
Also there is simple websocket-client to just send messages.
Tornado includes a websocket client: http://www.tornadoweb.org/en/stable/websocket.html#client-side-support
Maybe I'm missing something here in the asynchronous designs of Twisted, but I can't seem to find a way to call the sendMessage() method "externaly". By this I mean, sending messages without being solely at the callback methods of Twisted/AutobahnWebsockets (like at onOpen or when receiving data from server at onMessage())
Of course I could launch a thread and call my_protocol_instance.sendMessage("hello") but that would defeat every purpose of the asynchronous design right?
In a concrete example, I need to have a top wrapper class which opens the connection and manages it, and whenever I need I call my_class.send_my_toplevel_message(msg). How can I implement this?
Hope I've been clear on my explanation.
Thanks
Why do you need a thread to launch protocolInstance.sendMessage() ?
This can be done in a normal reactor loop.
The core of a twisted is reactor and it gives a much easier look at things when you consider twisted itself reactive - meaning it does something as a reaction (response) to something else.
Now I assume that the thread you are talking about, also gets created and made in calling sendMessage because of certain events or activity or status. I can hardly imagine a case where you would just need to send a message out of the blue without any reason to react.
If however there is an event which should trigger sendMessage, there is no need to invoke that in thread: just use twisted mechanisms for catching that event and then calling sendMessage from that particular event's callback.
Now on to your concrete example: can you specify what "whenever I need" means exactly in the context of this question? An input from another connection? An input from the user? Looping activity?
I managed to implement what I needed by running Twisted in another thread, keeping my program free to run and allowing it to trigger send data in Twisted with reactor.callFromThread().
What do you think?
# ----- twisted ----------
class _WebSocketClientProtocol(WebSocketClientProtocol):
def __init__(self, factory):
self.factory = factory
def onOpen(self):
log.debug("Client connected")
self.factory.protocol_instance = self
self.factory.base_client._connected_event.set()
class _WebSocketClientFactory(WebSocketClientFactory):
def __init__(self, *args, **kwargs):
WebSocketClientFactory.__init__(self, *args, **kwargs)
self.protocol_instance = None
self.base_client = None
def buildProtocol(self, addr):
return _WebSocketClientProtocol(self)
# ------ end twisted -------
class BaseWBClient(object):
def __init__(self, websocket_settings):
self.settings = websocket_settings
# instance to be set by the own factory
self.factory = None
# this event will be triggered on onOpen()
self._connected_event = threading.Event()
# queue to hold not yet dispatched messages
self._send_queue = Queue.Queue()
self._reactor_thread = None
def connect(self):
log.debug("Connecting to %(host)s:%(port)d" % self.settings)
self.factory = _WebSocketClientFactory(
"ws://%(host)s:%(port)d" % self.settings,
debug=True)
self.factory.base_client = self
c = connectWS(self.factory)
self._reactor_thread = threading.Thread(target=reactor.run,
args=(False,))
self._reactor_thread.daemon = True
self._reactor_thread.start()
def send_message(self, body):
if not self._check_connection():
return
log.debug("Queing send")
self._send_queue.put(body)
reactor.callFromThread(self._dispatch)
def _check_connection(self):
if not self._connected_event.wait(timeout=10):
log.error("Unable to connect to server")
self.close()
return False
return True
def _dispatch(self):
log.debug("Dispatching")
while True:
try:
body = self._send_queue.get(block=False)
except Queue.Empty:
break
self.factory.protocol_instance.sendMessage(body)
def close(self):
reactor.callFromThread(reactor.stop)