I am building a service that receives messages from rabbitmq using pika. and push messages to clients using socket.io.
The socket.io server and pika server are both blocking the main thread.
This will be the same also for celery with flask or Django.
What is the proper approach to solving this and run them both under the same context?
You can use the Pub/Sub model, Start the consume process in another thread register user that want to receive from the queue and send data to subscribed users.
import json
import pika
import gevent
from flask import Flask
from flask_sockets import Sockets
connection_url = 'localhost'
channel_queue = 'test'
class PubSubListener(threading.Thread):
def __init__(self, queue_name):
threading.Thread.__init__(self)
self.clients = []
self.queue_name = queue_name
connection = pika.BlockingConnection(pika.ConnectionParameters(connection_url))
self.channel = connection.channel()
self.channel.queue_declare(queue=self.queue_name)
threading.Thread(target=self.channel.basic_consume(queue=self.queue_name,
auto_ack=True,
on_message_callback=self._callback))
def run(self):
self.channel.start_consuming()
def publish(self, body):
self.channel.basic_publish(exchange='',
routing_key=self.queue_name,
body=body)
def subscribe(self, client):
self.clients.append(client)
def _callback(self, channel, method, properties, body):
time.sleep(0.001)
message = json.loads(body)
print(message)
self.send(message)
def send(self, data):
for client in self.clients:
try:
client.send(data)
except Exception:
self.clients.remove(client)
pslistener = PubSubListener(channel_queue)
app = Flask(__name__)
sockets = Sockets(app)
#sockets.route('/echo')
def echo_socket(ws):
pslistener.subscribe(ws)
while not ws.closed:
gevent.sleep(0.1)
#app.route('/')
def hello():
return 'Hello World!'
if __name__ == "__main__":
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
pslistener.start()
print("Started")
server = pywsgi.WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
server.serve_forever()
Related
Setup
I have a python application, which should consume messages from a RabbitMQ and act as a SocketIO server to a Vue2 APP. When it receives messages from RabbitMQ it should send out a message over SocketIO to the Vue2 APP. Therefore I wrote 2 classes RabbitMQHandler and SocketIOHandler. I am starting the RabbitMQHandler in a separate thread so that both the RabbitMQ consume and the wsgi server can run in parallel.
Code
import random
import threading
import socketio
import eventlet
import sys
import os
import uuid
import pika
from dotenv import load_dotenv
import logging
class RabbitMQHandler():
def __init__(self, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
self.queue_name = 'myqueue'
self.exchange_name = 'myqueue'
credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PW)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(RABBITMQ_IP, 5672, '/', credentials))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name)
self.channel.exchange_declare(exchange=self.exchange_name, exchange_type='fanout')
self.channel.queue_bind(exchange=self.exchange_name, queue=self.queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
def run(self, callback):
logging.info('start consuming messages...')
self.channel.basic_consume(queue=self.queue_name,auto_ack=True, on_message_callback=callback)
self.channel.start_consuming()
class SocketIOHandler():
def __init__(self):
self.id = str(uuid.uuid4())
# create a Socket.IO server
self.sio = socketio.Server(async_mode='eventlet', cors_allowed_origins='*')
# wrap with a WSGI application
self.app = socketio.WSGIApp(self.sio)
self.sio.on('connect_to_backend', self.handle_connect)
self.sio.on('random_number', self.handle_random_number)
def handle_connect(self, sid, msg):
logging.info('new socket io message')
self.emit('connect_success', {
'success': True,
})
def handle_random_number(self, sid, msg):
logging.info('handle_random_number')
self.emit('response_random_number', { 'number': random.randint(0,10)})
def emit(self, event, msg):
logging.info('socket server: {}'.format(self.id))
logging.info('sending event: "{}"'.format(event))
self.sio.emit(event, msg)
logging.info('sent event: "{}"'.format(event))
def run(self):
logging.info('start web socket on port 8765...')
eventlet.wsgi.server(eventlet.listen(('', 8765)), self.app)
def start_rabbitmq_handler(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
def callback(ch, method, properties, body):
logging.info('rabbitmq handler')
socketio_handler.emit('response_random_number', { 'number': random.randint(0,10)})
with RabbitMQHandler(RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP) as rabbitmq_handler:
rabbitmq_handler.run(callback=callback)
threads = []
def main():
global threads
load_dotenv()
RABBITMQ_USER = os.getenv('RABBITMQ_USER')
RABBITMQ_PW = os.getenv('RABBITMQ_PW')
RABBITMQ_IP = os.getenv('RABBITMQ_IP')
socketio_handler = SocketIOHandler()
rabbitmq_thread = threading.Thread(target=start_rabbitmq_handler, args=(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP))
threads.append(rabbitmq_thread)
rabbitmq_thread.start()
socketio_handler.run()
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").propagate = False
main()
except KeyboardInterrupt:
try:
for t in threads:
t.exit()
sys.exit(0)
except SystemExit:
for t in threads:
t.exit()
os._exit(0)
Problem
The Problem is, that when the RabbitMQHandler receives a message the event response_random_number does not get through to the Vue2 APP. Even though it is emited in the callback function. When I send the random_number event from the Vue2 APP to the python application I do get the response_random_number event back from the python application in the Vue2 APP.
So all connections work on their own, but not together. My guess would be, that there is some sort of threading communication error. I added the id to the SocketIOHandler class to make sure it is the same instanced object and the prints are the same.
The logs 'socket server: ...', sending event: ... and sent event: ... tell me, that the function is being called correctly.
I have a Flask application as such
from flask import Flask
from flask_restful import Resource, Api
from mq_handler import MessageBroker
import pika
app = Flask(__name__)
api = Api(app)
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
mb = MessageBroker(connection)
class HelloWorld(Resource):
def get(self):
mb.run()
return {'hello': 'world'}
class LogHandler(Resource):
def get(self, table):
return {'TableName': table}
api.add_resource(HelloWorld, '/')
api.add_resource(LogHandler, '/log/<string:table>')
if __name__ == '__main__':
app.run(debug=True)
I have added a MessageBroker class to handle all my rabbitMq messages
import pika
import json
class MessageBroker:
def __init__(self, connection):
self.connection = connection
self.channel = connection.channel()
def run(self):
self.channel.start_consuming()
self.channel.basic_consume(queue='logs',
auto_ack=True,
on_message_callback=self.handle_log)
self.channel.start_consuming()
def handle_log(self, ch, method, properties, body):
decoded_content = body.decode('utf-8')
json_payload = json.loads(decoded_content)
print(" [x] Received %r" % json_payload['message'])
I have tried different solutions, but have can I get both services to run simultaneously on the same server? can somebody explain that please?
In general.. how is it possible to have several services running listening on my flask server?
I am not sure about running the consumer on an end-point will be a good idea. Because, when you start a consumer it runs an IO loop to fetch and process messages from the server continuously. The loop will not exit unless it is done externally or any exception in the message processing causing the connection to close. Can you please state your scenario for running the consumer in the end-point?
I'm trying to integrate an aiohttp web server into a Crossbar+Autobahn system architecture.
More in detail, when the aiohttp server receive a certain API call, it has to publish a message to a Crossbar router.
I've seen this example on the official repos but i've no clue on how to integrate it on my application.
Ideally, i would like to be able to do this
# class SampleTaskController(object):
async def handle_get_request(self, request: web.Request) -> web.Response:
self.publisher.publish('com.myapp.topic1', 'Hello World!')
return web.HTTPOk()
where self il an instance of SampleTaskController(object) which defines all the routes handler of the web server.
def main(argv):
cfg_path = "./task_cfg.json"
if len(argv) > 1:
cfg_path = argv[0]
logging.basicConfig(level=logging.DEBUG,
format=LOG_FORMAT)
loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(loop)
app = web.Application(loop=loop)
with open(cfg_path, 'r') as f:
task_cfg = json.load(f)
task_cfg['__cfg_path'] = cfg_path
controller = SampleTaskController(task_cfg)
controller.restore()
app['controller'] = controller
controller.setup_routes(app)
app.on_startup.append(controller.on_startup)
app.on_cleanup.append(controller.on_cleanup)
web.run_app(app,
host=task_cfg['webserver_address'],
port=task_cfg['webserver_port'])
Notice that i'm using an zmq.asyncio.ZMQEventLoop because the server is also listening on a zmq socket, which is configured inside the controller.on_startup method.
Instead of using autobahn, i've also tried to publish the message to Crossbar using wampy and it works, but the autobahn subscribers couldn't correctly parse the message.
# autobahn subscriber
class ClientSession(ApplicationSession):
async def onJoin(self, details):
self.log.info("Client session joined {details}", details=details)
self.log.info("Connected: {details}", details=details)
self._ident = details.authid
self._type = u'Python'
self.log.info("Component ID is {ident}", ident=self._ident)
self.log.info("Component type is {type}", type=self._type)
# SUBSCRIBE
def gen_on_something(thing):
def on_something(counter, id, type):
print('----------------------------')
self.log.info("'on_{something}' event, counter value: {message}",something=thing, message=counter)
self.log.info("from component {id} ({type})", id=id, type=type)
return on_something
await self.subscribe(gen_on_something('landscape'), 'landscape')
await self.subscribe(gen_on_something('nature'), 'nature')
-
# wampy publisher
async def publish():
router = Crossbar(config_path='./crossbar.json')
logging.getLogger().debug(router.realm)
logging.getLogger().debug(router.url)
logging.getLogger().debug(router.port)
client = Client(router=router)
client.start()
result = client.publish(topic="nature", message=0)
logging.getLogger().debug(result)
With this configuration the subscriber receive the message published, but it get an exception while parsing it.
TypeError: on_something() got an unexpected keyword argument 'message'
Recently I tried to use aiohttp and autobahn simultaneously. I reworked the example from crossbar documentation (originally using twisted) and got the following code:
import asyncio
import logging
from aiohttp import web
from aiohttp.web_exceptions import HTTPOk, HTTPInternalServerError
from autobahn.asyncio.component import Component
# Setup logging
logger = logging.getLogger(__name__)
class WebApplication(object):
"""
A simple Web application that publishes an event every time the
url "/" is visited.
"""
count = 0
def __init__(self, app, wamp_comp):
self._app = app
self._wamp = wamp_comp
self._session = None # "None" while we're disconnected from WAMP router
# associate ourselves with WAMP session lifecycle
self._wamp.on('join', self._initialize)
self._wamp.on('leave', self._uninitialize)
self._app.router.add_get('/', self._render_slash)
def _initialize(self, session, details):
logger.info("Connected to WAMP router (session: %s, details: %s)", session, details)
self._session = session
def _uninitialize(self, session, reason):
logger.warning("Lost WAMP connection (session: %s, reason: %s)", session, reason)
self._session = None
async def _render_slash(self, request):
if self._session is None:
return HTTPInternalServerError(reason="No WAMP session")
self.count += 1
self._session.publish(u"com.myapp.request_served", self.count, count=self.count)
return HTTPOk(text="Published to 'com.myapp.request_served'")
def main():
REALM = "crossbardemo"
BROKER_URI = "ws://wamp_broker:9091/ws"
BIND_ADDR = "0.0.0.0"
BIND_PORT = 8080
logging.basicConfig(
level='DEBUG',
format='[%(asctime)s %(levelname)s %(name)s:%(lineno)d]: %(message)s')
logger.info("Starting aiohttp backend at %s:%s...", BIND_ADDR, BIND_PORT)
loop = asyncio.get_event_loop()
component = Component(
transports=BROKER_URI,
realm=REALM,
)
component.start(loop=loop)
# When not using run() we also must start logging ourselves.
import txaio
txaio.start_logging(level='info')
app = web.Application(
loop=loop)
_ = WebApplication(app, component)
web.run_app(app, host=BIND_ADDR, port=BIND_PORT)
if __name__ == '__main__':
main()
I have a pyhton-socketio server that servers on port 8000 and handles connections perfectly. I want to emit the following emit message every 2 seconds. When emitting the client doesn't receive a message. If I emit from where I point out in the comment it works perfect.
How can I edit this to be able to emit from inside the while loop successfully?
The python server code:
import socketio
import eventlet
import threading
import time
from flask import Flask, render_template
sio = socketio.Server()
app = Flask(__name__)
#app.route('/')
def index():
return render_template('index.html')
# as a decorator:
#sio.on('connect')
def connect_handler(sid, environ):
print('IP->' + environ['REMOTE_ADDR'])
# If I emit here it works e.g. sio.emit('status-update', {'core0_in': 8, 'core1_in': 12,'cpu_usage_in': 5, 'users': 7})
#sio.on('disconnect')
def disconnect(sid):
print('disconnect ', sid)
class Server(threading.Thread):
def __init__(self, thread_id):
threading.Thread.__init__(self)
self.threadID = thread_id
def run(self):
print("Starting " + self.name)
serve()
def serve():
if __name__ == '__main__':
global app
# wrap Flask application with socketio's middleware
app = socketio.Middleware(sio, app)
# deploy as an eventlet WSGI server
eventlet.wsgi.server(eventlet.listen(('', 8000)), app)
server_thread = Server("Server-thread")
server_thread.start()
while True:
print("Emitting...")
sio.emit('status-update', {'core0_in': 8, 'core1_in': 12,'cpu_usage_in': 5, 'users': 7}) # when emitting here the client doesn't receive anything
time.sleep(2)
Pika library support tornado adapter, here is an example about how to publish message using Asynchronous adapter.
I want use pika in tornado application, just an example, I want put tornado request data to RabbitMQ, But don't know how to do it.
Two question don't know how to solve.
1 Pika use tornado adapter has its own ioloop,
self._connection = pika.SelectConnection(pika.URLParameters(self._url),
self.on_connection_open)
self._connection.ioloop.start()
Tornado application has its own ioloop,
tornado.ioloop.IOLoop.instance().start()
How to combine those two ioloop?
2 The Pika example publish same message again and again, but I want to publish request data, how to pass request data to publish method?
On my search for exactly the same thing I found this blog post of Kevin Jing Qiu.
I went the rabbitmq hole a bit further to give every websocket his own set of channel and queues.
The extract from my project can be found below. A tornado application bound to RabbitMQ consists of these parts:
The Tornado Application that will handle web requests. I only see long lived websockets here, but you can do so with short lived http requests as well.
A (one) RabbitMQ connection hold by the PikaClient Instance
a web connection that defines its channels, queues and exchanges when the open method is triggered.
Now a websocket connection can receive data from tornado (data from the browser) via on_message and send it to RabbitMQ.
The websocket connection will receive data from RabbitMQ via basic_consume.
This is not fully functional, but you should get the idea.
class PikaClient(object):
def __init__(self, io_loop):
logger.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 0
"""
Pika-Tornado connection setup
The setup process is a series of callback methods.
connect:connect to rabbitmq and build connection to tornado io loop ->
on_connected: create a channel to rabbitmq ->
on_channel_open: declare queue tornado, bind that queue to exchange
chatserver_out and start consuming messages.
"""
def connect(self):
if self.connecting:
#logger.info('PikaClient: Already connecting to RabbitMQ')
return
#logger.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected,stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
# now you are able to call the pika api to do things
# this could be exchange setup for websocket connections to
# basic_publish to later.
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('PikaClient: Channel %s open, Declaring exchange' % channel)
self.channel = channel
def on_closed(self, connection):
logger.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
class MyWebSocketHandler(websocket.WebSocketHandler):
def __init__(self):
self.status = 'not connected yet'
def open(self, *args, **kwargs):
self.status = "ws open"
self.rabbit_connect() # connect this websocket object to rabbitmq
def rabbit_connect():
self.application.pc.connection.channel(self.rabbit_channel_in_ok)
def rabbit_channel_in_ok(self,channel):
self.channel_in = channel
self.channel_in.queue_declare(self.rabbit_declare_ok,
exclusive=True,auto_delete=True)
# and so on...
handlers = [ your_definitions_here_like_websockets_or_such ]
settings = { your_settings_here }
application = tornado.web.Application(handlers,**settings)
def main():
io_loop = tornado.ioloop.IOLoop.instance()
# PikaClient is our rabbitmq consumer
pc = PikaClient(io_loop)
application.pc = pc
application.pc.connect()
application.listen(config.tornadoport)
try:
io_loop.start()
except KeyboardInterrupt:
io_loop.stop()
if __name__ == '__main__':
main()
Finally,i figure out it!
the previous solution is outdated for the newest pika component!
1.my pika Version is 1.0.1.
warning :
TornadoConnection Class has changed is package by the newest push request.
from pika.adapters import tornado_connection
2.there is an example: log() and config() should ignore(delete it)
import tornado.web
from handlers.notify import NotifyHandler
from function.function import config
from utils.utils import log
import pika
from pika.adapters import tornado_connection
HANDLERS = [(r'/notify', NotifyHandler)]
class PikaClient():
def __init__(self, io_loop):
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 9
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host="10.xxx.xxx.75", credentials=cred)
self.connection = tornado_connection.TornadoConnection(param, custom_ioloop = self.io_loop, on_open_callback = self.on_connected)
self.connection.add_on_open_error_callback(self.err)
self.connection.add_on_close_callback(self.on_closed)
def err(self, conn):
log('socket error', conn)
pass
def on_connected(self, conn):
log('connected')
self.connected = True
self.connection = conn
self.connection.channel(channel_number = 1, on_open_callback = self.on_channel_open)
def on_message(self, channel, method, properties, body):
log(body)
print('body : ', body)
pass
def on_channel_open(self, channel):
self.channel = channel
channel.basic_consume(on_message_callback = self.on_message, queue='hello', auto_ack=True)
return
def on_closed(self, conn, c):
log('pika close!')
self.io_loop.stop()
pass
def main():
port = 3002
is_debug = config('sys', 'debug')
print('DEBUG', is_debug)
app = tornado.web.Application(
HANDLERS,
debug = is_debug,
)
io_loop = tornado.ioloop.IOLoop.instance()
app.pc = PikaClient(io_loop)
app.pc.connect()
http_server = tornado.httpserver.HTTPServer(app)
app.listen(port)
io_loop.start()
print('listen {}'.format(port))
if __name__ == '__main__':
main()