Develop Reference

Python consume RabbitMQ and run SocketIO server

Setup
I have a python application, which should consume messages from a RabbitMQ and act as a SocketIO server to a Vue2 APP. When it receives messages from RabbitMQ it should send out a message over SocketIO to the Vue2 APP. Therefore I wrote 2 classes RabbitMQHandler and SocketIOHandler. I am starting the RabbitMQHandler in a separate thread so that both the RabbitMQ consume and the wsgi server can run in parallel.
Code
import random
import threading
import socketio
import eventlet
import sys
import os
import uuid
import pika
from dotenv import load_dotenv
import logging
class RabbitMQHandler():
def __init__(self, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
self.queue_name = 'myqueue'
self.exchange_name = 'myqueue'
credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PW)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(RABBITMQ_IP, 5672, '/', credentials))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name)
self.channel.exchange_declare(exchange=self.exchange_name, exchange_type='fanout')
self.channel.queue_bind(exchange=self.exchange_name, queue=self.queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
def run(self, callback):
logging.info('start consuming messages...')
self.channel.basic_consume(queue=self.queue_name,auto_ack=True, on_message_callback=callback)
self.channel.start_consuming()
class SocketIOHandler():
def __init__(self):
self.id = str(uuid.uuid4())
# create a Socket.IO server
self.sio = socketio.Server(async_mode='eventlet', cors_allowed_origins='*')
# wrap with a WSGI application
self.app = socketio.WSGIApp(self.sio)
self.sio.on('connect_to_backend', self.handle_connect)
self.sio.on('random_number', self.handle_random_number)
def handle_connect(self, sid, msg):
logging.info('new socket io message')
self.emit('connect_success', {
'success': True,
})
def handle_random_number(self, sid, msg):
logging.info('handle_random_number')
self.emit('response_random_number', { 'number': random.randint(0,10)})
def emit(self, event, msg):
logging.info('socket server: {}'.format(self.id))
logging.info('sending event: "{}"'.format(event))
self.sio.emit(event, msg)
logging.info('sent event: "{}"'.format(event))
def run(self):
logging.info('start web socket on port 8765...')
eventlet.wsgi.server(eventlet.listen(('', 8765)), self.app)
def start_rabbitmq_handler(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
def callback(ch, method, properties, body):
logging.info('rabbitmq handler')
socketio_handler.emit('response_random_number', { 'number': random.randint(0,10)})
with RabbitMQHandler(RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP) as rabbitmq_handler:
rabbitmq_handler.run(callback=callback)
threads = []
def main():
global threads
load_dotenv()
RABBITMQ_USER = os.getenv('RABBITMQ_USER')
RABBITMQ_PW = os.getenv('RABBITMQ_PW')
RABBITMQ_IP = os.getenv('RABBITMQ_IP')
socketio_handler = SocketIOHandler()
rabbitmq_thread = threading.Thread(target=start_rabbitmq_handler, args=(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP))
threads.append(rabbitmq_thread)
rabbitmq_thread.start()
socketio_handler.run()
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").propagate = False
main()
except KeyboardInterrupt:
try:
for t in threads:
t.exit()
sys.exit(0)
except SystemExit:
for t in threads:
t.exit()
os._exit(0)
Problem
The Problem is, that when the RabbitMQHandler receives a message the event response_random_number does not get through to the Vue2 APP. Even though it is emited in the callback function. When I send the random_number event from the Vue2 APP to the python application I do get the response_random_number event back from the python application in the Vue2 APP.
So all connections work on their own, but not together. My guess would be, that there is some sort of threading communication error. I added the id to the SocketIOHandler class to make sure it is the same instanced object and the prints are the same.
The logs 'socket server: ...', sending event: ... and sent event: ... tell me, that the function is being called correctly.

Consume RabbitMq using Pika and push using Socket.io

I am building a service that receives messages from rabbitmq using pika. and push messages to clients using socket.io.
The socket.io server and pika server are both blocking the main thread.
This will be the same also for celery with flask or Django.
What is the proper approach to solving this and run them both under the same context?

You can use the Pub/Sub model, Start the consume process in another thread register user that want to receive from the queue and send data to subscribed users.
import json
import pika
import gevent
from flask import Flask
from flask_sockets import Sockets
connection_url = 'localhost'
channel_queue = 'test'
class PubSubListener(threading.Thread):
def __init__(self, queue_name):
threading.Thread.__init__(self)
self.clients = []
self.queue_name = queue_name
connection = pika.BlockingConnection(pika.ConnectionParameters(connection_url))
self.channel = connection.channel()
self.channel.queue_declare(queue=self.queue_name)
threading.Thread(target=self.channel.basic_consume(queue=self.queue_name,
auto_ack=True,
on_message_callback=self._callback))
def run(self):
self.channel.start_consuming()
def publish(self, body):
self.channel.basic_publish(exchange='',
routing_key=self.queue_name,
body=body)
def subscribe(self, client):
self.clients.append(client)
def _callback(self, channel, method, properties, body):
time.sleep(0.001)
message = json.loads(body)
print(message)
self.send(message)
def send(self, data):
for client in self.clients:
try:
client.send(data)
except Exception:
self.clients.remove(client)
pslistener = PubSubListener(channel_queue)
app = Flask(__name__)
sockets = Sockets(app)
#sockets.route('/echo')
def echo_socket(ws):
pslistener.subscribe(ws)
while not ws.closed:
gevent.sleep(0.1)
#app.route('/')
def hello():
return 'Hello World!'
if __name__ == "__main__":
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
pslistener.start()
print("Started")
server = pywsgi.WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
server.serve_forever()

Python-socketio not emitting events sometimes on FastAPI Heroku server

I have a FastAPI server running on Heroku, using python-socketio(5.3.0) to handle socket connections. I have an endpoint that is called when an event needs to be emitted to a certain socket. The problem I am facing is that the event doesn't emit every single time. It only emits some of the times. It's working perfectly on local.
FastAPI server with python-socketio:
import uvicorn
from fastapi import FastAPI
import socketio
from socketio.asyncio_namespace import AsyncNamespace
import json
notification_users = {}
notification_sockets = {}
class NotificationConnector(AsyncNamespace):
async def on_connect(self, sid, *args, **kwargs):
print("User connected: ", sid)
async def on_disconnect(self, sid):
user_id = chat_sockets.get(sid)
print("User unsubscribed to chats: ", user_id)
connection_list = chat_users.get(user_id)
for socket in connection_list:
if(socket==sid):
connection_list.remove(sid)
if connection_list == []:
chat_users.pop(user_id)
else:
chat_users[user_id] = connection_list
chat_sockets.pop(sid)
async def on_subscribe(self, sid, data):
user_id = str(data.get("userID"))
print("User subscribed to notifications: ", user_id)
if user_id in notification_users.keys():
notification_users[user_id].append(sid)
else:
notification_users[user_id] = [sid]
notification_sockets[sid] = user_id
fast_app = FastAPI()
sio = socketio.AsyncServer(
async_mode='asgi',
cors_allowed_origins='*',
logger=True,
engineio_logger=True
)
sio.register_namespace(NotificationConnector("/notifications"))
app = socketio.ASGIApp(
socketio_server=sio,
other_asgi_app=fast_app,
socketio_path='/socket.io/'
)
#fast_app.post("/send")
async def send_notification(item: NotificationData):
user_id = item.userId
notification_content = json.dumps(item.notification)
socket_list = notification_users.get(user_id,[])
print(f"Socket list for {user_id} is {socket_list}")
for socket in socket_list:
print(f"Sending '{notification_content}' to socket Id: {socket}")
await sio.emit('notifications', notification_content, room=socket,
namespace="/notifications")
return {"success":True},200

The issue had nothing to do with python-socketio. It seems to be caused by how uvicorn deals with global variables. By default uvicorn creates multiple workers and each worker has its own copy of the global variables.
So, when a worker dealt with a /send POST request, it's copy of notification_users might not have had the information of the socket id to which the event needed to be emitted to.
Changing the number of workers to 1 fixed the issue. Will probably take a look at Redis instead of using global variables.
A better explanation of how uvicorn workers deal with global variables.

Listening for RabbitMQ and running flask on same server

I have a Flask application as such
from flask import Flask
from flask_restful import Resource, Api
from mq_handler import MessageBroker
import pika
app = Flask(__name__)
api = Api(app)
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
mb = MessageBroker(connection)
class HelloWorld(Resource):
def get(self):
mb.run()
return {'hello': 'world'}
class LogHandler(Resource):
def get(self, table):
return {'TableName': table}
api.add_resource(HelloWorld, '/')
api.add_resource(LogHandler, '/log/<string:table>')
if __name__ == '__main__':
app.run(debug=True)
I have added a MessageBroker class to handle all my rabbitMq messages
import pika
import json
class MessageBroker:
def __init__(self, connection):
self.connection = connection
self.channel = connection.channel()
def run(self):
self.channel.start_consuming()
self.channel.basic_consume(queue='logs',
auto_ack=True,
on_message_callback=self.handle_log)
self.channel.start_consuming()
def handle_log(self, ch, method, properties, body):
decoded_content = body.decode('utf-8')
json_payload = json.loads(decoded_content)
print(" [x] Received %r" % json_payload['message'])
I have tried different solutions, but have can I get both services to run simultaneously on the same server? can somebody explain that please?
In general.. how is it possible to have several services running listening on my flask server?

I am not sure about running the consumer on an end-point will be a good idea. Because, when you start a consumer it runs an IO loop to fetch and process messages from the server continuously. The loop will not exit unless it is done externally or any exception in the message processing causing the connection to close. Can you please state your scenario for running the consumer in the end-point?

How to communicate RabbitMQ(Pika library) in tornado application

Pika library support tornado adapter, here is an example about how to publish message using Asynchronous adapter.
I want use pika in tornado application, just an example, I want put tornado request data to RabbitMQ, But don't know how to do it.
Two question don't know how to solve.
1 Pika use tornado adapter has its own ioloop,
self._connection = pika.SelectConnection(pika.URLParameters(self._url),
self.on_connection_open)
self._connection.ioloop.start()
Tornado application has its own ioloop,
tornado.ioloop.IOLoop.instance().start()
How to combine those two ioloop?
2 The Pika example publish same message again and again, but I want to publish request data, how to pass request data to publish method?

On my search for exactly the same thing I found this blog post of Kevin Jing Qiu.
I went the rabbitmq hole a bit further to give every websocket his own set of channel and queues.
The extract from my project can be found below. A tornado application bound to RabbitMQ consists of these parts:
The Tornado Application that will handle web requests. I only see long lived websockets here, but you can do so with short lived http requests as well.
A (one) RabbitMQ connection hold by the PikaClient Instance
a web connection that defines its channels, queues and exchanges when the open method is triggered.
Now a websocket connection can receive data from tornado (data from the browser) via on_message and send it to RabbitMQ.
The websocket connection will receive data from RabbitMQ via basic_consume.
This is not fully functional, but you should get the idea.
class PikaClient(object):
def __init__(self, io_loop):
logger.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 0
"""
Pika-Tornado connection setup
The setup process is a series of callback methods.
connect:connect to rabbitmq and build connection to tornado io loop ->
on_connected: create a channel to rabbitmq ->
on_channel_open: declare queue tornado, bind that queue to exchange
chatserver_out and start consuming messages.
"""
def connect(self):
if self.connecting:
#logger.info('PikaClient: Already connecting to RabbitMQ')
return
#logger.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected,stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
# now you are able to call the pika api to do things
# this could be exchange setup for websocket connections to
# basic_publish to later.
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('PikaClient: Channel %s open, Declaring exchange' % channel)
self.channel = channel
def on_closed(self, connection):
logger.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
class MyWebSocketHandler(websocket.WebSocketHandler):
def __init__(self):
self.status = 'not connected yet'
def open(self, *args, **kwargs):
self.status = "ws open"
self.rabbit_connect() # connect this websocket object to rabbitmq
def rabbit_connect():
self.application.pc.connection.channel(self.rabbit_channel_in_ok)
def rabbit_channel_in_ok(self,channel):
self.channel_in = channel
self.channel_in.queue_declare(self.rabbit_declare_ok,
exclusive=True,auto_delete=True)
# and so on...
handlers = [ your_definitions_here_like_websockets_or_such ]
settings = { your_settings_here }
application = tornado.web.Application(handlers,**settings)
def main():
io_loop = tornado.ioloop.IOLoop.instance()
# PikaClient is our rabbitmq consumer
pc = PikaClient(io_loop)
application.pc = pc
application.pc.connect()
application.listen(config.tornadoport)
try:
io_loop.start()
except KeyboardInterrupt:
io_loop.stop()
if __name__ == '__main__':
main()

Finally,i figure out it!
the previous solution is outdated for the newest pika component!
1.my pika Version is 1.0.1.
warning :
TornadoConnection Class has changed is package by the newest push request.
from pika.adapters import tornado_connection
2.there is an example: log() and config() should ignore(delete it)
import tornado.web
from handlers.notify import NotifyHandler
from function.function import config
from utils.utils import log
import pika
from pika.adapters import tornado_connection
HANDLERS = [(r'/notify', NotifyHandler)]
class PikaClient():
def __init__(self, io_loop):
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 9
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host="10.xxx.xxx.75", credentials=cred)
self.connection = tornado_connection.TornadoConnection(param, custom_ioloop = self.io_loop, on_open_callback = self.on_connected)
self.connection.add_on_open_error_callback(self.err)
self.connection.add_on_close_callback(self.on_closed)
def err(self, conn):
log('socket error', conn)
pass
def on_connected(self, conn):
log('connected')
self.connected = True
self.connection = conn
self.connection.channel(channel_number = 1, on_open_callback = self.on_channel_open)
def on_message(self, channel, method, properties, body):
log(body)
print('body : ', body)
pass
def on_channel_open(self, channel):
self.channel = channel
channel.basic_consume(on_message_callback = self.on_message, queue='hello', auto_ack=True)
return
def on_closed(self, conn, c):
log('pika close!')
self.io_loop.stop()
pass
def main():
port = 3002
is_debug = config('sys', 'debug')
print('DEBUG', is_debug)
app = tornado.web.Application(
HANDLERS,
debug = is_debug,
)
io_loop = tornado.ioloop.IOLoop.instance()
app.pc = PikaClient(io_loop)
app.pc.connect()
http_server = tornado.httpserver.HTTPServer(app)
app.listen(port)
io_loop.start()
print('listen {}'.format(port))
if __name__ == '__main__':
main()