Pika library support tornado adapter, here is an example about how to publish message using Asynchronous adapter.
I want use pika in tornado application, just an example, I want put tornado request data to RabbitMQ, But don't know how to do it.
Two question don't know how to solve.
1 Pika use tornado adapter has its own ioloop,
self._connection = pika.SelectConnection(pika.URLParameters(self._url),
self.on_connection_open)
self._connection.ioloop.start()
Tornado application has its own ioloop,
tornado.ioloop.IOLoop.instance().start()
How to combine those two ioloop?
2 The Pika example publish same message again and again, but I want to publish request data, how to pass request data to publish method?
On my search for exactly the same thing I found this blog post of Kevin Jing Qiu.
I went the rabbitmq hole a bit further to give every websocket his own set of channel and queues.
The extract from my project can be found below. A tornado application bound to RabbitMQ consists of these parts:
The Tornado Application that will handle web requests. I only see long lived websockets here, but you can do so with short lived http requests as well.
A (one) RabbitMQ connection hold by the PikaClient Instance
a web connection that defines its channels, queues and exchanges when the open method is triggered.
Now a websocket connection can receive data from tornado (data from the browser) via on_message and send it to RabbitMQ.
The websocket connection will receive data from RabbitMQ via basic_consume.
This is not fully functional, but you should get the idea.
class PikaClient(object):
def __init__(self, io_loop):
logger.info('PikaClient: __init__')
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 0
"""
Pika-Tornado connection setup
The setup process is a series of callback methods.
connect:connect to rabbitmq and build connection to tornado io loop ->
on_connected: create a channel to rabbitmq ->
on_channel_open: declare queue tornado, bind that queue to exchange
chatserver_out and start consuming messages.
"""
def connect(self):
if self.connecting:
#logger.info('PikaClient: Already connecting to RabbitMQ')
return
#logger.info('PikaClient: Connecting to RabbitMQ')
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected,stop_ioloop_on_close=False)
self.connection.add_on_close_callback(self.on_closed)
def on_connected(self, connection):
logger.info('PikaClient: connected to RabbitMQ')
self.connected = True
self.connection = connection
# now you are able to call the pika api to do things
# this could be exchange setup for websocket connections to
# basic_publish to later.
self.connection.channel(self.on_channel_open)
def on_channel_open(self, channel):
logger.info('PikaClient: Channel %s open, Declaring exchange' % channel)
self.channel = channel
def on_closed(self, connection):
logger.info('PikaClient: rabbit connection closed')
self.io_loop.stop()
class MyWebSocketHandler(websocket.WebSocketHandler):
def __init__(self):
self.status = 'not connected yet'
def open(self, *args, **kwargs):
self.status = "ws open"
self.rabbit_connect() # connect this websocket object to rabbitmq
def rabbit_connect():
self.application.pc.connection.channel(self.rabbit_channel_in_ok)
def rabbit_channel_in_ok(self,channel):
self.channel_in = channel
self.channel_in.queue_declare(self.rabbit_declare_ok,
exclusive=True,auto_delete=True)
# and so on...
handlers = [ your_definitions_here_like_websockets_or_such ]
settings = { your_settings_here }
application = tornado.web.Application(handlers,**settings)
def main():
io_loop = tornado.ioloop.IOLoop.instance()
# PikaClient is our rabbitmq consumer
pc = PikaClient(io_loop)
application.pc = pc
application.pc.connect()
application.listen(config.tornadoport)
try:
io_loop.start()
except KeyboardInterrupt:
io_loop.stop()
if __name__ == '__main__':
main()
Finally,i figure out it!
the previous solution is outdated for the newest pika component!
1.my pika Version is 1.0.1.
warning :
TornadoConnection Class has changed is package by the newest push request.
from pika.adapters import tornado_connection
2.there is an example: log() and config() should ignore(delete it)
import tornado.web
from handlers.notify import NotifyHandler
from function.function import config
from utils.utils import log
import pika
from pika.adapters import tornado_connection
HANDLERS = [(r'/notify', NotifyHandler)]
class PikaClient():
def __init__(self, io_loop):
self.io_loop = io_loop
self.connected = False
self.connecting = False
self.connection = None
self.channel = None
self.message_count = 9
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(host="10.xxx.xxx.75", credentials=cred)
self.connection = tornado_connection.TornadoConnection(param, custom_ioloop = self.io_loop, on_open_callback = self.on_connected)
self.connection.add_on_open_error_callback(self.err)
self.connection.add_on_close_callback(self.on_closed)
def err(self, conn):
log('socket error', conn)
pass
def on_connected(self, conn):
log('connected')
self.connected = True
self.connection = conn
self.connection.channel(channel_number = 1, on_open_callback = self.on_channel_open)
def on_message(self, channel, method, properties, body):
log(body)
print('body : ', body)
pass
def on_channel_open(self, channel):
self.channel = channel
channel.basic_consume(on_message_callback = self.on_message, queue='hello', auto_ack=True)
return
def on_closed(self, conn, c):
log('pika close!')
self.io_loop.stop()
pass
def main():
port = 3002
is_debug = config('sys', 'debug')
print('DEBUG', is_debug)
app = tornado.web.Application(
HANDLERS,
debug = is_debug,
)
io_loop = tornado.ioloop.IOLoop.instance()
app.pc = PikaClient(io_loop)
app.pc.connect()
http_server = tornado.httpserver.HTTPServer(app)
app.listen(port)
io_loop.start()
print('listen {}'.format(port))
if __name__ == '__main__':
main()
Related
Setup
I have a python application, which should consume messages from a RabbitMQ and act as a SocketIO server to a Vue2 APP. When it receives messages from RabbitMQ it should send out a message over SocketIO to the Vue2 APP. Therefore I wrote 2 classes RabbitMQHandler and SocketIOHandler. I am starting the RabbitMQHandler in a separate thread so that both the RabbitMQ consume and the wsgi server can run in parallel.
Code
import random
import threading
import socketio
import eventlet
import sys
import os
import uuid
import pika
from dotenv import load_dotenv
import logging
class RabbitMQHandler():
def __init__(self, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
self.queue_name = 'myqueue'
self.exchange_name = 'myqueue'
credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PW)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(RABBITMQ_IP, 5672, '/', credentials))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name)
self.channel.exchange_declare(exchange=self.exchange_name, exchange_type='fanout')
self.channel.queue_bind(exchange=self.exchange_name, queue=self.queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
def run(self, callback):
logging.info('start consuming messages...')
self.channel.basic_consume(queue=self.queue_name,auto_ack=True, on_message_callback=callback)
self.channel.start_consuming()
class SocketIOHandler():
def __init__(self):
self.id = str(uuid.uuid4())
# create a Socket.IO server
self.sio = socketio.Server(async_mode='eventlet', cors_allowed_origins='*')
# wrap with a WSGI application
self.app = socketio.WSGIApp(self.sio)
self.sio.on('connect_to_backend', self.handle_connect)
self.sio.on('random_number', self.handle_random_number)
def handle_connect(self, sid, msg):
logging.info('new socket io message')
self.emit('connect_success', {
'success': True,
})
def handle_random_number(self, sid, msg):
logging.info('handle_random_number')
self.emit('response_random_number', { 'number': random.randint(0,10)})
def emit(self, event, msg):
logging.info('socket server: {}'.format(self.id))
logging.info('sending event: "{}"'.format(event))
self.sio.emit(event, msg)
logging.info('sent event: "{}"'.format(event))
def run(self):
logging.info('start web socket on port 8765...')
eventlet.wsgi.server(eventlet.listen(('', 8765)), self.app)
def start_rabbitmq_handler(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
def callback(ch, method, properties, body):
logging.info('rabbitmq handler')
socketio_handler.emit('response_random_number', { 'number': random.randint(0,10)})
with RabbitMQHandler(RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP) as rabbitmq_handler:
rabbitmq_handler.run(callback=callback)
threads = []
def main():
global threads
load_dotenv()
RABBITMQ_USER = os.getenv('RABBITMQ_USER')
RABBITMQ_PW = os.getenv('RABBITMQ_PW')
RABBITMQ_IP = os.getenv('RABBITMQ_IP')
socketio_handler = SocketIOHandler()
rabbitmq_thread = threading.Thread(target=start_rabbitmq_handler, args=(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP))
threads.append(rabbitmq_thread)
rabbitmq_thread.start()
socketio_handler.run()
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").propagate = False
main()
except KeyboardInterrupt:
try:
for t in threads:
t.exit()
sys.exit(0)
except SystemExit:
for t in threads:
t.exit()
os._exit(0)
Problem
The Problem is, that when the RabbitMQHandler receives a message the event response_random_number does not get through to the Vue2 APP. Even though it is emited in the callback function. When I send the random_number event from the Vue2 APP to the python application I do get the response_random_number event back from the python application in the Vue2 APP.
So all connections work on their own, but not together. My guess would be, that there is some sort of threading communication error. I added the id to the SocketIOHandler class to make sure it is the same instanced object and the prints are the same.
The logs 'socket server: ...', sending event: ... and sent event: ... tell me, that the function is being called correctly.
I am building a service that receives messages from rabbitmq using pika. and push messages to clients using socket.io.
The socket.io server and pika server are both blocking the main thread.
This will be the same also for celery with flask or Django.
What is the proper approach to solving this and run them both under the same context?
You can use the Pub/Sub model, Start the consume process in another thread register user that want to receive from the queue and send data to subscribed users.
import json
import pika
import gevent
from flask import Flask
from flask_sockets import Sockets
connection_url = 'localhost'
channel_queue = 'test'
class PubSubListener(threading.Thread):
def __init__(self, queue_name):
threading.Thread.__init__(self)
self.clients = []
self.queue_name = queue_name
connection = pika.BlockingConnection(pika.ConnectionParameters(connection_url))
self.channel = connection.channel()
self.channel.queue_declare(queue=self.queue_name)
threading.Thread(target=self.channel.basic_consume(queue=self.queue_name,
auto_ack=True,
on_message_callback=self._callback))
def run(self):
self.channel.start_consuming()
def publish(self, body):
self.channel.basic_publish(exchange='',
routing_key=self.queue_name,
body=body)
def subscribe(self, client):
self.clients.append(client)
def _callback(self, channel, method, properties, body):
time.sleep(0.001)
message = json.loads(body)
print(message)
self.send(message)
def send(self, data):
for client in self.clients:
try:
client.send(data)
except Exception:
self.clients.remove(client)
pslistener = PubSubListener(channel_queue)
app = Flask(__name__)
sockets = Sockets(app)
#sockets.route('/echo')
def echo_socket(ws):
pslistener.subscribe(ws)
while not ws.closed:
gevent.sleep(0.1)
#app.route('/')
def hello():
return 'Hello World!'
if __name__ == "__main__":
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
pslistener.start()
print("Started")
server = pywsgi.WSGIServer(('', 5000), app, handler_class=WebSocketHandler)
server.serve_forever()
I'm trying to integrate an aiohttp web server into a Crossbar+Autobahn system architecture.
More in detail, when the aiohttp server receive a certain API call, it has to publish a message to a Crossbar router.
I've seen this example on the official repos but i've no clue on how to integrate it on my application.
Ideally, i would like to be able to do this
# class SampleTaskController(object):
async def handle_get_request(self, request: web.Request) -> web.Response:
self.publisher.publish('com.myapp.topic1', 'Hello World!')
return web.HTTPOk()
where self il an instance of SampleTaskController(object) which defines all the routes handler of the web server.
def main(argv):
cfg_path = "./task_cfg.json"
if len(argv) > 1:
cfg_path = argv[0]
logging.basicConfig(level=logging.DEBUG,
format=LOG_FORMAT)
loop = zmq.asyncio.ZMQEventLoop()
asyncio.set_event_loop(loop)
app = web.Application(loop=loop)
with open(cfg_path, 'r') as f:
task_cfg = json.load(f)
task_cfg['__cfg_path'] = cfg_path
controller = SampleTaskController(task_cfg)
controller.restore()
app['controller'] = controller
controller.setup_routes(app)
app.on_startup.append(controller.on_startup)
app.on_cleanup.append(controller.on_cleanup)
web.run_app(app,
host=task_cfg['webserver_address'],
port=task_cfg['webserver_port'])
Notice that i'm using an zmq.asyncio.ZMQEventLoop because the server is also listening on a zmq socket, which is configured inside the controller.on_startup method.
Instead of using autobahn, i've also tried to publish the message to Crossbar using wampy and it works, but the autobahn subscribers couldn't correctly parse the message.
# autobahn subscriber
class ClientSession(ApplicationSession):
async def onJoin(self, details):
self.log.info("Client session joined {details}", details=details)
self.log.info("Connected: {details}", details=details)
self._ident = details.authid
self._type = u'Python'
self.log.info("Component ID is {ident}", ident=self._ident)
self.log.info("Component type is {type}", type=self._type)
# SUBSCRIBE
def gen_on_something(thing):
def on_something(counter, id, type):
print('----------------------------')
self.log.info("'on_{something}' event, counter value: {message}",something=thing, message=counter)
self.log.info("from component {id} ({type})", id=id, type=type)
return on_something
await self.subscribe(gen_on_something('landscape'), 'landscape')
await self.subscribe(gen_on_something('nature'), 'nature')
-
# wampy publisher
async def publish():
router = Crossbar(config_path='./crossbar.json')
logging.getLogger().debug(router.realm)
logging.getLogger().debug(router.url)
logging.getLogger().debug(router.port)
client = Client(router=router)
client.start()
result = client.publish(topic="nature", message=0)
logging.getLogger().debug(result)
With this configuration the subscriber receive the message published, but it get an exception while parsing it.
TypeError: on_something() got an unexpected keyword argument 'message'
Recently I tried to use aiohttp and autobahn simultaneously. I reworked the example from crossbar documentation (originally using twisted) and got the following code:
import asyncio
import logging
from aiohttp import web
from aiohttp.web_exceptions import HTTPOk, HTTPInternalServerError
from autobahn.asyncio.component import Component
# Setup logging
logger = logging.getLogger(__name__)
class WebApplication(object):
"""
A simple Web application that publishes an event every time the
url "/" is visited.
"""
count = 0
def __init__(self, app, wamp_comp):
self._app = app
self._wamp = wamp_comp
self._session = None # "None" while we're disconnected from WAMP router
# associate ourselves with WAMP session lifecycle
self._wamp.on('join', self._initialize)
self._wamp.on('leave', self._uninitialize)
self._app.router.add_get('/', self._render_slash)
def _initialize(self, session, details):
logger.info("Connected to WAMP router (session: %s, details: %s)", session, details)
self._session = session
def _uninitialize(self, session, reason):
logger.warning("Lost WAMP connection (session: %s, reason: %s)", session, reason)
self._session = None
async def _render_slash(self, request):
if self._session is None:
return HTTPInternalServerError(reason="No WAMP session")
self.count += 1
self._session.publish(u"com.myapp.request_served", self.count, count=self.count)
return HTTPOk(text="Published to 'com.myapp.request_served'")
def main():
REALM = "crossbardemo"
BROKER_URI = "ws://wamp_broker:9091/ws"
BIND_ADDR = "0.0.0.0"
BIND_PORT = 8080
logging.basicConfig(
level='DEBUG',
format='[%(asctime)s %(levelname)s %(name)s:%(lineno)d]: %(message)s')
logger.info("Starting aiohttp backend at %s:%s...", BIND_ADDR, BIND_PORT)
loop = asyncio.get_event_loop()
component = Component(
transports=BROKER_URI,
realm=REALM,
)
component.start(loop=loop)
# When not using run() we also must start logging ourselves.
import txaio
txaio.start_logging(level='info')
app = web.Application(
loop=loop)
_ = WebApplication(app, component)
web.run_app(app, host=BIND_ADDR, port=BIND_PORT)
if __name__ == '__main__':
main()
I'm working on a project which uses python asyncio socket server. The problem is that the implementation of the server doesn't call .close() on the transport when the server stops. This seems to leave clients connected and causes crashes in other parts of the code.
Python documents say that transports need to be closed explicitly, but in this project I don't know where I can close them because there is no reference to the transports that are created for each client.
https://docs.python.org/3/library/asyncio-dev.html#close-transports-and-event-loops
Here is the code:
"""
Socket server forwarding request to internal server
"""
import logging
try:
# we prefer to use bundles asyncio version, otherwise fallback to trollius
import asyncio
except ImportError:
import trollius as asyncio
from opcua import ua
from opcua.server.uaprocessor import UaProcessor
logger = logging.getLogger(__name__)
class BinaryServer(object):
def __init__(self, internal_server, hostname, port):
self.logger = logging.getLogger(__name__)
self.hostname = hostname
self.port = port
self.iserver = internal_server
self.loop = internal_server.loop
self._server = None
self._policies = []
def set_policies(self, policies):
self._policies = policies
def start(self):
class OPCUAProtocol(asyncio.Protocol):
"""
instanciated for every connection
defined as internal class since it needs access
to the internal server object
FIXME: find another solution
"""
iserver = self.iserver
loop = self.loop
logger = self.logger
policies = self._policies
def connection_made(self, transport):
self.peername = transport.get_extra_info('peername')
self.logger.info('New connection from %s', self.peername)
self.transport = transport
self.processor = UaProcessor(self.iserver, self.transport)
self.processor.set_policies(self.policies)
self.data = b""
def connection_lost(self, ex):
self.logger.info('Lost connection from %s, %s', self.peername, ex)
self.transport.close()
self.processor.close()
def data_received(self, data):
logger.debug("received %s bytes from socket", len(data))
if self.data:
data = self.data + data
self.data = b""
self._process_data(data)
def _process_data(self, data):
buf = ua.utils.Buffer(data)
while True:
try:
backup_buf = buf.copy()
try:
hdr = ua.Header.from_string(buf)
except ua.utils.NotEnoughData:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
if len(buf) < hdr.body_size:
logger.info("We did not receive enough data from client, waiting for more")
self.data = backup_buf.read(len(backup_buf))
return
ret = self.processor.process(hdr, buf)
if not ret:
logger.info("processor returned False, we close connection from %s", self.peername)
self.transport.close()
return
if len(buf) == 0:
return
except Exception:
logger.exception("Exception raised while parsing message from client, closing")
self.transport.close()
break
coro = self.loop.create_server(OPCUAProtocol, self.hostname, self.port)
self._server = self.loop.run_coro_and_wait(coro)
print('Listening on {}:{}'.format(self.hostname, self.port))
def stop(self):
self.logger.info("Closing asyncio socket server")
self.loop.call_soon(self._server.close)
self.loop.run_coro_and_wait(self._server.wait_closed())
As you can see when we call stop() on this server class the asyncio server calls it's close method. However if clients are connected the created transports never get closed.
The project repository is here https://github.com/FreeOpcUa/python-opcua/ , you can take a look at Issue 137.
What is the correct way to close the transport object?
I solve this by applying this approach:
#self.OPCUAServer - this is my opcua server
nodes = []
nodes.append(self.OPCUAServer.get_node("ns=0; s=Measurements")) #Adding two root nodes
nodes.append(self.OPCUAServer.get_node("ns=1; s=Calibrations")) #to the list
self.OPCUAServer.delete_nodes(nodes, True) # Recursively call delete_nodes with this list
self.OPCUAServer.stop()
I'm trying to send a message (a string) to a server (say server A) as a response to a particular string message received from the server. I've used a client connection to send messages to the server. I've created my own server (say server B) using socket module to receive the messages from the the above mentioned server. If a certain message is received, to client should send a message to that server A. I used threads so that both my client and server can work concurrently.
Here's my client script (socket1.py).
import socket
import time
s = socket.socket()
s2 = socket.socket()
class MyClient():
def __init__(self):
self.s=socket.socket()
def start(self):
self.s.connect(('localhost',6000))
self.s.settimeout(1000)
self.s.send('JOIN#')
while True:
if self.s.gettimeout():
break
def move(self):
self.s.send('LEFT#')
def close(self):
self.s.close()
def socStart():
global s,s2
s.connect(('localhost',6000))
#s2.connect(('localhost',6000))
s.send('JOIN#')
#time.sleep(10)
#s.send('DOWN#')
#s.close()
def move():
global s1
s1.send('DOWN#')
def close():
global s,s2
s.close()
#s2.close()
Here's my server script (server1.py)
import socket
class MyServer():
def __init__(self):
self.s= socket.socket()
def serStart(self):
self.s.bind(('localhost',7000))
#self.s.settimeout(30)
self.s.listen(5)
self.started = False
while True:
self.c, self.addr = self.s.accept()
self.msg = self.c.recv(1024)
if self.msg[0] == 'S':
self.started = True
print self.addr, '>>', self.msg
self.c.close()
if self.s.gettimeout():
break
self.s.close()
def getStarted(self):
return self.started
Here's the script in which I used threads with both socket1 & server1 module.
import threading
import datetime
import socket1
import server1
import time
class ThreadClass(threading.Thread):
def initialize(self,client,server):
self.client = client
self.server = server
def run(self):
self.client.start()
if self.server.getStarted():
self.client.move()
self.client.close()
class ThreadServer(threading.Thread):
def initialize(self, client, server):
self.client = client
self.server = server
def run(self):
self.server.serStart()
client = socket1.MyClient()
server = server1.MyServer()
t = ThreadClass()
tS=ThreadServer()
t.initialize(client, server)
tS.initialize(client, server)
t.start()
tS.start()
Though server class changes it's started varialble to True the move method doesn't work in ThreadClass. How do I perform such event triggering and responding accordingly with a outside server in python?