I have a class for a pika consumer client that's based on pika's example code for the TornadoConnection. I'm trying to consume from a topic queue. The problem is that since the connection is established in an asynchronous way, there is no way for me to know when the channel is established or when the queue is declared. My class:
class PikaClient(object):
""" Based on:
http://pika.readthedocs.org/en/latest/examples/tornado_consumer.html
https://reminiscential.wordpress.com/2012/04/07/realtime-notification-delivery-using-rabbitmq-tornado-and-websocket/
"""
def __init__(self, exchange, exchange_type):
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self.exchange = exchange
self.exchange_type = exchange_type
self.queue = None
self.event_listeners = set([])
def connect(self):
logger.info('Connecting to RabbitMQ')
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred,
)
return pika.adapters.TornadoConnection(param,
on_open_callback=self.on_connection_open)
def close_connection(self):
logger.info('Closing connection')
self._connection.close()
def on_connection_closed(self, connection, reply_code, reply_text):
self._channel = None
if not self._closing:
logger.warning('Connection closed, reopening in 5 seconds: (%s) %s',
reply_code, reply_text)
self._connection.add_timeout(5, self.reconnect)
def on_connection_open(self, connection):
logger.info('Connected to RabbitMQ')
self._connection.add_on_close_callback(self.on_connection_closed)
self._connection.channel(self.on_channel_open)
def reconnect(self):
if not self._closing:
# Create a new connection
self._connection = self.connect()
def on_channel_closed(self, channel, reply_code, reply_text):
logger.warning('Channel %i was closed: (%s) %s',
channel, reply_code, reply_text)
self._connection.close()
def on_channel_open(self, channel):
logger.info('Channel open, declaring exchange')
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
self._channel.exchange_declare(self.on_exchange_declareok,
self.exchange,
self.exchange_type,
passive=True,
)
def on_exchange_declareok(self, unused_frame):
logger.info('Exchange declared, declaring queue')
self._channel.queue_declare(self.on_queue_declareok,
exclusive=True,
auto_delete=True,
)
def on_queue_declareok(self, method_frame):
self.queue = method_frame.method.queue
def bind_key(self, routing_key):
logger.info('Binding %s to %s with %s',
self.exchange, self.queue, routing_key)
self._channel.queue_bind(self.on_bindok, self.queue,
self.exchange, routing_key)
def add_on_cancel_callback(self):
logger.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
logger.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def on_message(self, unused_channel, basic_deliver, properties, body):
logger.debug('Received message # %s from %s',
basic_deliver.delivery_tag, properties.app_id)
#self.notify_listeners(body)
def on_cancelok(self, unused_frame):
logger.info('RabbitMQ acknowledged the cancellation of the consumer')
self.close_channel()
def stop_consuming(self):
if self._channel:
logger.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def start_consuming(self):
logger.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self.on_message, no_ack=True)
def on_bindok(self, unused_frame):
logger.info('Queue bound')
self.start_consuming()
def close_channel(self):
logger.info('Closing the channel')
self._channel.close()
def open_channel(self):
logger.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def run(self):
self._connection = self.connect()
def stop(self):
logger.info('Stopping')
self._closing = True
self.stop_consuming()
logger.info('Stopped')
An example for code using it (inside a WebSocketHandler.open):
self.pc = PikaClient('agents', 'topic')
self.pc.run()
self.pc.bind_key('mytopic.*')
When trying to run this, bind_key throws an exception because the _channel is still None. But I haven't found a way to block until the channel and queue are established. Is there any way to do this with a dynamic list of topics (that might change after the consumer starts running)?
You actually do have a way to know when the queue is established - the method on_queue_declareok(). That callback will executed once the self.queue_declare method has finished, and self.queue_declare is executed once _channel.exchance_declare has finished, etc. You can follow the chain all the way back to your run method:
run -> connect -> on_connection_open -> _connection.channel -> on_channel_open -> _channel.exchange_declare -> on_exchange_declareok -> _channel.queue_declare -> on_queue_declareok
So, you just add your call(s) to bind_key to on_queue_declareok, and that will trigger a call to on_bindok, which will call start_consuming. At that point your client is actually listening for messages. If you want to be able to dynamically provide topics, just take them in the constructor of PikaClient. Then you can call bind_key on each inside on_queue_declareok. You'd also need to add a flag to indicate you've already started consuming, so you don't try to do that twice.
Something like this (assume all methods not shown below stay the same):
def __init__(self, exchange, exchange_type, topics=None):
self._topics = [] if topics is None else topics
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._consuming = False
self.exchange = exchange
self.exchange_type = exchange_type
self.queue = None
self.event_listeners = set([])
def on_queue_declareok(self, method_frame):
self.queue = method_frame.method.queue
for topic in self._topics:
self.bind_key(topic)
def start_consuming(self):
if self._consuming:
return
logger.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self.on_message, no_ack=True)
self._consuming = True
Related
import gnsq
class something():
def __init__(self, pb=None, pk=None, address=None):
self.pb = pb
self.pk = pk
self.address = address
def connect(self):
consumer = gnsq.Consumer(self.pb, 'ch', self.address)
#consumer.on_message.connect
def response_handler(consumer, msg):
return msg.body
consumer.start()
how would i get the return value of response_handler so in turn, I'd be able to pass to the parent function connect(), so when i call it, it will be returning the value of message.body from the child function.
I would think something like the following:
import gnsq
class something():
def __init__(self, pb=None, pk=None, address=None):
self.pb = pb
self.pk = pk
self.address = address
def connect(self):
consumer = gnsq.Consumer(self.pb, 'ch', self.address)
#consumer.on_message.connect
def response_handler(consumer, msg):
return msg.body
consumer.start()
return response_handler
nsq = something('pb', 'pk', 'address')
# should print whatever message.body is
print nsq.connect()
but It's not working. Note: consumer.start() is blocking
What you're asking doesn't make sense in the context of what the Consumer() really is.
In your connect() method, you set up a consumer, set up a response handler and start the consumer with consumer.start(). From that point onward, whenever there is a message to consume, the consumer will call the handler with that message. Not just once, but again and again.
Your handler may be called many times and unless the consumer is closed, you never know when it will be done - so, there's no way your connect() method could return the complete result.
What you could do is have the connect method return a reference to a collection that will at any time contain all the messages collected so far. It would be empty at first, but after some time, could contain all the received messages.
Something like:
import gnsq
class Collector():
def __init__(self, topic, address):
self.topic = topic
self.address = address
self.messages = []
def connect(self):
self.messages = []
consumer = gnsq.Consumer(self.pb, 'ch', self.address)
#consumer.on_message.connect
def response_handler(consumer, msg):
self.messages.append(msg)
consumer.start()
return self.messages
I don't think this is really how you want to be using this, it would only really make sense if you provide more context on why and how you want to use this output.
I have two servers, created with asyncio.start_server:
asyncio.start_server(self.handle_connection, host = host, port = port) and running in one loop:
loop.run_until_complete(asyncio.gather(server1, server2))
loop.run_forever()
I'm using asyncio.Queue to communicate between servers. Messages from Server2, added via queue.put(msg) successfully receives by queue.get() in Server1. I'm running queue.get() by asyncio.ensure_future and using as callback for
add_done_callback method from Server1:
def callback(self, future):
msg = future.result()
self.msg = msg
But this callback not working as expected - self.msg do not updates. What am I doing wrong?
UPDATED
with additional code to show max full example:
class Queue(object):
def __init__(self, loop, maxsize: int):
self.instance = asyncio.Queue(loop = loop, maxsize = maxsize)
async def put(self, data):
await self.instance.put(data)
async def get(self):
data = await self.instance.get()
self.instance.task_done()
return data
#staticmethod
def get_instance():
return Queue(loop = asyncio.get_event_loop(), maxsize = 10)
Server class:
class BaseServer(object):
def __init__(self, host, port):
self.instance = asyncio.start_server(self.handle_connection, host = host, port = port)
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
pass
def get_instance(self):
return self.instance
#staticmethod
def create():
return BaseServer(None, None)
Next I'm running the servers:
loop.run_until_complete(asyncio.gather(server1.get_instance(), server2.get_instance()))
loop.run_forever()
In the handle_connection of server2 I'm calling queue.put(msg), in the handle_connection of server1 I'm registered queue.get() as task:
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
The process_queue method of server1:
def process_queue(self, future):
msg = future.result()
self.msg = msg
The handle_connection method of server1:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
task_queue = asyncio.ensure_future(queue.get())
task_queue.add_done_callback(self.process_queue)
while self.msg != SPECIAL_VALUE:
# doing something
Although task_queue is done, self.process_queue called, self.msg never updates.
Basically as you are using asynchronous structure, I think you can directly await the result:
async def handle_connection(self, reader: StreamReader, writer: StreamWriter):
msg = await queue.get()
process_queue(msg) # change it to accept real value instead of a future.
# do something
I have the Tornado server. It receives messages from websocket-connections. I need to run the worker function as a separate process and the worker should answer to client. The main idea is to work in a parallel mode. Something like this
def worker(ws,message):
input = json.loads(message)
t = input["time"]
time.sleep(t)
ws.write_message("Hello, World!"*int(t))
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.render('index.html')
class WebSocket(tornado.websocket.WebSocketHandler):
def check_origin(self, origin):
return True
def open(self):
print("WebSocket opened")
self.application.webSocketsPool.append(self)
def on_message(self, message):
for key, value in enumerate(self.application.webSocketsPool):
if value == self:
p = Process(target=worker, args=(value.ws_connection,message,))
p.start()
def on_close(self):
print("WebSocket closed")
for key, value in enumerate(self.application.webSocketsPool):
if value == self:
del self.application.webSocketsPool[key]
Of course, this doesn't work because of pickling error. How to solve this problem?
I am using tornado websocket for simple test code.
In the test code, i want to get tornado.websocket.WebSocketHandler.
For example, I used this way below.
class ConvPlayerInterface(object):
class WebsocketHandler(tornado.websocket.WebSocketHandler):
client = None
queue = ipcQueue.IpcQueue()
def open(self):
print 'new connection'
self.client = self #in my simple code, it handles only one client.
self.write_message("Connection Open")
def on_message(self, message):
self.queue.put(message)
def on_close(self):
print 'connection closed'
def __init__(self, url='/ws'):
self.application = tornado.web.Application([(url, self.WebsocketHandler),])
self.httpServer = tornado.httpserver.HTTPServer(self.application)
self.httpServer.listen(8888)
self.queue = self.WebsocketHandler.queue
self.ioLoop = threading.Thread(target = tornado.ioloop.IOLoop.instance().start)
def start(self):
self.ioLoop.start()
def get(self):
return self.queue.get()
def put(self, command):
self.WebsocketHandler.client.write_message(command)
But the point when it calls self.WebsocketHandler.client.write_message(command) in put() method, Python says client is Non type.
Any advice?
And how usually it is used to get client connection handler object in tornado?
In this part of your code
def put(self, command):
self.WebsocketHandler.client.write_message(command)
you are accessing to WebsocketHandler class, not a class member.
And the "client" attribute of WebsocketHandler is None, as expected.
WebsocketHandler instance will be created for each request tornado will accept, so there can be several websocket handlers simultaneously.
If you really want to have handle only one connection - you can do something like this:
class ConvPlayerInterface(object):
the_only_handler = None
class WebsocketHandler(tornado.websocket.WebSocketHandler):
client = None
queue = ipcQueue.IpcQueue()
def open(self):
print 'new connection'
ConvPlayerInterface.the_only_handler = self
self.write_message("Connection Open")
def on_message(self, message):
self.queue.put(message)
def on_close(self):
ConvPlayerInterface.the_only_handler = None
print 'connection closed'
def __init__(self, url='/ws'):
self.application = tornado.web.Application([(url, self.WebsocketHandler),])
self.httpServer = tornado.httpserver.HTTPServer(self.application)
self.httpServer.listen(8888)
self.queue = self.WebsocketHandler.queue
self.ioLoop = threading.Thread(target = tornado.ioloop.IOLoop.instance().start)
def start(self):
self.ioLoop.start()
def get(self):
return self.queue.get()
def put(self, command):
if self.the_only_handler is not None
self.the_only_handler.write_message(command)
I'm new to rabbitmq and pika, and is having trouble with stopping consuming.
channel and queue setting:
connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.queue_declare(queue=new_task_id, durable=True, auto_delete=True)
Basically, consumer and producer are like this:
consumer:
def task(task_id):
def callback(channel, method, properties, body):
if body != "quit":
print(body)
else:
print(body)
channel.stop_consuming(task_id)
channel.basic_consume(callback, queue=task_id, no_ack=True)
channel.start_consuming()
print("finish")
return "finish"
producer:
proc = Popen(['app/sample.sh'], shell=True, stdout=PIPE)
while proc.returncode is None: # running
line = proc.stdout.readline()
if line:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body=line
)
else:
channel.basic_publish(
exchange='',
routing_key=self.request.id,
body="quit"
)
break
consumer task gave me output:
# ... output from sample.sh, as expected
quit
�}q(UstatusqUSUCCESSqU tracebackqNUresultqNUtask_idqU
1419350416qUchildrenq]u.
However, "finish" didn't get printed, so I'm guessing it's because channel.stop_consuming(task_id) didn't stop consuming. If so, what is the correct way to do? Thank you.
I had the same problem. It seems to be caused by the fact that internally, start_consuming calls self.connection.process_data_events(time_limit=None). This time_limit=None makes it hang.
I managed to workaround this problem by replacing the call to channel.start_consuming() with its implemenation, hacked:
while channel._consumer_infos:
channel.connection.process_data_events(time_limit=1) # 1 second
I have a class defined with member variables of channel and connection. These are initialized by a seperate thread. The consumer of MyClient Class uses the close() method and the the connection and consumer is stopped!
class MyClient:
def __init__(self, unique_client_code):
self.Channel = None
self.Conn: pika.BlockingConnection = None
self.ClientThread = self.init_client_driver()
def _close_callback(self):
self.Channel.stop_consuming()
self.Channel.close()
self.Conn.close()
def _client_driver_thread(self, tmout=None):
print("Starting Driver Thread...")
self.Conn = pika.BlockingConnection(pika.ConnectionParameters("localhost"))
self.Channel = self.Conn.channel()
def init_client_driver(self, tmout=None):
kwargs = {'tmout': tmout}
t = threading.Thread(target=self._client_driver_thread, kwargs=kwargs)
t.daemon = True
t.start()
return t
def close(self):
self.Conn.add_callback_threadsafe(self._close_callback)
self.ClientThread.join()