Develop Reference

how can i stop the consumer if the queue is empty RabbitMQ?

so im new to RabbitMQ, i have implemented a simple producer-consumer and for my use case i need to stop the consumer if the queue is empty but i can't find any solution.
sender:
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='hello')
channel.basic_publish(exchange='', routing_key='hello', body='Hello World!')
print(" [x] Sent 'Hello World!'")
connection.close()
reciver:
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='hello')
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
channel.basic_consume(queue='hello', on_message_callback=callback, auto_ack=True)
print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()

You can get the count of messages in queue, then exit the loop if its equal to 0.
import pika
connection = pika.BlockingConnection()
channel = connection.channel()
q = channel.queue_declare(q_name)
q_len = q.method.message_count

Alternatively, you could use a timeout functionality. As in, if after certain time passes by and your consumer is sitting idle, you can kill your worker/process/program.
import pika
import _thread
from threading import Timer
q_name = "hello"
conn = pika.BlockingConnection()
ch = conn.channel()
ch.queue_declare(q_name)
timeout_sec = 5 # times out in 5s
def timer():
return Timer(timeout_sec, lambda: _thread.interrupt_main())
def callback(t, ch, method, properties, body):
t.cancel()
print("[x] Received:", body)
t = timer()
t.start()
try:
t = timer()
t.start()
ch.basic_consume(
queue=q_name,
on_message_callback=lambda *args, **kwargs: callback(t, *args, **kwargs),
auto_ack=True,
)
ch.start_consuming()
except KeyboardInterrupt:
print("Nothing left in queue exiting....")
The above program will die if the consumer sits idle for 5 seconds.
Also note that my on_message_callback is encapsulating a regular callback that's expected by pika. That is essentially passing an instance of timer to pika's callback in order to start/stop the timer.

Python Pipe Hang

main0 doesn't hang but main1 hangs. Why??? I thought wrapping the thing into a class should be harmless ...
The child process should simply send the msg it receives back to the main process.
Python3 code:
from multiprocessing import Process, Pipe
def child(conn):
print("child started")
while True:
msg = conn.recv()
if msg == "quit":
break
print("child recv:"+msg)
print("child sending:" + msg)
conn.send(msg)
conn.close()
print("child ended")
def main0():
parent_conn, child_conn = Pipe()
p = Process(target=child, args=(child_conn,))
p.start()
parent_conn.send("ping")
print(parent_conn.recv())
parent_conn.send("quit")
print("#parent ended#")
p.join()
class Parent(object):
def __init__(self):
self.parent_conn = None
self.child_conn = None
self.p = None
def start(self):
self.parent_conn, self.child_conn = Pipe()
self.p = Process(target=child, args=(self.child_conn,))
self.p.start() # <--- i initially missed this line
print("started")
def send(self, msg):
print("try to send: " + msg)
self.parent_conn.send(msg)
return self.parent_conn.recv()
def close(self):
self.parent_conn.send("quit")
self.p.join()
def main1():
a = Parent()
a.start()
print(a.send("ping"))
print(a.send("quit"))
a.close()
if __name__ == '__main__':
main0() # doesn't hang
main1() # hangs.
output:
~~~ main 0 ~~~
child started
child recv:ping
child sending:ping
ping
#parent ended#
child ended
~~~ main 1 ~~~
started <Process(Process-1, started)>
try to send: ping
waiting to recv
child started
child recv:ping
child sending:ping
ping
try to send: quit
waiting to recv
child ended
*still hangs ... after adding self.p.start()*

I missed self.p.start() line at start method.
Also sending quit message to child would not return any message.
The correct main1() should be:
def main1():
a = Parent()
a.start()
print(a.send("ping"))
a.close()
Then this is fixed.

How to send message from the child process to websocket-client in Tornado?

I have the Tornado server. It receives messages from websocket-connections. I need to run the worker function as a separate process and the worker should answer to client. The main idea is to work in a parallel mode. Something like this
def worker(ws,message):
input = json.loads(message)
t = input["time"]
time.sleep(t)
ws.write_message("Hello, World!"*int(t))
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.render('index.html')
class WebSocket(tornado.websocket.WebSocketHandler):
def check_origin(self, origin):
return True
def open(self):
print("WebSocket opened")
self.application.webSocketsPool.append(self)
def on_message(self, message):
for key, value in enumerate(self.application.webSocketsPool):
if value == self:
p = Process(target=worker, args=(value.ws_connection,message,))
p.start()
def on_close(self):
print("WebSocket closed")
for key, value in enumerate(self.application.webSocketsPool):
if value == self:
del self.application.webSocketsPool[key]
Of course, this doesn't work because of pickling error. How to solve this problem?

Consuming on multiple topics with pika and TornadoConnection

I have a class for a pika consumer client that's based on pika's example code for the TornadoConnection. I'm trying to consume from a topic queue. The problem is that since the connection is established in an asynchronous way, there is no way for me to know when the channel is established or when the queue is declared. My class:
class PikaClient(object):
""" Based on:
http://pika.readthedocs.org/en/latest/examples/tornado_consumer.html
https://reminiscential.wordpress.com/2012/04/07/realtime-notification-delivery-using-rabbitmq-tornado-and-websocket/
"""
def __init__(self, exchange, exchange_type):
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self.exchange = exchange
self.exchange_type = exchange_type
self.queue = None
self.event_listeners = set([])
def connect(self):
logger.info('Connecting to RabbitMQ')
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred,
)
return pika.adapters.TornadoConnection(param,
on_open_callback=self.on_connection_open)
def close_connection(self):
logger.info('Closing connection')
self._connection.close()
def on_connection_closed(self, connection, reply_code, reply_text):
self._channel = None
if not self._closing:
logger.warning('Connection closed, reopening in 5 seconds: (%s) %s',
reply_code, reply_text)
self._connection.add_timeout(5, self.reconnect)
def on_connection_open(self, connection):
logger.info('Connected to RabbitMQ')
self._connection.add_on_close_callback(self.on_connection_closed)
self._connection.channel(self.on_channel_open)
def reconnect(self):
if not self._closing:
# Create a new connection
self._connection = self.connect()
def on_channel_closed(self, channel, reply_code, reply_text):
logger.warning('Channel %i was closed: (%s) %s',
channel, reply_code, reply_text)
self._connection.close()
def on_channel_open(self, channel):
logger.info('Channel open, declaring exchange')
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
self._channel.exchange_declare(self.on_exchange_declareok,
self.exchange,
self.exchange_type,
passive=True,
)
def on_exchange_declareok(self, unused_frame):
logger.info('Exchange declared, declaring queue')
self._channel.queue_declare(self.on_queue_declareok,
exclusive=True,
auto_delete=True,
)
def on_queue_declareok(self, method_frame):
self.queue = method_frame.method.queue
def bind_key(self, routing_key):
logger.info('Binding %s to %s with %s',
self.exchange, self.queue, routing_key)
self._channel.queue_bind(self.on_bindok, self.queue,
self.exchange, routing_key)
def add_on_cancel_callback(self):
logger.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
logger.info('Consumer was cancelled remotely, shutting down: %r',
method_frame)
if self._channel:
self._channel.close()
def on_message(self, unused_channel, basic_deliver, properties, body):
logger.debug('Received message # %s from %s',
basic_deliver.delivery_tag, properties.app_id)
#self.notify_listeners(body)
def on_cancelok(self, unused_frame):
logger.info('RabbitMQ acknowledged the cancellation of the consumer')
self.close_channel()
def stop_consuming(self):
if self._channel:
logger.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
def start_consuming(self):
logger.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self.on_message, no_ack=True)
def on_bindok(self, unused_frame):
logger.info('Queue bound')
self.start_consuming()
def close_channel(self):
logger.info('Closing the channel')
self._channel.close()
def open_channel(self):
logger.info('Creating a new channel')
self._connection.channel(on_open_callback=self.on_channel_open)
def run(self):
self._connection = self.connect()
def stop(self):
logger.info('Stopping')
self._closing = True
self.stop_consuming()
logger.info('Stopped')
An example for code using it (inside a WebSocketHandler.open):
self.pc = PikaClient('agents', 'topic')
self.pc.run()
self.pc.bind_key('mytopic.*')
When trying to run this, bind_key throws an exception because the _channel is still None. But I haven't found a way to block until the channel and queue are established. Is there any way to do this with a dynamic list of topics (that might change after the consumer starts running)?

You actually do have a way to know when the queue is established - the method on_queue_declareok(). That callback will executed once the self.queue_declare method has finished, and self.queue_declare is executed once _channel.exchance_declare has finished, etc. You can follow the chain all the way back to your run method:
run -> connect -> on_connection_open -> _connection.channel -> on_channel_open -> _channel.exchange_declare -> on_exchange_declareok -> _channel.queue_declare -> on_queue_declareok
So, you just add your call(s) to bind_key to on_queue_declareok, and that will trigger a call to on_bindok, which will call start_consuming. At that point your client is actually listening for messages. If you want to be able to dynamically provide topics, just take them in the constructor of PikaClient. Then you can call bind_key on each inside on_queue_declareok. You'd also need to add a flag to indicate you've already started consuming, so you don't try to do that twice.
Something like this (assume all methods not shown below stay the same):
def __init__(self, exchange, exchange_type, topics=None):
self._topics = [] if topics is None else topics
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._consuming = False
self.exchange = exchange
self.exchange_type = exchange_type
self.queue = None
self.event_listeners = set([])
def on_queue_declareok(self, method_frame):
self.queue = method_frame.method.queue
for topic in self._topics:
self.bind_key(topic)
def start_consuming(self):
if self._consuming:
return
logger.info('Issuing consumer related RPC commands')
self.add_on_cancel_callback()
self._consumer_tag = self._channel.basic_consume(self.on_message, no_ack=True)
self._consuming = True

how to run 2 different processes at the same time?

One process is receiving data from a socket and then putting it in a queue, and the other one is processing the queued data. How to make them both run at the same time?
This socket is serve_forever while the processing of data will only run if the queue is not empty.

i have it working. i don't know if this is THE ANSWER. but it's working, well, so far. maybe there are or there will be bugs (hopefully none). any suggestions for improvements is welcome.
import multiprocessing
import socket
from multiprocessing import Process, Queue
import time
def handle(connection, address):
try:
while True:
data = connection.recv(1024)
if data == "":
break
else :
print "RECEIVE DATA : " + str(data)
xdata = data.strip()
xdata = data.split(" ")
for xd in xdata :
print "PUT Task : " + str(xd)
QueueTask.put((xd), block=True, timeout=5)
connection.sendall(data)
except:
print "Problem handling request"
finally:
connection.close()
class Server(object):
def __init__(self, hostname, port):
self.hostname = hostname
self.port = port
def start(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.bind((self.hostname, self.port))
self.socket.listen(1)
while True:
conn, address = self.socket.accept()
process = multiprocessing.Process(target=handle, args=(conn, address))
process.daemon = True
process.start()
def f_Processor():
time.sleep(10)
print 'PROCESSOR Starting'
while 1:
try :
job = QueueTask.get(True,1)
print "GET Task : " + str(job)
time.sleep(5)
except Exception as err :
pass
print 'PROCESSOR Exiting'
if __name__ == "__main__":
server = Server("localhost", 9999)
QueueTask = Queue()
try:
p = multiprocessing.Process(name='Processing', target=f_Processor)
p.start()
server.start()
p.join()
except:
print "Unexpected exception"
finally:
for process in multiprocessing.active_children():
process.terminate()
process.join()
print "All done"

Also it's depend, if you have a server or client application. If it is a server than you can use
SocketServer.TCPServer.allow_reuse_address = True
self.server = TCPFactory( ( HOST, PORT ), TCPRequestHandler, params )
# Start a thread with the server
self.server_thread = threading.Thread( target = self.server.serve_forever )
self.server_thread.setDaemon( True )
self.server_thread.start()
class TCPFactory( SocketServer.ThreadingTCPServer ):
def __init__( self, server_address, RequestHandlerClass, params ):
"""
"""
SocketServer.ThreadingTCPServer.__init__( self, server_address, RequestHandlerClass )
self.patrams = params
class TCPRequestHandler( SocketServer.BaseRequestHandler ):
def setup( self ):
print self.server.params
pass
def handle( self ):
pass
So if a client connected to the server it will start a new thread. The setup and
handler function will be called automatically.
For the other thread you can use a timer, or an other thread
myt = Timer( 2, chackque, () )
myt.start()
def chackque():
if not myq.empty():
#Do what you want
or just start an other thread:
mythread = threading.Thread( target = chackque, args = ( myargs, ) )
mythread.setDaemon( True )
mythread.start()
def chackque():
while True:
if not myq.empty():
#Do what you want