def response(return_JSON):
connection = pika.BlockingConnection(pika.ConnectionParameters(host=HOST))
channel = connection.channel()
channel.exchange_declare(exchange=EXCHANGE, type='direct', durable='True')
channel.queue_declare(queue=QUEUE_NAME, durable='True')
channel.queue_bind(exchange=EXCHANGE, queue=QUEUE_NAME)
channel.basic_publish(exchange=EXCHANGE, routing_key=RESPONSE_ROUTING_KEY, body=return_JSON)
connection.close()
print " [x] Finished sending msg to sender..."
def callback(ch, method, properties, body):
print " [x] Received message from sender!"
print body
result_JSON = Work().compute(json.loads(body))
if result_JSON is not None:
#result_JSON = json.dumps(result_JSON)
response(result_JSON)
channel.basic_ack(delivery_tag = method.delivery_tag)
print "Acknowledged to sender"
if __name__ == '__main__':
connection = pika.BlockingConnection(pika.ConnectionParameters(host=HOST))
channel = connection.channel()
channel.queue_declare(queue=QUEUE_NAME, durable='True')
channel.basic_consume(callback, queue=QUEUE_NAME, no_ack=False)
print " [*] Waiting for messages. To exit press CTRL+C"
channel.start_consuming()
This often works for a while and then falls over. Only need to consume messages one at a time, process them and send back a response message with a different routing KEY. Is there a smarter more reliable way to create a minimal consumer processing messages one at a time.
Error message is:
Connection reset by peer
Related
I have deployed python script in azure kubernetes services which will make connection with rabbitmq to get input data and process it and pass the output data back to the rabbitmq in the form of messages , but i am getting below error when there is no data or messages in the queue for long time , i want my script should wait until data comes in rabbitmq
below is the error:
error: http2: client connection lost
Below is my code to connect with rabbitmq in python
def callback(ch, method, properties, body):
try:
rabbit_data=[]
ch.exchange_declare(exchange='DataEx', exchange_type='fanout',durable=True)
if body != None:
rabbit_data.append(body)
if len(rabbit_data) > 0:
for i in rabbit_data:
final_packet=call_it(i)
if final_packet:
a_logger.info(str(len(final_packet)))
#out = { "reading_data": json.loads(a),
# "emission_data":json.loads(b)}
for data_output in final_packet:
ch.basic_publish(exchange='DataEx',routing_key='',
body=json.dumps(data_output),
properties=pika.BasicProperties(
content_type="text/plain",
delivery_mode=2))
#
rabbit_data.remove(i)
else:
#ch.basic_reject(delivery_tag = method.delivery_tag,requeue=True)
pass
else:
ch.stop_consuming()
ch.basic_ack(delivery_tag = method.delivery_tag,multiple=False)
except Exception as e:
a_logger.info("error in rabbitmq callback as : {}".format(str(e)))
credentials = pika.PlainCredentials(username, password)
parameters = pika.ConnectionParameters(host=Api_url,virtual_host=rmqvhost,credentials=credentials,heartbeat=0)
print (username,password)
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='Plume',durable=True)
channel.basic_qos(prefetch_size=0,prefetch_count=1) # this is for acknowdeging packet one by one
channel.basic_consume(queue='Plume', on_message_callback=callback,auto_ack=False)
a_logger.info(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()
I need to bind an exchange (amq.topic) with a specific queue in Python.
How I can do this?
Now I bind with the RabbitMQ's GUI.
I'm refering to this TAB of Rabbit
I've done only the binding of a queue with this code:
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='topic_logs', exchange_type='topic')
result = channel.queue_declare(queue='coda-di-prova', exclusive=False)
queue_name = result.method.queue
channel.queue_bind(exchange='topic_logs', queue=queue_name, routing_key="test.*")
print(' [*] Waiting for logs. To exit press CTRL+C')
def callback(ch, method, properties, body):
print(" [x] %r:%r" % (method.routing_key, body))
channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True)
channel.start_consuming()
What I need to add in my code in order to get this result?
Result IMAGE
I maybe didn't understand the question. But it is enough to use amq.topic
import pika
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
result = channel.queue_declare(queue='coda-di-prova', exclusive=False)
queue_name = result.method.queue
channel.queue_bind(exchange='amq.topic', queue=queue_name, routing_key="test.*")
print(' [*] Waiting for logs. To exit press CTRL+C')
def callback(ch, method, properties, body):
print(" [x] %r:%r" % (method.routing_key, body))
channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True)
channel.start_consuming()
I have a RabbitMQ consumer. I would like to have that consumer do some message processing, simulated by time.sleep(10), then publish a message to a different queue. I know the consumer callback has a channel that in theory could be used to do the publish, but this seems like a bad implementation because if the basic_publish() somehow manages for force close the channel, then the consumer dies. What is the best way to handle this?
import time
import pika
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='logs', exchange_type='fanout')
result = channel.queue_declare(queue='original_queue', exclusive=True)
channel.queue_bind(exchange='logs', queue='original_queue')
print(' [*] Waiting for logs. To exit press CTRL+C')
def callback(ch, method, properties, body):
time.sleep(10)
ch.basic_publish(exchange='logs', routing_key='different_queue', body='hello_world')
channel.basic_consume(
queue='original_queue', on_message_callback=callback, auto_ack=True)
channel.start_consuming()
You can implement your consumer in a way that it automatically reconnects to the RabbitMQ server if the connection gets closed. Hope this helps(I didn't put much thought on the design part, feel free to suggest some!)
import time
import pika
reconnect_on_failure = True
def consumer(connection, channel):
channel.exchange_declare(exchange='logs', exchange_type='fanout')
result = channel.queue_declare(queue='original_queue', exclusive=True)
channel.queue_bind(exchange='logs', queue='original_queue')
print(' [*] Waiting for logs. To exit press CTRL+C')
def callback(ch, method, properties, body):
time.sleep(10)
ch.basic_publish(exchange='logs', routing_key='different_queue', body='hello_world')
channel.basic_consume(
queue='original_queue', on_message_callback=callback, auto_ack=True)
channel.start_consuming()
def get_connection_and_channel():
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
def start(reconnect_on_failure):
connection, channel = get_connection_and_channel()
consumer(connection, channel)
# the if condition will be executed when the consumer's start_consuming loop exists
if reconnect_on_failure:
# cleanly close the connection and channel
if not connection.is_closed():
connection.close()
if not channel.is_close():
channel.close()
start(reconnect_on_failure)
start(reconnect_on_failure)
I am just starting to use RabbitMQ and Python in general. I have been reading the tuts on the rabbit official page, but I have no idea how to use Rabbitmq to do another things.
I have been trying to run the example of this [tutorial] (https://www.rabbitmq.com/tutorials/tutorial-three-python.html), It runs well,.. BUT I need to know how can I create more than one fucntions and call them throug Rabbitmq Messages?... (I am also using this [example] (Python and RabbitMQ - Best way to listen to consume events from multiple channels?) to guide me. )
I hope someone have some idea how to do this... (I will repeat again, I am very new on this topics)...
This some code what i have.
I use this code to send the message as the tutorial..
import pika
import sys
url = 'amqp://oamogcgg:xxxxxxxxxxxxxxxxxxxxxxxxx#salamander.rmq.cloudamqp.com/oamogcgg'
params = pika.URLParameters(url)
connection = pika.BlockingConnection(params)
channel = connection.channel()
channel.exchange_declare(exchange='logs', exchange_type='fanout')
message = ' '.join(sys.argv[1:]) or "info: THIS IS A TEST MESSAGE !!!!!!"
channel.basic_publish(exchange='logs', routing_key='', body=message)
print(" [x] Sent %r" % message)
connection.close()
And in this file is where I receive the message according to me.
import pika
import sys
import threading
threads=[]
#function 1
def validator1(channel):
channel.queue_declare(queue='queue_name')
print (' [*] Waiting messsaes for valiadtor1 press CTRL+C')
def callback(ch, method, properties, body):
print (" Received %s" % (body))
sleep(2) #I need stop it for two minutes
channel.basic_consume(callback, queue='queue_name', no_ack=True)
channel.start_consuming()
#function 2
def validator2(channel):
channel.queue_declare(queue='queue_name')
print (' [*] Waiting messsaes for valiadtor2 press CTRL+C')
def callback(ch, method, properties, body):
print (" Received %s" % (body))
sleep(2) #I need stop it for two minutes
channel.basic_consume(callback, queue='queue_name', no_ack=True)
channel.start_consuming()
def manager():
url = 'amqp://oamogcgg:xxxxxxxxxxxxxxxxxxxxxxxxx#salamander.rmq.cloudamqp.com/oamogcgg'
params = pika.URLParameters(url)
#channel 1
connection1= pika.BlockingConnection(params)
channel1 = connection1.channel()
channel1.exchange_declare(exchange='logs', exchange_type='fanout')
result = channel1.queue_declare(queue='', exclusive=True)
queue_name = result.method.queue
channel1.queue_bind(exchange='logs', queue=queue_name)
#channel 2
connection2= pika.BlockingConnection(params)
channel2 = connection2.channel()
channel2.exchange_declare(exchange='logs', exchange_type='fanout')
result = channel2.queue_declare(queue='', exclusive=True)
queue_name = result.method.queue
channel2.queue_bind(exchange='logs', queue=queue_name)
#creating threads
t1 = threading.Thread(target=validator1, args=(channel1,))
t1.daemon = True
threads.append(t1)
t1.start()
t2 = threading.Thread(target=valiadtor2, args=(channel2,))
t2.daemon = True
threads.append(t2)
t2.start()
for t in threads:
t.join()
manager()
If your functions are dependent(by dependent I mean, one function works on the output of the other) then, you can call all those functions one by one in your callback function. Once the last function is executed successfully you can acknowledge the message.
If those functions are independent then you can maintain multiple queues for each of the function you wanna execute on the message. The same message can be routed to multiple queues using a fanout exchange as mentioned in RabbitMQ Tutorial-3.
I have implemented RabbitMQ in my servers. So basically what it does is that the main server passes messages to the worker server.
The problem that I am facing is that all the message that I pass is not received by the server.
i.e if i send 10 messages only 4 of them are received.
Any idea where am I going wrong.
Receiving code
import pika
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='hello')
def callback(ch, method, properties, body):
print(" [x] Received %r" % body)
channel.basic_consume(callback,
queue='hello',
no_ack=True)
print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()
Publishing code
import pika
import sys
connection = pika.BlockingConnection(pika.ConnectionParameters(
host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue', durable=True)
message = ' '.join(sys.argv[1:]) or "Hello World!"
channel.basic_publish(exchange='',
routing_key='task_queue',
body=message,
properties=pika.BasicProperties(
delivery_mode = 2, # make message persistent
))
print(" [x] Sent %r" % message)
connection.close()
Assuming that you are publishing to the same queue (as the examples you posted shows otherwise). I would recommend that you enable the confirm delivery flag. This will ensure that your message gets delivered, and if not it will either throw an exception, or publish will return False.
channel = connection.channel()
channel.confirm_delivery()
published = channel.basic_publish(...)
if not published:
raise Exception("Unable to publish message!")
It might also be worth to install the management plugin for RabbitMQ and inspect the queue before you start consuming messages. This way you can verify that the messages got published, and later consumed.