What I understood from celery's documentations, when publishing tasks, you send them to exchange first and then exchange delegates it to queues. Now I want to send a task to specific custom made exchange which will delegate all tasks it receives to 3 different queues, which will have different consumers in the background, performing different tasks.
class Tasks(object):
def __init__(self, config_object={}):
self.celery = Celery()
self.celery.config_from_object(config_object)
self.task_publisher = task_publisher
def publish(self, task_name, job_id=None, params={}):
if not job_id:
job_id = uuid.uuid4()
self.celery.send_task(task_name, [job_id, params], queue='new_queue')
class config_object(object):
CELERY_IGNORE_RESULT = True
BROKER_PORT = 5672
BROKER_URL = 'amqp://guest:guest#localhost'
CELERY_RESULT_BACKEND = 'amqp'
tasks_service = Tasks(config_object)
tasks_service.publish('logger.log_event', params={'a': 'b'})
This is how I can send a task to specific queue, if I Dont define the queue it gets sent to a default one, but my question is how do I define the exchange to send to?
not sure if you have solved this problem.
i came across the same thing last week.
I am on Celery 4.1 and the solution I came up with was to just define the exchange name and the routing_key
so in your publish method, you would do something like:
def publish(self, task_name, job_id=None, params={}):
if not job_id:
job_id = uuid.uuid4()
self.celery.send_task(
task_name,
[job_id, params],
exchange='name.of.exchange',
routing_key='*'
)
Related
In the following code, an API gives a task to a task broker, who puts it in a queue, where it is picked up by a worker. The worker will then execute the task and notify the task broker (using a redis message channel) that he is done, after which the task broker will remove it from its queue. This works.
What I'd like is that the task broker is then able to return the result of the task to the API. But I'm unsure on how to do so since it is asynchronous code and I'm having difficulty figuring it out. Can you help?
Simplified the code is roughly as follows, but incomplete.
The API code:
#router.post('', response_model=BaseDocument)
async def post_document(document: BaseDocument):
"""Create the document with a specific type and an optional name given in the payload"""
task = DocumentTask({ <SNIP>
})
task_broker.give_task(task)
result = await task_broker.get_task_result(task)
return result
The task broker code, first part is giving the task, the second part is removing the task and the final part is what I assume should be a blocking call on the status of the removed task
def give_task(self, task_obj):
self.add_task_to_queue(task_obj)
<SNIP>
self.message_channel.publish(task_obj)
# ...
def remove_task_from_queue(self, task):
id_task_to_remove = task.id
for i in range(len(task_queue)):
if task_queue[i]["id"] == id_task_to_remove:
removed_task = task_queue.pop(i)
logger.debug(
f"[TaskBroker] Task with id '{id_task_to_remove}' succesfully removed !"
)
removed_task["status"] = "DONE"
return
# ...
async def get_task_result(self, task):
return task.result
My intuition would like to implement a way in get_task_result that blocks on task.result until it is modified, where I would modify it in remove_task_from_queue when it is removed from the queue (and thus done).
Any idea in how to do this, asynchronously?
I'm losing my mind trying to find a reliable and testable way to get the number of tasks contained in a given Celery queue.
I've already read these two related discussions:
Django Celery get task count
Note: I'm not using Django nor any other Python web framework.
Retrieve list of tasks in a queue in Celery
But I have not been able to solve my issue using the methods described in those threads.
I'm using Redis as backend, but I would like to have a backend independent and flexible solution, especially for tests.
This is my current situation: I've defined an EnhancedCelery class which inherits from Celery and adds a couple of methods, specifically get_queue_size() is the one I'm trying to properly implement/test.
The following is the code in my test case:
celery_test_app = EnhancedCelery(__name__)
# this is needed to avoid exception for ping command
# which is automatically triggered by the worker once started
celery_test_app.loader.import_module('celery.contrib.testing.tasks')
# in memory backend
celery_test_app.conf.broker_url = 'memory://'
celery_test_app.conf.result_backend = 'cache+memory://'
# We have to setup queues manually,
# since it seems that auto queue creation doesn't work in tests :(
celery_test_app.conf.task_create_missing_queues = False
celery_test_app.conf.task_default_queue = 'default'
celery_test_app.conf.task_queues = (
Queue('default', routing_key='task.#'),
Queue('queue_1', routing_key='q1'),
Queue('queue_2', routing_key='q2'),
Queue('queue_3', routing_key='q3'),
)
celery_test_app.conf.task_default_exchange = 'tasks'
celery_test_app.conf.task_default_exchange_type = 'topic'
celery_test_app.conf.task_default_routing_key = 'task.default'
celery_test_app.conf.task_routes = {
'sample_task': {
'queue': 'default',
'routing_key': 'task.default',
},
'sample_task_in_queue_1': {
'queue': 'queue_1',
'routing_key': 'q1',
},
'sample_task_in_queue_2': {
'queue': 'queue_2',
'routing_key': 'q2',
},
'sample_task_in_queue_3': {
'queue': 'queue_3',
'routing_key': 'q3',
},
}
#celery_test_app.task()
def sample_task():
return 'sample_task_result'
#celery_test_app.task(queue='queue_1')
def sample_task_in_queue_1():
return 'sample_task_in_queue_1_result'
#celery_test_app.task(queue='queue_2')
def sample_task_in_queue_2():
return 'sample_task_in_queue_2_result'
#celery_test_app.task(queue='queue_3')
def sample_task_in_queue_3():
return 'sample_task_in_queue_3_result'
class EnhancedCeleryTest(TestCase):
def test_get_queue_size_returns_expected_value(self):
def add_task(task):
task.apply_async()
with start_worker(celery_test_app):
for _ in range(7):
add_task(sample_task_in_queue_1)
for _ in range(4):
add_task(sample_task_in_queue_2)
for _ in range(2):
add_task(sample_task_in_queue_3)
self.assertEqual(celery_test_app.get_queue_size('queue_1'), 7)
self.assertEqual(celery_test_app.get_queue_size('queue_2'), 4)
self.assertEqual(celery_test_app.get_queue_size('queue_3'), 2)
Here are my attempts to implement get_queue_size():
This always returns zero (jobs == 0):
def get_queue_size(self, queue_name: str) -> Optional[int]:
with self.connection_or_acquire() as connection:
channel = connection.default_channel
try:
name, jobs, consumers = channel.queue_declare(queue=queue_name, passive=True)
return jobs
except (ChannelError, NotFound):
pass
This also always returns zero:
def get_queue_size(self, queue_name: str) -> Optional[int]:
inspection = self.control.inspect()
return inspection.active() # zero!
# or:
return inspection.scheduled() # zero!
# or:
return inspection.reserved() # zero!
This works by returning the expected number for each queue, but only in the test environment, because the channel.queues property does not exist when using the redis backend:
def get_queue_size(self, queue_name: str) -> Optional[int]:
with self.connection_or_acquire() as connection:
channel = connection.default_channel
if hasattr(channel, 'queues'):
queue = channel.queues.get(queue_name)
if queue is not None:
return queue.unfinished_tasks
None of the solutions you mentioned are entirely correct in my humble opinion. As you already mentioned this is backend-specific so you would have to wrap handlers for all backends supported by Celery to provide backend-agnostic queue inspection. In the Redis case you have to directly connect to Redis and LLEN the queue you want to inspect. In the case of RabbitMQ you find this information in completely different way. Same story with SQS...
This has all been discussed in the Retrieve list of tasks in a queue in Celery thread...
Finally, there is a reason why Celery does not provide this functionality out of box - the information is, I believe, useless. By the time you get what is in the queue it may already be empty!
If you want to monitor what is going on with your queues I suggest another approach. - Write your own real-time monitor. The example just captures task-failed events, but you should be able to modify it easily to capture all events you care about, and gather data about those tasks (queue, time, host it was executed on, etc). Clearly is an example how it is done in a more serious project.
You can see how it's implemented in the Flower (real-time monitor for Celery) here They have different Broker class implementation for redis and rabbitmq.
Another way - use celery's task events: calculate how many tasks were sent and how many were succeed/failed
I am using Celery with RabbitMQ as broker.
The code that creates Celery app instance is
from celery import Celery
name = __file__.split('.')[0]
app = Celery(name)
app.config_from_object('celery_config')
#app.task
def fetch_url(url):
resp = requests.get(url)
print resp.status_code
#app.task
def post(url, **kwargs):
body = kwargs.get(payload)
auth = kwrags.get(auth)
resp = requests.put(url, data=body, auth=auth)
Now I want to have 2 separate Queues, one for GET and one for POST.
Now I know that I must define the 2 queues in celery config module like
CELERY_QUEUES = (
Queue('default', Exchange('default'), routing_key='default'),
Queue('get', Exchange('get')),
Queue('post', Exchange('post')),
)
What I don't get is exactly what string to specify for the 'routing_key' option? Should it be the name of the tasks(get & post in this case) or there are rules for defining the routing_key?
No need to define queues or deal with routing keys, bindings or exchanges for simple task routing as your case. This is much simplified now (version 4.1) using the automatic routing.(http://docs.celeryproject.org/en/latest/userguide/routing.html#automatic-routing)
Name your tasks. For example, lets say you give names as below.
#app.task(name='get_task')
def fetch_url(url):
#app.task(name='post_task')
def post(url):
Add below line in your celery config file to route tasks to proper queues.
task_routes = {
'get_task': {'queue': 'get_queue'},
'post_task': {'queue': 'post_queue'}
}
Since task_create_missing_queues config is enabled by default, celery will take care of creating the queues for you.
I have three Celery tasks that run on three different servers respectively.
tasks.send_push_notification
tasks.send_sms
tasks.send_email
I want to setup a workflow such that if sending push notification fails, I should try sending sms. And if sending sms fails, I should send email.
If those 3 tasks and their code base was on the same server, I would have followed the example on chained tasks and done something like
from celery import chain
from tasks import send_push_notification, send_sms, send_email
import json
# some paylaod
payload = json.dumps({})
res = chain(
send_push_notification.subtask(payload),
send_sms.subtask(payload),
send_email.subtask(payload)
)()
But the tasks are kept on 3 different servers!
I have tried
# 1
from celery import chain
from my_celery_app import app
res = chain(
app.send_task('tasks.send_push_notification', payload),
app.send_task('tasks.send_sms', payload),
app.send_task('tasks.send_email', payload)
)()
# Which fails because I am chaining tasks not subtasks
and
# 2
from celery import chain, subtask
res = chain(
subtask('tasks.send_push_notification', payload),
subtask('tasks.send_sms', payload),
subtask('tasks.send_email', payload)
)()
# fails because I am not adding the tasks on the broker
How can this be done?
Update:
I can do it using link NOT chain.
from celery import subtask
res = app.send_task(
'tasks.send_push_notification', (payload, ),
link=subtask(
'tasks.send_sms', (payload, ),
link=subtask(
'tasks.send_email', (payload, ),
)
)
)
There is a lot of nesting. And because I actually need to create a database driven workflow, it will be complicated to create it this way.
Why not handle it in your tasks,
def push_notification_task(payload):
if not send_push_notification(payload):
sms_notification_task.delay(payload)
def sms_notification_task(payload):
if not send_sms_notification(payload):
email_notification_task.delay(payload)
def email_notification_task(payload):
send_email_notification(payload)
Moreover, chain will execute all of your tasks in the given order, whereas you want next task to run only if first failed.
To my knowledge, Celery acts as both the producer and consumer of messages. This is not what I want to achieve. I want Celery to act as the consumer only, to fire certain tasks based on messages that I send to my AMQP broker of choice. Is this possible?
Or do I need to make soup by adding carrot to my stack?
Celery brokers acts as a message stores and publish them to one or more workers that subscribe for those,
so: celery pulishes messages to a broker (rabbitmq, redist, celery itself through django db, etc..) those messages are retrieved by a worker following the protocol of the broker, that memorizes them (usually they are persistent but maybe it dependes on your broker), and got executed by you workers.
Task results are available on the executing worker task's, and you can configure where to store those results and you can retrieve them with this method .
You can publish tasks with celery passing parameters to your "receiver function" (the task you define, the documentation has some examples, usually you do not want to pass big things here (say a queryset), but only the minimal information that permits you to retrieve what you need when executing the task.
one easy example could be:
You register a task
#task
def add(x,x):
return x+y
and you call the from another module with:
from mytasks import add
metadata1 = 1
metadata2 = 2
myasyncresult = add.delay(1,2)
myasyncresult.get() == 3
EDIT
after your edit I saw that probably you want to construct messages from other sources other that celery, you could see here the message format, they default as pickled objects that respect that format, so you post those message in the right queue of your rabbitmq broker and you are right to go retrieving them from your workers.
Celery uses the message broker architectural pattern. A number of implementations / broker transports can be used with Celery including RabbitMQ and a Django database.
From Wikipedia:
A message broker is an architectural pattern for message validation, message transformation and message routing. It mediates communication amongst applications, minimizing the mutual awareness that applications should have of each other in order to be able to exchange messages, effectively implementing decoupling.
Keeping results is optional and requires a result backend. You can use different broker and result backends. The Celery Getting Started guide contains further information.
The answer to your question is yes you can fire specific tasks passing arguments without addding Carrot to the mix.
Celery Custom Consumer will be a feature released in 3.1v and now is under development, you can read http://docs.celeryproject.org/en/master/userguide/extending.html about it.
In order to consume message from celery you need to create message thats celery can consume. You can create celery message as follows:-
def get_celery_worker_message(task_name,args,kwargs,routing_key,id,exchange=None,exchange_type=None):
message=(args, kwargs, None)
application_headers={
'lang': 'py',
'task': task_name,
'id':id,
'argsrepr': repr(args),
'kwargsrepr': repr(kwargs)
#, 'origin': '#'.join([os.getpid(), socket.gethostname()])
}
properties={
'correlation_id':id,
'content_type': 'application/json',
'content_encoding': 'utf-8',
}
body, content_type, content_encoding = prepare(
message, 'json', 'application/json', 'utf-8',None, application_headers)
prep_message = prepare_message(body,None,content_type,content_encoding,application_headers,properties)
inplace_augment_message(prep_message, exchange, exchange_type, routing_key,id)
# dump_json = json.dumps(prep_message)
# print(f"json encoder:- {dump_json}")
return prep_message
You need to prep the message by first defining serializer, content_type, content_encoding, compression, headers based on the consumer.
def prepare( body, serializer=None, content_type=None,
content_encoding=None, compression=None, headers=None):
# No content_type? Then we're serializing the data internally.
if not content_type:
serializer = serializer
(content_type, content_encoding,
body) = dumps(body, serializer=serializer)
else:
# If the programmer doesn't want us to serialize,
# make sure content_encoding is set.
if isinstance(body, str):
if not content_encoding:
content_encoding = 'utf-8'
body = body.encode(content_encoding)
# If they passed in a string, we can't know anything
# about it. So assume it's binary data.
elif not content_encoding:
content_encoding = 'binary'
if compression:
body, headers['compression'] = compress(body, compression)
return body, content_type, content_encoding
def prepare_message( body, priority=None, content_type=None,
content_encoding=None, headers=None, properties=None):
"""Prepare message data."""
properties = properties or {}
properties.setdefault('delivery_info', {})
properties.setdefault('priority', priority )
return {'body': body,
'content-encoding': content_encoding,
'content-type': content_type,
'headers': headers or {},
'properties': properties or {}}
Once the message is created you need to add arguments to make it readable by celery consumer.
def inplace_augment_message(message, exchange,exchange_type, routing_key,next_delivery_tag):
body_encoding_64 = 'base64'
message['body'], body_encoding = encode_body(
str(json.dumps(message['body'])), body_encoding_64
)
props = message['properties']
props.update(
body_encoding=body_encoding,
delivery_tag=next_delivery_tag,
)
if exchange and exchange_type:
props['delivery_info'].update(
exchange=exchange,
exchange_type=exchange_type,
routing_key=routing_key,
)
elif exchange:
props['delivery_info'].update(
exchange=exchange,
routing_key=routing_key,
)
else:
props['delivery_info'].update(
exchange=None,
routing_key=routing_key,
)
class Base64:
"""Base64 codec."""
def encode(self, s):
return bytes_to_str(base64.b64encode(str_to_bytes(s)))
def decode(self, s):
return base64.b64decode(str_to_bytes(s))
def encode_body( body, encoding=None):
codecs = {'base64': Base64()}
if encoding:
return codecs.get(encoding).encode(body), encoding
return body, encoding