i want to run a manage.py cmd from celery as a periodic task every x Minutes but every time i try to accomplish that as show below i get the following error:
[2019-01-17 01:36:00,006: INFO/MainProcess] Received task: Delete
unused media file(s)[3dd2b93b-e32a-4736-8b24-028b9ad8da35]
[2019-01-17 01:36:00,007: WARNING/ForkPoolWorker-3] Scanning for
unused media files [2019-01-17 01:36:00,008: WARNING/ForkPoolWorker-3]
Unknown command: 'cleanup_unused_media --noinput --remove-empty-dirs'
[2019-01-17 01:36:00,008: INFO/ForkPoolWorker-3] Task Delete unused
media file(s)[3dd2b93b-e32a-4736-8b24-028b9ad8da35] succeeded in
0.0008139749998008483s: None
tasks.py
from celery import Celery
from celery.schedules import crontab
from celery.task import periodic_task
from celery.utils.log import get_task_logger
import requests
from django.core import management
logger = get_task_logger(__name__)
app = Celery('tasks', broker='redis://127.0.0.1')
...
#periodic_task(run_every=(crontab(minute='*/90')), name="Delete unused media file(s)", ignore_result=True)
def delete_unused_media():
try:
print("Scanning for unused media files")
management.call_command('cleanup_unused_media --noinput --remove-empty-dirs')
return "success, old media files have been deleted"
except Exception as e:
print(e)
that management cmd comes from the following project:
https://github.com/akolpakov/django-unused-media
is my management call simply wrong or whats the deal?
thanks in advance
UPDATE (Celery Config):
settings.py
INSTALLED_APPS = [
...
'celery',
'django_unused_media',
...
# Celery Settings:
BROKER_URL = 'redis://127.0.0.1:6379'
CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'UTC'
...
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": [
"redis://127.0.0.1:6379/0",
#"redis://127.0.0.1:6379/1",
],
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"SOCKET_CONNECT_TIMEOUT": 15, # in seconds
"SOCKET_TIMEOUT": 15, # in seconds
"COMPRESSOR": "django_redis.compressors.zlib.ZlibCompressor",
"CONNECTION_POOL_KWARGS": {"max_connections": 1000, "retry_on_timeout": True}
}
}
}
celery.py
from __future__ import absolute_import, unicode_literals
from celery import Celery
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'MyProject.settings')
app = Celery('MyProject')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
_init__.py
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app')
e.g. This task is working fine:
#periodic_task(run_every=(crontab(minute='*/1')), name="Get exchange rate(s)", ignore_result=True)
def get_exchange_rate():
api_url = "https://api.coinmarketcap.com/v1/ticker/?limit=1"
try:
exchange_rate = requests.get(api_url).json()
logger.info("BTC Exchange rate updated successfully.")
except Exception as e:
print(e)
exchange_rate = dict()
return exchange_rate
The issue is the way you're calling call_command. call_command takes the name of the command as its first argument, followed by the arguments passed positionally. You're passing the whole lot as a single string. Try changing it to:
management.call_command('cleanup_unused_media', '--noinput', '--remove-empty-dirs')
For anyone using Django 1.7+, it seems that simply import the settings module is not enough.
i got it working like this
from celery import Celery
from celery.schedules import crontab
from celery.task import periodic_task
from celery.utils.log import get_task_logger
import requests, os, django
from django.core import management
logger = get_task_logger(__name__)
app = Celery('tasks', broker='redis://127.0.0.1')
#periodic_task(run_every=(crontab(minute='*/1')), name="Delete unused media file(s)", ignore_result=True)
def delete_unused_media():
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "MyProject.settings")
django.setup()
try:
print("Scanning for unused media files")
management.call_command('cleanup_unused_media', '--noinput', '--remove-empty-dirs')
return "success, old media files have been deleted"
except Exception as e:
print(e)
Related
When i'm trying to run celery background task then it giving me error: KeyError: 'myproject.tasks.async_task'
I'm using python version: 3.8.10, django verion: 4.1.2 , celery version: 5.2.7 and rabbitmq version: 3.8.2
here below screenshot is my project structure:
here below is my code for background task:
settings.py:
CELERY_BROKER_URL = 'amqp://localhost:5672'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
myproject/init.py:
from __future__ import absolute_import, unicode_literals
from myproject.celery import app as celery_app
__all__ = ['celery_app']
celery.py
from __future__ import absolute_import
import os, sys
from celery import Celery
import django
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myproject.settings')
app = Celery('myproject')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
tasks.py:
from __future__ import absolute_import
from celery import shared_task
from time import sleep
import os
#shared_task
def sleepy(duration):
sleep(duration)
return None
#shared_task
def async_task(save_path, name_of_file):
sleep(30)
completeName = os.path.join(save_path, name_of_file+".html")
file1 = open(completeName, "w")
toFile = 'test data'
file1.write(toFile)
file1.close()
return 'task complete'
views.py
def add_product(request):
if request.method == 'POST':
id_col_data = request.POST.getlist('services')
target_col = request.POST.get('target_col')
file_name = request.POST.get('file_name')
amz_columns_dict = {'id_col': id_col_data,
'target_col': target_col,
'wt_col': None}
import os.path
save_path = '/home/satyajit/Desktop/'
if os.path.exists(save_path):
try:
from myproject.tasks import async_task
name_of_file = file_name
status = async_task.delay(save_path, name_of_file) #celery task
print('status---->', status)
except Exception as e:
print('task error is ------>', e)
return render(request,'data/error.html', {'message': 'async task error'})
else:
print('error occured')
return HttpResponse('product added successfully')
return render(request, 'add_product.html')
When i run this celery command celery -A myproject worker -l info to check the celery process work then i get below trackback.
The traceback full details on below screenshot:
I'm trying to set up django so it send automatic email when a certain date in my models i reached. However i setup a Heroku-Redis server and am trying to connect to it. I created a simple task to test out if celery is working but it always returns the following error:
Error while reading from socket: (10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None)
I setup celery according to the website:
celery.py:
import os
from celery import Celery
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'incFleet.settings')
app = Celery('incFleet')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django apps.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print(f'Request: {self.request!r}')
Tasks.py
import datetime
from celery import shared_task, task
from time import sleep
from .models import trucks
from datetime import datetime
#shared_task()
def sleepy(duration):
sleep(duration)
return 0
##shared_task()
#def send_warning():
My views:
def index(request):
sleepy.delay(5)
return render(request, 'Inventory/index.html')
And my settings.py
# Celery Broker - Redis
CELERY_BROKER_URL = 'redis://:p0445df1196b44ba70a9bd0c84545315fec8a5dcbd77c8e8c4bd22ff4cd0a2ff4#ec2-54-167-58-171.compute-1.amazonaws.com:11900/'
CELERY_RESULT_BACKEND = 'redis://:p0445df1196b44ba70a9bd0c84545315fec8a5dcbd77c8e8c4bd22ff4cd0a2ff4#ec2-54-167-58-171.compute-1.amazonaws.com:11900/'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
After a while i realized the issue was with the network in my office. The port i was trying to access was closed. As soon as i activated redis on the actual server it started working fine.
heroku config | grep REDIS
check the actual url using this code
While calling the .delay() method of an imported task from a django application, the process gets stuck and the request is never completed.
We also don't get any error on the console.
Setting up a set_trace() with pdb results in the same thing.
The following questions were reviewed which didn't help resolve the issue:
Calling celery task hangs for delay and apply_async
celery .delay hangs (recent, not an auth problem)
Eg.:
backend/settings.py
CELERY_BROKER_URL = os.environ.get("CELERY_BROKER", RABBIT_URL)
CELERY_RESULT_BACKEND = os.environ.get("CELERY_BROKER", RABBIT_URL)
backend/celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings')
app = Celery('backend')
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
app/tasks.py
import time
from celery import shared_task
#shared_task
def upload_file(request_id):
time.sleep(request_id)
return True
app/views.py
from rest_framework.views import APIView
from .tasks import upload_file
class UploadCreateAPIView(APIView):
# other methods...
def post(self, request, *args, **kwargs):
id = request.data.get("id", None)
# business logic ...
print("Going to submit task.")
import pdb; pdb.set_trace()
upload_file.delay(id) # <- this hangs the runserver as well as the set_trace()
print("Submitted task.")
The issue was with the setup of the celery application with Django. We need to make sure that the celery app is imported and initialized in the following file:
backend\__init__.py
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app',)
I've run into this issue that Celery calls through delay or apply_async may randomly hang the program indefinitely. I tried the all broker_transport_options and retry_policy options to let Celery to recover, but it still happens. Then I found this solution to enforce an execution time limit for an execution block/function by using underlying Python signal handlers.
#contextmanager
def time_limit(seconds):
def signal_handler(signum, frame):
raise TimeoutException("Timed out!")
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
def my_function():
with time_limit(3):
celery_call.apply_sync(kwargs={"k1", "v1"}, expires=30)
Unable to run periodic tasks along with asynchronous tasks together. Although, if I comment out the periodic task, asynchronous tasks are executed fine, else asynchronous tasks are stuck.
Running: celery==4.0.2, Django==2.0, django-celery-beat==1.1.0, django-celery-results==1.0.1
Referred: https://github.com/celery/celery/issues/4184 to choose celery==4.0.2 version, as it seems to work.
Seems to be a known issue
https://github.com/celery/django-celery-beat/issues/27
I've also done some digging the ONLY way I've found to get it back to
normal is to remove all periodic tasks and restart celery beat. ~ rh0dium
celery.py
import django
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bid.settings')
# Setup django project
django.setup()
app = Celery('bid')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
settings.py
INSTALLED_APPS = (
...
'django_celery_results',
'django_celery_beat',
)
# Celery related settings
CELERY_BROKER_URL = 'redis://localhost:6379/0'
CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 43200, }
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_CONTENT_ENCODING = 'utf-8'
CELERY_ENABLE_REMOTE_CONTROL = False
CELERY_SEND_EVENTS = False
CELERY_TIMEZONE = 'Asia/Kolkata'
CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
Periodic task
#periodic_task(run_every=crontab(hour=7, minute=30), name="send-vendor-status-everyday")
def send_vendor_status():
return timezone.now()
Async task
#shared_task
def vendor_creation_email(id):
return "Email Sent"
Async task caller
vendor_creation_email.apply_async(args=[instance.id, ]) # main thread gets stuck here, if periodic jobs are scheduled.
Running the worker, with beat as follows
celery worker -A bid -l debug -B
Please help.
Here are a few observations, resulted from multiple trial and errors, and diving into celery's source code.
#periodic_task is deprecated. Hence it would not work.
from their source code:
#venv36/lib/python3.6/site-packages/celery/task/base.py
def periodic_task(*args, **options):
"""Deprecated decorator, please use :setting:`beat_schedule`."""
return task(**dict({'base': PeriodicTask}, **options))
Use UTC as base timezone, to avoid timezone related confusions later on. Configure periodic task to fire on calculated times with respect to UTC. e.g. for 'Asia/Calcutta' reduce the time by 5hours 30mins.
Create a celery.py as follows:
celery.py
import django
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
from celery.schedules import crontab
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')
# Setup django project
django.setup()
app = Celery('proj')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
app.conf.beat_schedule = {
'test_task': {
'task': 'test_task',
'schedule': crontab(hour=2,minute=0),
}
}
and task could be in tasks.py under any app, as follows
#shared_task(name="test_task")
def test_add():
print("Testing beat service")
Use celery worker -A proj -l info and celery beat -A proj -l info for worker and beat, along with a broker e.g. redis. and this setup should work fine.
I have the following setup in my django settings:
CELERY_TASK_RESULT_EXPIRES = timedelta(minutes=30)
CELERY_CHORD_PROPAGATES = True
CELERY_ACCEPT_CONTENT = ['json', 'msgpack', 'yaml']
CELERY_ALWAYS_EAGER = True
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
BROKER_URL = 'django://'
CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend'
I've included this under my installed apps:
'djcelery',
'kombu.transport.django'
My project structure is (django 1.5)
proj
|_proj
__init__.py
celery.py
|_apps
|_myapp1
|_models.py
|_tasks.py
This is my celery.py file:
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings.dev')
app = Celery('proj')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS, related_name='tasks')
In the main __init__.pyI have:
from __future__ import absolute_import
from .celery import app as celery_app
And finally in myapp1/tasks.py I define my task:
#task()
def retrieve():
# Do my stuff
Now, if I launch a django interactive shell and I launch the retrieve task:
result = retrieve.delay()
it always seems to be a blocking call, meaning that the prompt is bloked until the function returns. The result status is SUCCESS, the function actually performs the operations BUT it seems not to be async. What am I missing?
it seems like CELERY_ALWAYS_EAGER causes this
if this is True, all tasks will be executed locally by blocking until
the task returns. apply_async() and Task.delay() will return an
EagerResult instance, which emulates the API and behavior of
AsyncResult, except the result is already evaluated.
That is, tasks will be executed locally instead of being sent to the
queue.