Develop Reference

How to fetch a record from db asynchronously while unit testing?

In this unit test I would like to check if the device has been created and that the expiry date is 7 days in future.
database.py
import databases
database = databases.Database(settings.sqlalchemy_database_uri)
Unit Test:
from database.database import database
def test_successful_register_expiry_set_to_seven_days():
response = client.post(
"/register/",
headers={},
json={"device_id": "u1"},
)
assert response.status_code == 201
query = device.select(whereclause=device.c.id == "u1")
d = database.fetch_one(query)
assert d.expires_at == datetime.utcnow().replace(microsecond=0) + timedelta(days=7)
Because d is a coroutine object it fails with the message:
AttributeError: 'coroutine' object has no attribute 'expires_at'
And I can't use await inside a unit test.
d = await database.fetch_one(query)
What am I missing, please?

Well it is never awaited, your code returns the coroutine before that semaphore going in to the scheduler.
If you are using an asynchronous driver, you need to await it. Are there any workarounds for this? Yes.
You can use asyncio.run(awaitable) to run the coroutine inside an event loop.
import asyncio
d = asyncio.run(database.fetch_one(query))
If you have an currently running event loop you may want to use that event loop instead. You can achieve that by asyncio.get_event_loop(), which will run the function inside the running loop.
import asyncio
asyncio.get_event_loop().run_until_complete(database.fetch_one(query))
You can also use #pytest.mark.asyncio decorator (see documentation).
#pytest.mark.asyncio
async def dummy():
await some_awaitable()

What is the simple way to run a non-blocking coroutine?

In Go language, running another task in non-blocking way is pretty simple.
import "net/http"
func Call(request http.Request) http.Response {
response := getResponse(request)
go doTask(request) // Non-blocking. No need to wait for result.
return response
}
func getResponse(request http.Request) http.Response {
// Do something synchronously here
return http.Response{}
}
func doTask(r http.Request) {
// Some task which takes time to finish
}
How can I achieve this in python?
I tried like this:
import asyncio
import threading
from asyncio.events import AbstractEventLoop
loop: AbstractEventLoop
def initialize():
global loop
loop = asyncio.new_event_loop()
thread = threading.Thread(target=run_event_loop)
thread.start()
def run_event_loop():
loop.run_forever()
def call(request):
response = get_response(request)
# This should not block
asyncio.run_coroutine_threadsafe(do_task(request), loop)
return response
def get_response(r):
# Do something synchronously here
return 42
async def do_task(r):
# Some task which takes time to finish
return
This works, but it is kind of cumbersome.
Besides, the python code only make use of one thread for tasks, while Go automatically dispatches tasks to multiple processes (and so does Kotlin).
Is there better way?

We can run non-blocking tasks by using the concurrent.futures module. The ThreadPoolExecutor object is used for this purpose. You could modify your code to use this approach by making the following adjustments:
from concurrent.futures import ThreadPoolExecutor
def call(request):
response = get_response(request)
with ThreadPoolExecutor(max_workers=4) as e:
e.submit(do_task)
return response
def get_response(r):
# Do something synchronously here
return 42
async def do_task(r):
# Some task which takes time to finish
return
response would be returned instantly and do_task would run after the result is returned.

Write back through the callback attached to IOLoop in Tornado

There is a tricky post handler, sometimes it can take a lots of time (depending on a input values), sometimes not.
What I want is to write back whenever 1 second passes, dynamically allocating the response.
def post():
def callback():
self.write('too-late')
self.finish()
timeout_obj = IOLoop.current().add_timeout(
dt.timedelta(seconds=1),
callback,
)
# some asynchronous operations
if not self.request.connection.stream.closed():
self.write('here is your response')
self.finish()
IOLoop.current().remove_timeout(timeout_obj)
Turns out I can't do much from within callback.
Even raising an exception is suppressed by the inner context and won't be passed through the post method.
Any other ways to achieve the goal?
Thank you.
UPD 2020-05-15:
I found similar question
Thanks #ionut-ticus, using with_timeout() is much more convenient.
After some tries, I think I came really close to what i'm looking for:
def wait(fn):
#gen.coroutine
#wraps(fn)
def wrap(*args):
try:
result = yield gen.with_timeout(
dt.timedelta(seconds=20),
IOLoop.current().run_in_executor(None, fn, *args),
)
raise gen.Return(result)
except gen.TimeoutError:
logging.error('### TOO LONG')
raise gen.Return('Next time, bro')
return wrap
#wait
def blocking_func(item):
time.sleep(30)
# this is not a Subprocess.
# It is a file IO and DB
return 'we are done here'
Still not sure, should wait() decorator being wrapped in a
coroutine?
Some times in a chain of calls of a blocking_func(), there can
be another ThreadPoolExecutor. I have a concern, would this work
without making "mine" one global, and passing to the
Tornado's run_in_executor()?
Tornado: v5.1.1

An example of usage of tornado.gen.with_timeout. Keep in mind the task needs to be async or else the IOLoop will be blocked and won't be able to process the timeout:
#gen.coroutine
def async_task():
# some async code
#gen.coroutine
def get(self):
delta = datetime.timedelta(seconds=1)
try:
task = self.async_task()
result = yield gen.with_timeout(delta, task)
self.write("success")
except gen.TimeoutError:
self.write("timeout")

I'd advise to use https://github.com/aio-libs/async-timeout:
import asyncio
import async_timeout
def post():
try:
async with async_timeout.timeout(1):
# some asynchronous operations
if not self.request.connection.stream.closed():
self.write('here is your response')
self.finish()
IOLoop.current().remove_timeout(timeout_obj)
except asyncio.TimeoutError:
self.write('too-late')
self.finish()

Python tornado gen.coroutine blocks request

I am newbie on tornado and python. A couple days ago i started to write a non-blocking rest api, but i couldn't accomplish the mission yet. When i send two request to this endpoint "localhost:8080/async" at the same time, the second request takes response after 20 seconds! That explains i am doing something wrong.
MAX_WORKERS = 4
class ASYNCHandler(tornado.web.RequestHandler):
executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
counter = 0
def pow_task(self, x, y):
time.sleep(10)
return pow(x,y)
async def background_task(self):
future = ASYNCHandler.executor.submit(self.pow_task, 2, 3)
return future
#gen.coroutine
def get(self, *args, **kwargs):
future = yield from self.background_task()
response= dumps({"result":future.result()}, default=json_util.default)
print(response)
application = tornado.web.Application([
('/async', ASYNCHandler),
('/sync', SYNCHandler),
], db=db, debug=True)
application.listen(8888)
tornado.ioloop.IOLoop.current().start()

Never use time.sleep in Tornado code! Use IOLoop.add_timeout to schedule a callback later, or in a coroutine yield gen.sleep(n).
http://www.tornadoweb.org/en/latest/faq.html#why-isn-t-this-example-with-time-sleep-running-in-parallel

That's strange that returning the ThreadPoolExecutor future, essentially blocks tornado's event loop. If anyone from the tornado team reads this and knows why that is, can they please give an explaination? I had planned to do some stuff with threads in tornado but after dealing with this question, I see that it's not going to be as simple as I originally anticipated. In any case, here is the code which does what you expect (I've trimmed your original example down a bit so that anyone can run it quickly):
from concurrent.futures import ThreadPoolExecutor
from json import dumps
import time
from tornado.platform.asyncio import to_tornado_future
from tornado.ioloop import IOLoop
from tornado import gen, web
MAX_WORKERS = 4
class ASYNCHandler(web.RequestHandler):
executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
counter = 0
def pow_task(self, x, y):
time.sleep(5)
return pow(x,y)
async def background_task(self):
future = self.executor.submit(self.pow_task, 2, 3)
result = await to_tornado_future(future) # convert to tornado future
return result
#gen.coroutine
def get(self, *args, **kwargs):
result = yield from self.background_task()
response = dumps({"result": result})
self.write(response)
application = web.Application([
('/async', ASYNCHandler),
], debug=True)
application.listen(8888)
IOLoop.current().start()
The main differences are in the background_tasks() method. I convert the asyncio future to a tornado future, wait for the result, then return the result. The code you provided in the question, blocked for some reason when yielding from background_task() and you were unable to await the result because the future wasn't a tornado future.
On a slightly different note, this simple example can easily be implemented using a single thread/async designs and chances are your code can also be done without threads. Threads are easy to implement but equally easy to get wrong and can lead to very sticky situations. When attempting to write threaded code please remember this photo :)

How to combine Celery with asyncio?

How can I create a wrapper that makes celery tasks look like asyncio.Task? Or is there a better way to integrate Celery with asyncio?
#asksol, the creator of Celery, said this::
It's quite common to use Celery as a distributed layer on top of async I/O frameworks (top tip: routing CPU-bound tasks to a prefork worker means they will not block your event loop).
But I could not find any code examples specifically for asyncio framework.

EDIT: 01/12/2021 previous answer (find it at the bottom) didn't age well therefore I added a combination of possible solutions that may satisfy those who still look on how to co-use asyncio and Celery
Lets quickly break up the use cases first (more in-depth analysis here: asyncio and coroutines vs task queues):
If the task is I/O bound then it tends to be better to use coroutines and asyncio.
If the task is CPU bound then it tends to be better to use Celery or other similar task management systems.
So it makes sense in the context of Python's "Do one thing and do it well" to not try and mix asyncio and celery together.
BUT what happens in cases where we want to be able to run a method both asynchronously and as an async task? then we have some options to consider:
The best example that I was able to find is the following: https://johnfraney.ca/posts/2018/12/20/writing-unit-tests-celery-tasks-async-functions/ (and I just found out that it is #Franey's response):
Define your async method.
Use asgiref's sync.async_to_sync module to wrap the async method and run it synchronously inside a celery task:
# tasks.py
import asyncio
from asgiref.sync import async_to_sync
from celery import Celery
app = Celery('async_test', broker='a_broker_url_goes_here')
async def return_hello():
await asyncio.sleep(1)
return 'hello'
#app.task(name="sync_task")
def sync_task():
async_to_sync(return_hello)()
A use case that I came upon in a FastAPI application was the reverse of the previous example:
An intense CPU bound process is hogging up the async endpoints.
The solution is to refactor the async CPU bound process into a celery task and pass a task instance for execution from the Celery queue.
A minimal example for visualization of that case:
import asyncio
import uvicorn
from celery import Celery
from fastapi import FastAPI
app = FastAPI(title='Example')
worker = Celery('worker', broker='a_broker_url_goes_here')
#worker.task(name='cpu_boun')
def cpu_bound_task():
# Does stuff but let's simplify it
print([n for n in range(1000)])
#app.get('/calculate')
async def calculate():
cpu_bound_task.delay()
if __name__ == "__main__":
uvicorn.run('main:app', host='0.0.0.0', port=8000)
Another solution seems to be what #juanra and #danius are proposing in their answers, but we have to keep in mind that performance tends to take a hit when we intermix sync and async executions, thus those answers need monitoring before we can decide to use them in a prod environment.
Finally, there are some ready-made solutions, that I cannot recommend (because I have not used them myself) but I will list them here:
Celery Pool AsyncIO which seems to solve exactly what Celery 5.0 didn't, but keep in mind that it seems a bit experimental (version 0.2.0 today 01/12/2021)
aiotasks claims to be "a Celery like task manager that distributes Asyncio coroutines" but seems a bit stale (latest commit around 2 years ago)
Well that didn't age so well did it? Version 5.0 of Celery didn't implement asyncio compatibility thus we cannot know when and if this will ever be implemented... Leaving this here for response legacy reasons (as it was the answer at the time) and for comment continuation.
That will be possible from Celery version 5.0 as stated on the official site:
http://docs.celeryproject.org/en/4.0/whatsnew-4.0.html#preface
The next major version of Celery will support Python 3.5 only, where we are planning to take advantage of the new asyncio library.
Dropping support for Python 2 will enable us to remove massive amounts of compatibility code, and going with Python 3.5 allows us to take advantage of typing, async/await, asyncio, and similar concepts there’s no alternative for in older versions.
The above was quoted from the previous link.
So the best thing to do is wait for version 5.0 to be distributed!
In the meantime, happy coding :)

This simple way worked fine for me:
import asyncio
from celery import Celery
app = Celery('tasks')
async def async_function(param1, param2):
# more async stuff...
pass
#app.task(name='tasks.task_name', queue='queue_name')
def task_name(param1, param2):
asyncio.run(async_function(param1, param2))

You can wrap any blocking call into a Task using run_in_executor as described in documentation, I also added in the example a custom timeout:
def run_async_task(
target,
*args,
timeout = 60,
**keywords
) -> Future:
loop = asyncio.get_event_loop()
return asyncio.wait_for(
loop.run_in_executor(
executor,
functools.partial(target, *args, **keywords)
),
timeout=timeout,
loop=loop
)
loop = asyncio.get_event_loop()
async_result = loop.run_until_complete(
run_async_task, your_task.delay, some_arg, some_karg=""
)
result = loop.run_until_complete(
run_async_task, async_result.result
)

Here is a simple helper that you can use to make a Celery task awaitable:
import asyncio
from asgiref.sync import sync_to_async
# Converts a Celery tasks to an async function
def task_to_async(task):
async def wrapper(*args, **kwargs):
delay = 0.1
async_result = await sync_to_async(task.delay)(*args, **kwargs)
while not async_result.ready():
await asyncio.sleep(delay)
delay = min(delay * 1.5, 2) # exponential backoff, max 2 seconds
return async_result.get()
return wrapper
Like sync_to_async, it can be used as a direct wrapper:
#shared_task
def get_answer():
sleep(10) # simulate long computation
return 42
result = await task_to_async(get_answer)()
...and as a decorator:
#task_to_async
#shared_task
def get_answer():
sleep(10) # simulate long computation
return 42
result = await get_answer()
Of course, this is not a perfect solution since it relies on polling.
However, it should be a good workaround to call Celery tasks from Django async views until Celery officially provides a better solution.
EDIT 2021/03/02: added the call to sync_to_async to support eager mode.

The cleanest way I've found to do this is to wrap the async function in asgiref.sync.async_to_sync (from asgiref):
from asgiref.sync import async_to_sync
from celery.task import periodic_task
async def return_hello():
await sleep(1)
return 'hello'
#periodic_task(
run_every=2,
name='return_hello',
)
def task_return_hello():
async_to_sync(return_hello)()
I pulled this example from a blog post I wrote.

I solved problem by combining Celery and asyncio in the celery-pool-asyncio library.

Here's my implementation of Celery handling async coroutines when necessary:
Wrap the Celery class to extend its functionnality:
from celery import Celery
from inspect import isawaitable
import asyncio
class AsyncCelery(Celery):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.patch_task()
if 'app' in kwargs:
self.init_app(kwargs['app'])
def patch_task(self):
TaskBase = self.Task
class ContextTask(TaskBase):
abstract = True
async def _run(self, *args, **kwargs):
result = TaskBase.__call__(self, *args, **kwargs)
if isawaitable(result):
await result
def __call__(self, *args, **kwargs):
asyncio.run(self._run(*args, **kwargs))
self.Task = ContextTask
def init_app(self, app):
self.app = app
conf = {}
for key in app.config.keys():
if key[0:7] == 'CELERY_':
conf[key[7:].lower()] = app.config[key]
if 'broker_transport_options' not in conf and conf.get('broker_url', '')[0:4] == 'sqs:':
conf['broker_transport_options'] = {'region': 'eu-west-1'}
self.config_from_object(conf)
celery = AsyncCelery()

For anyone who stumbles on this looking for help specifically with async sqlalchemy (ie, using the asyncio extension) and Celery tasks, explicitly disposing of the engine will fix the issue. This particular example worked with asyncpg.
Example:
from sqlalchemy.ext.asyncio import (
AsyncSession,
create_async_engine,
)
from sqlalchemy.orm import sessionmaker
from asgiref.sync import async_to_sync
engine = create_async_engine("some_uri", future=True)
async_session_factory = sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
#celery_app.task(name="task-name")
def sync_func() -> None:
async_to_sync(some_func)()
async def some_func() -> None:
async with get_db_session() as session:
result = await some_db_query(session)
# engine.dispose will be called on exit
#contextlib.asynccontextmanager
async def get_db_session() -> AsyncGenerator:
try:
db = async_session_factory()
yield db
finally:
await db.close()
await engine.dispose()

A nice way to implement Celery with asyncio:
import asyncio
from celery import Celery
app = Celery()
async def async_function(param):
print('do something')
#app.task()
def celery_task(param):
loop = asyncio.get_event_loop()
return loop.run_until_complete(async_function(param))