Develop Reference

How to have a non-blocking indefinite loop that can be interrupted in Python?

I am trying to create a process that would run indefinitely until being shut down (either from the command prompt or from another notebook cell) by changing a class member. My attempt looks like the follow:
def async_run(*awaitables: Awaitable,
timeout: float = None):
loop = asyncio.get_event_loop()
if not awaitables:
if loop.is_running():
return
loop.run_forever()
result = None
if sys.version_info >= (3, 7):
all_tasks = asyncio.all_tasks(loop) # type: ignore
else:
all_tasks = asyncio.Task.all_tasks() # type: ignore
if all_tasks:
# cancel pending tasks
f = asyncio.gather(*all_tasks)
f.cancel()
try:
loop.run_until_complete(f)
except asyncio.CancelledError:
pass
else:
if len(awaitables) == 1:
future = awaitables[0]
else:
future = asyncio.gather(*awaitables)
if timeout:
future = asyncio.wait_for(future, timeout)
task = asyncio.ensure_future(future)
def onError(_):
task.cancel()
global_error_event.connect(onError)
try:
result = loop.run_until_complete(task)
except asyncio.CancelledError as e:
raise global_error_event.value() or e
finally:
global_error_event.disconnect(onError)
return result
class Platform:
def __init__(self):
self.is_active: bool = True
async def kickstart(self):
while self.is_active:
await async_run(asyncio.sleep(0))
However it seems that the kickstart() function would simply finish running, even though my intention is for it to run indefinitely. If I remove the async, it does run indefinitely but then it becomes blocking. I'd appreciate some help on this, thanks in advance.

How to correctly specify timeout in ThreadPoolExecutor (Python)?

I have the following code where I am trying to call functions with different timeouts. It might happen that the first function times out but the second one could have been executed in the specified time.
import time
from concurrent.futures import ThreadPoolExecutor
def test1(a, b, c):
time.sleep(5)
d=a+b+c
print(d)
def test2(a, b):
time.sleep(5)
d=a+b
print(d)
with ThreadPoolExecutor(max_workers=1) as executor1:
try:
executor1.submit(test1, 1,2,3).result(timeout=1)
except:
executor1.shutdown(wait=False)
print("Pass")
with ThreadPoolExecutor(max_workers=1) as executor2:
try:
executor2.submit(test2, 1,2).result(timeout=8)
except:
executor2.shutdown(wait=False)
print("Pass-2")
Expected Output
Pass
3
Actual Output
Pass
6
3
What I'd like to have is stop the execution of first executor as soon as there is a timeout. And continue with the next executor.

Finally, it is implemented using this link. The final code is shared below :-
import time
from concurrent.futures import ThreadPoolExecutor
import ctypes
def terminate_thread(thread):
"""Terminates a python thread from another thread.
:param thread: a threading.Thread instance
"""
if not thread.isAlive():
return
exc = ctypes.py_object(SystemExit)
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(
ctypes.c_long(thread.ident), exc)
if res == 0:
raise ValueError("nonexistent thread id")
elif res > 1:
# """if it returns a number greater than one, you're in trouble,
# and you should call it again with exc=NULL to revert the effect"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, None)
raise SystemError("PyThreadState_SetAsyncExc failed")
def test1(a, b, c):
time.sleep(5)
d=a+b+c
print(d)
def test2(a, b):
time.sleep(5)
d=a+b
print(d)
with ThreadPoolExecutor(max_workers=1) as executor:
try:
executor.submit(test1, 1,2,3).result(timeout=1)
except:
executor.shutdown(wait=False)
for t in executor._threads:
terminate_thread(t)
print("Pass")
with ThreadPoolExecutor(max_workers=1) as executor:
try:
executor.submit(test2, 1,2).result(timeout=8)
except:
executor.shutdown(wait=False)
for t in executor._threads:
terminate_thread(t)
print("Pass-2")

Can't pickle coroutine objects when ProcessPoolExecutor is used in class

I'm trying to get asyncio work with subprocesses and limitations. I've accomplish this in functional way, but when I tried to implement same logic in opp style several problems showd up. Mostly Can't pickle coroutine/generator errors. I tracked some of theese, but not all
import asyncio
from concurrent.futures import ProcessPoolExecutor
from itertools import islice
from random import randint
class async_runner(object):
def __init__(self):
self.futures = [] # container to store current futures
self.futures_total = []
self.loop = asyncio.get_event_loop() # main event_loop
self.executor = ProcessPoolExecutor()
self.limit = 1
def run(self, func, *args):
temp_loop = asyncio.new_event_loop()
try:
coro = func(*args)
asyncio.set_event_loop(temp_loop)
ret = temp_loop.run_until_complete(coro)
return ret
finally:
temp_loop.close()
def limit_futures(self, futures, limit):
self.futures_total = iter(futures)
self.futures = [future for future in islice(self.futures_total,0,limit)]
async def first_to_finish():
while True:
await asyncio.sleep(0)
for f in self.futures:
if f.done(): # here raised TypeError: can't pickle coroutine objects
print(f.done())
self.futures.remove(f)
try:
#newf = next(self.futures_total)
#self.futures.append(newf)
print(f.done())
except StopIteration as e:
pass
return f.result()
while len(self.futures) > 0:
yield first_to_finish()
async def run_limited(self, func, args, limit):
self.limit = int(limit)
self.futures_total = (self.loop.run_in_executor(self.executor, self.run, func, x) for x in range(110000,119990))
for ret in self.limit_futures(self.futures_total, 4): # limitation - 4 per all processes
await ret
def set_execution(self, func, args, limit):
ret = self.loop.run_until_complete(self.run_limited(func, args, limit))
return ret
async def asy(x):
print('enter: ', x)
await asyncio.sleep(randint(1,3))
print('finishing ', x)
return x
runner = async_runner()
ret = runner.set_execution(asy,urls,2)
print(ret)
But this works fine:
import asyncio
from concurrent.futures import ProcessPoolExecutor
from itertools import islice
import time
async def asy(x):
print('enter: ', x)
await asyncio.sleep(1)
print('finishing ', x)
return x
def run(corofn, *args):
loop = asyncio.new_event_loop()
try:
coro = corofn(*args)
asyncio.set_event_loop(loop)
ret = loop.run_until_complete(coro)
#print(ret)
return ret
finally:
loop.close()
def limit_futures(futures, limit):
futures_sl = [
c for c in islice(futures, 0, limit)
]
print(len(futures_sl))
async def first_to_finish(futures):
while True:
await asyncio.sleep(0)
for f in futures_sl:
if f.done():
futures_sl.remove(f)
try:
newf = next(futures)
futures_sl.append(newf)
except StopIteration as e:
pass
return f.result()
while len(futures_sl) > 0:
yield first_to_finish(futures)
async def main():
loop = asyncio.get_event_loop()
executor = ProcessPoolExecutor()
futures = (loop.run_in_executor(executor, run, asy, x) for x in range(110000,119990))
'''
CASE balls to the wall!
await asyncio.gather(*futures)
'''
for ret in limit_futures(futures, 4): # limitation - 4 per all processes
await ret
if __name__ == '__main__':
start = time.time()
'''
# CASE single
ret = [asy(x) for x in range(510000,510040)]
exit()
'''
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
print("Elapsed time: {:.3f} sec".format(time.time() - start))
I've cant understand why multiprocessing module trying to pickle anything only when objects are in use, but not in any scenario

The reason why multiprocessing needs to pickle the async_runner instance is because self.runner is a bound method, meaning that it "contains" the async_runner instance.
Since you're not actually using self in the run method, you can just make it a staticmethod to avoid this problem.

Timeout handling while using run_in_executor and asyncio

I'm using asyncio to run a piece of blocking code like this:
result = await loop.run_in_executor(None, long_running_function)
My question is: Can I impose a timeout for the execution of long_running_function?
Basically I don't want long_running_function to last more than 2 seconds and I can't do proper timeout handling within it because that function comes from a third-party library.

A warning about cancelling long running functions:
Although wrapping the Future returned by loop.run_in_executor with an asyncio.wait_for call will allow the event loop to stop waiting for long_running_function after some x seconds, it won't necessarily stop the underlying long_running_function. This is one of the shortcomings of concurrent.futures and to the best of my knowledge there is no simple way to just cancel a concurrent.futures.Future.

You could use asyncio.wait_for:
future = loop.run_in_executor(None, long_running_function)
result = await asyncio.wait_for(future, timeout, loop=loop)

although not using run_in_executor, i have some workaround about "wrap a block function async with timeout handling"
import asyncio
import threading
import time
import ctypes
def terminate_thread(t: threading.Thread, exc_type=SystemExit):
if not t.is_alive(): return
try:
tid = next(tid for tid, tobj in threading._active.items() if tobj is t)
except StopIteration:
raise ValueError("tid not found")
if ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exc_type)) != 1:
raise SystemError("PyThreadState_SetAsyncExc failed")
class AsyncResEvent(asyncio.Event):
def __init__(self):
super().__init__()
self.res = None
self.is_exc = False
self._loop = asyncio.get_event_loop()
def set(self, data=None) -> None:
self.res = data
self.is_exc = False
self._loop.call_soon_threadsafe(super().set)
def set_exception(self, exc) -> None:
self.res = exc
self.is_exc = True
self._loop.call_soon_threadsafe(super().set)
async def wait(self, timeout: float | None = None):
await asyncio.wait_for(super().wait(), timeout)
if self.is_exc:
raise self.res
else:
return self.res
async def sub_thread_async(func, *args, _timeout: float | None = None, **kwargs):
res = AsyncResEvent()
def f():
try:
res.set(func(*args, **kwargs))
except Exception as e:
res.set_exception(e)
except SystemExit:
res.set_exception(TimeoutError)
(t := threading.Thread(target=f)).start()
try:
return await res.wait(_timeout)
except TimeoutError:
raise TimeoutError
finally:
if not res.is_set():
terminate_thread(t)
_lock = threading.Lock()
def test(n):
_tid = threading.get_ident()
for i in range(n):
with _lock:
print(f'print from thread {_tid} ({i})')
time.sleep(1)
return n
async def main():
res_normal = await asyncio.gather(*(sub_thread_async(test, 5) for _ in range(2)))
print(res_normal) # [5,5]
res_normal_2 = await asyncio.gather(*(sub_thread_async(test, 2, _timeout=3) for _ in range(2)))
print(res_normal_2) # [2,2]
res_should_not_get = await asyncio.gather(*(sub_thread_async(test, 5, _timeout=3) for _ in range(2)))
print(res_should_not_get) # timeout error
if __name__ == '__main__':
asyncio.new_event_loop().run_until_complete(main())

Multiple threads with the same coroutine?

Can I run multiple threads running the same copies of a coroutine?
for example if I change the threaded function from this tutorial to
#coroutine
def threaded(count, target):
messages = Queue()
def run_target():
while True:
item = messages.get()
if item is GeneratorExit:
target.close()
return
else:
target.send(item)
for i in xrange(count):
Thread(target=run_target).start()
try:
while True:
item = (yield)
messages.put(item)
except GeneratorExit:
messages.put(GeneratorExit)
Does that really work? How do I verify whether it is working?

I think I got it fixed, I needed to change the function to something like this for it to work
#coroutine
def _threaded(self, count, target_func):
"""
Given a target coroutine, spawn $count threads to run copies of them. In
order to properly use this, do not call the coroutine before calling this,
e.g.
#coroutine
def foo(self):
...
def bar(self):
...
self._threaded(10, self.foo) # <- do not call self.foo,
# just the reference
#param count The number of threads to spawn
#param target_func The reference to the target coroutine
#returns The subnet mask
"""
result = None
messages = Queue()
def default_target_run(index):
target = target_func()
while True:
item = messages.get()
if item is GeneratorExit:
target.close()
return
else:
target.send({'index': index, 'item': item})
# ensure code is testable
target_run = default_target_run
try:
target_run = self._threaded.target_run
except AttributeError:
pass
result = ThreadPool(count).map_async(target_run, range(count))
try:
while True:
item = (yield)
messages.put(item)
except GeneratorExit:
# allow all threads to quit
# by making sure all of them receives the exit message
for i in xrange(count):
messages.put(GeneratorExit)
result.ready()