I constantly catch a warning when trying to close my Python program, that uses asyncio and multiprocessing (by pathos), via Ctrl + c.
There's a class that's responsible for sending messages via Telegram bot
class MyAgent:
def __init__(self, test_mode: bool = False):
token = settings.env.str('BOT_TOKEN')
self.updater = Updater(token=token, use_context=True)
self.bot = Bot(token=token)
self.chat_id = settings.CHAT_ID
self.loop = asyncio.get_event_loop()
self.messagers = []
self.fetching_tasks = []
self.test_mode = test_mode
def register_messagers(self, messagers: list):
for messager in messagers:
msgr = messager(self)
self.messagers.append(msgr)
# Need to keep fetching task instances alive in the list
self.fetching_tasks.append(msgr.create_task())
if __name__ == '__main__':
import argparse
from pathos import multiprocessing as mp
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--test", action='store_true')
ns = parser.parse_args()
agent = MyAgent(test_mode=ns.test)
agent.register_messagers([Messager1, Messager2])
# Start up TG polling in a separate process
p = mp.ProcessPool(nodes=1)
# start_polling source code https://github.com/python-telegram-bot/python-telegram-bot/blob/master/telegram/ext/updater.py#L221
p.pipe(agent.updater.start_polling)
try:
tasks = asyncio.gather(*agent.fetching_tasks)
agent.loop.run_until_complete(tasks)
except KeyboardInterrupt:
agent.loop.close()
# Shut down TG polling
p.close()
p.terminate()
p.join()
finally:
print("Bye!")
I have a couple of coroutines that run infinitely in an event loop:
class Messager1:
def __init__(self, agent):
self.agent = agent
def create_task(self):
return self.agent.loop.create_task(self._run_forever())
async def _run_forever(self):
while True:
await self.run()
await asyncio.sleep(0.1)
async def run(self):
current_time = datetime.now()
msgs = fetch_messages_from_db(current_time)
for _ in range(len(msgs)):
message = "This is coroutine 1"
if self.agent.test_mode:
print(message)
else:
self.agent.bot.send_message(chat_id=self.agent.chat_id, text=message)
await asyncio.sleep(0.001)
class Messager2:
def __init__(self, agent):
self.agent = agent
def create_task(self):
return self.agent.loop.create_task(self._run_forever())
async def _run_forever(self):
while True:
await self.run()
await asyncio.sleep(0.1)
async def run(self):
current_time = datetime.now()
msgs = fetch_messages_from_db(current_time)
for _ in range(len(msgs)):
message = "This is coroutine 1"
if self.agent.test_mode:
print(message)
else:
self.agent.bot.send_message(chat_id=self.agent.chat_id, text=message)
await asyncio.sleep(0.001)
Ignore boilerplate code for now please.
My problem is:
The code sometimes runs fine when Messagers print to console and with the asyncio sleeps I've set up: asyncio.sleep(0.1) in _run_forever and asyncio.sleep(0.001) in run. But if I try to reduce asyncio.sleep(0.1) to asyncio.sleep(0.01), then I on CTRL+C shutdown I always get a warning:
Traceback (most recent call last):
File ".venv/lib/python3.8/site-packages/multiprocess/process.py", line 313, in _bootstrap
self.run()
File ".venv/lib/python3.8/site-packages/multiprocess/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File ".venv/lib/python3.8/site-packages/multiprocess/pool.py", line 114, in worker
task = get()
File ".venv/lib/python3.8/site-packages/multiprocess/queues.py", line 359, in get
res = self._reader.recv_bytes()
File ".venv/lib/python3.8/site-packages/multiprocess/connection.py", line 219, in recv_bytes
buf = self._recv_bytes(maxlength)
File ".venv/lib/python3.8/site-packages/multiprocess/connection.py", line 382, in _recv
chunk = read(handle, remaining)
I also get this warning when not running in test mode and sending live messages via Telegram, while sleeps are unchanged. This problem seems to correlate with sleep times, amount of data fetched from DB and whether I am just printing messages to console or sending them to Telegram bot. I don't know the exact relationship, I'm assuming someone a lot more familiar with multiprocessing/asyncio in Python might point out the most likely reasons based on the fact of existence of correlation itself... I'll record the observations if necessary
I've done some googling and haven't found any particular answer. There are some discussion threads on bugs.python.org blaming a bug in GC, but they all seem to be resolved in <3.8 versions.
What might be causing this warning?
P.S. I'm aware that fetching data from disk in a coroutine is bad and won't scale, I'm planning to fix it, I just have other things to deal with currently. But comments on other design flaws are welcome.
Related
Within my python code, I am trying to design a piece of client code that connects to a WebSockets Server every second and then prints the timestamp and the obtained value from the server in a .csv file. This is given below:
import asyncio
import websockets
import logging
import datetime
import time
starttime = time.time() # start value for timed data acquisition
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
class Timer: # class for asynchronous (non-blocking) counter
def __init__(self, interval, first_immediately, callback):
self._interval = interval
self._first_immediately = first_immediately
self._callback = callback
self._is_first_call = True
self._ok = True
self._task = asyncio.ensure_future(self._job())
print("init timer done")
async def _job(self):
try:
while self._ok:
if not self._is_first_call or not self._first_immediately:
await asyncio.sleep(self._interval)
await self._callback(self)
self._is_first_call = False
except Exception as ex:
print(ex)
def cancel(self):
self._ok = False
self._task.cancel()
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/", ping_interval=None
) as websocket:
await websocket.send(
str(1.001)
) # send a message to the websocket server
response = (
await websocket.recv()
) # wait to get a response from the server
print(response)
dataline_pv1 = (
datetime.datetime.today().isoformat()
+ ","
+ str(response)
+ ","
+ str(0)
+ "\n"
) # format and assemble data line
file_name_pv1 = (
"{:%Y%m%d}".format(datetime.datetime.today()) + "_flow.csv"
) # generate file name
with open(
file_name_pv1, "a"
) as etherm_file1: # append dataline to file
etherm_file1.write(dataline_pv1)
# asyncio.get_event_loop().run_forever(test()) # run until test() is finished while True:
timer = Timer(interval=1, first_immediately=True, callback=test)
loop = asyncio.get_event_loop()
try:
asyncio.ensure_future(test())
loop.run_forever()
except KeyboardInterrupt:
timer.cancel()
pass
finally:
print("Closing Loop")
loop.close()
When this runs, I obtain the following message of my terminal (however the code does not crash):
test() takes 0 positional arguments but 1 was given
I have seen from this question (TypeError: takes 0 positional arguments but 1 was given) that this error occurs when a Class object is not defined properly, but my error seems to be occurring outside of a class framework. In addition, the desired .csv file is produced, however only one line is printed to the file, and does not repeat every second as desired.
What am I missing here? (also I am a complete novice with asyncio programming)
UPDATE: After changing the definition of test() to async def test(timer=None), my code now runs as expected and outputs the values to a .csv file every second (roughly), but still throws up an error. Specifically:
Task exception was never retrieved
future: <Task finished coro=<test() done, defined at flowmeterclient_v2.py:36> exception=ConnectionRefusedError(111, "Connect call failed ('198.162.1.177', 80)")>
Traceback (most recent call last):
File "flowmeterclient_v2.py", line 37, in test
async with websockets.connect("ws://198.162.1.177:80/", ping_interval=None) as websocket:
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 604, in __aenter__
return await self
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 622, in __await_impl__
transport, protocol = await self._create_connection()
File "/usr/lib64/python3.6/asyncio/base_events.py", line 798, in create_connection
raise exceptions[0]
File "/usr/lib64/python3.6/asyncio/base_events.py", line 785, in create_connection
yield from self.sock_connect(sock, address)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 439, in sock_connect
return (yield from fut)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 469, in _sock_connect_cb
raise OSError(err, 'Connect call failed %s' % (address,))
ConnectionRefusedError: [Errno 111] Connect call failed ('198.162.1.177', 80)
Your Timer code passes the timer itself as the first argument in
await self._callback(self)
You may wish to change the signature of test to
async def test(timer=None):
so you can call it with or without the timer.
I think you don't really need the Timer here at all. Simply have an asyncio task that loops forever and has an asyncio.sleep() internally.
This also doesn't reconnect to the websocket server for each request, like your previous code did.
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
while True:
await websocket.send(str(1.001))
response = await websocket.recv()
print(response)
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
await asyncio.sleep(1)
asyncio.run(test())
Following up on comments, if you actually do need to reconnect for each request, you can refactor this like so:
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(
logging.INFO
) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def get_data():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
await websocket.send(str(1.001))
response = await websocket.recv()
return response
def save_data(response):
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
async def test():
while True:
response = await get_data()
save_data(response)
await asyncio.sleep(1)
asyncio.run(test())
Does asyncio work with os.fork()?
Code Snippet 1:
import asyncio
import os
import aiohttp
async def main():
url = "https://google.com"
pid = os.fork()
if pid == 0:
# child process
print("in child process")
await fetch(url)
print("in child process done")
else:
print("in parent process")
await asyncio.sleep(20)
print("in parent process done")
async def fetch(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
return await response.text()
if __name__ == "__main__":
asyncio.run(main())
Code above works fine.
Code Snippet 2:
import asyncio
import os
import aiohttp
async def main():
url = "https://google.com"
pid = os.fork()
if pid == 0:
# child process
print("in child process")
await asyncio.sleep(10) # different with code snippet 1
# await fetch(url)
print("in child process done")
else:
print("in parent process")
await asyncio.sleep(20)
print("in parent process done")
async def fetch(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
return await response.text()
if __name__ == "__main__":
asyncio.run(main())
Code above will raise following exception:
Traceback (most recent call last):
File "fork_sleep.py", line 28, in <module>
asyncio.run(main())
File "/usr/lib/python3.8/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "fork_sleep.py", line 13, in main
await asyncio.sleep(10) # different with code snippet 1
File "/usr/lib/python3.8/asyncio/tasks.py", line 637, in sleep
loop = events.get_running_loop()
RuntimeError: no running event loop
The reason for the "no running event loop" exception is that function get_running_loop compare the os.getpid() and the pid saved in loop. When they are different, the exception above is raised.
Please refer to the following code from cpython source code.
def get_running_loop():
"""Return the running event loop. Raise a RuntimeError if there is none.
This function is thread-specific.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
loop = _get_running_loop()
if loop is None:
raise RuntimeError('no running event loop')
return loop
def _get_running_loop():
"""Return the running event loop or None.
This is a low-level function intended to be used by event loops.
This function is thread-specific.
"""
# NOTE: this function is implemented in C (see _asynciomodule.c)
running_loop, pid = _running_loop.loop_pid
if running_loop is not None and pid == os.getpid():
return running_loop
So it seems that asyncio event loop works fine in child process if you don't touch the function get_running_loop.
My question is, what is the by-design behavior? Why the author check the pid in function _get_running_loop? And what is the solution if you encounter the "no running event loop" in child process.
I have a Python Tornado Websocket server that stores clients in a shared set() so that I know how many clients are connected.
The challenge is that calling on_close after WebSocketClosedError raises a KeyError and the client-instance is not removed from the set of connected clients. This error has caused my server to accumulate over 1000 clients even when the active clients are only around 5.
My Code:
import tornado.iostream
import tornado.websocket
import asyncio
class SocketHandler(tornado.websocket.WebSocketHandler):
socket_active_message = {"status": "Socket Connection Active"}
waiters = set()
def initialize(self):
self.client_name = "newly_connected"
def open(self):
print('connection opened')
# https://kite.com/python/docs/tornado.websocket.WebSocketHandler.set_nodelay
self.set_nodelay(True)
SocketHandler.waiters.add(self)
def on_close(self):
print("CLOSED!", self.client_name)
SocketHandler.waiters.remove(self)
def check_origin(self, origin):
# Override the origin check if needed
return True
async def send_updates(self, message):
print('starting socket service loop')
loop_counter = 0
while True:
try:
await self.write_message({'status': 82317581})
except tornado.websocket.WebSocketClosedError:
self.on_close()
except tornado.iostream.StreamClosedError:
self.on_close()
except Exception as e:
self.on_close()
print('Exception e:', self.client_name)
await asyncio.sleep(0.05)
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message)
def run_server():
# Create tornado application and supply URL routes
webApp = tornado.web.Application(
[
(
r"/",
SocketHandler,
{},
),
]
)
application = tornado.httpserver.HTTPServer(webApp)
webApp.listen(3433)
# Start IO/Event loop
tornado.ioloop.IOLoop.instance().start()
run_server()
The Stack-trace:
Traceback (most recent call last):
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/web.py", line 1699, in _execute
result = await result
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 278, in get
await self.ws_connection.accept_connection(self)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 881, in accept_connection
await self._accept_connection(handler)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 964, in _accept_connection
await self._receive_frame_loop()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1118, in _receive_frame_loop
await self._receive_frame()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1209, in _receive_frame
await handled_future
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 658, in <lambda>
self.stream.io_loop.add_future(result, lambda f: f.result())
File "ask_So.py", line 50, in on_message
await self.send_updates(message)
File "ask_So.py", line 39, in send_updates
self.on_close()
File "ask_So.py", line 26, in on_close
SocketHandler.waiters.remove(self)
KeyError: <__main__.SocketHandler object at 0x7ffef9f25520>
I have tried moving the waiters set outside the class but it still produces the same behaviour.
To simulate WebSocketClosedError: open many browser tabs as clients and close one browser tab at a time.
It seems like self.on_close() is being called twice. Once you're calling it manually from inside send_updates() and then later, when a connection is actually closed, Tornado is also calling self.on_close(). Since the self object was already removed from the set the first time, it raises a KeyError the second time.
If you want to close the connection, just call self.close(). The self.on_close() method will be called by Tornado automatically.
Also, you can handle the exception in a try...except block inside on_close.
Update
The previous part of this answer should fix the KeyError related problem. This update is regarding why the clients are not being removed from waiters set.
So, I tested your code and found a major problem with it here:
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message) # <- This is problematic
Whenever a client sends a message, it will run self.send_updates method. So even if there's only one client that sends a message, let's say, 10 times, send_updates will also be called 10 times and, as a result, you will have 10 while loops running simultaneously!
As the number of loops increase, it ultimately blocks the server. That means Tornado has no time to run other code as it's busy juggling so many while loops. Hence, the clients from the waiters are never removed.
Solution
Instead of calling send_updates everytime a message arrives, you can call it just one time. Just have a single while loop to send updates to all clients.
I'd update the code like this:
class SocketHandler(...):
# Make it a classmethod so that it can be
# called without an instance
#classmethod
async def send_updates(cls):
print('starting socket service loop')
loop_counter = 0
while True:
for waiter in cls.waiters:
# use `waiter` instead of `self`
try:
await waiter.write_message({'status': 82317581})
...
await asyncio.sleep(0.05)
Instead of calling send_updates from on_message, you'll have to tel IOLoop to call it once:
def run_server():
...
# schedule SocketHandler.send_updates to be run
tornado.ioloop.IOLoop.current().add_callback(SocketHandler.send_updates)
tornado.ioloop.IOLoop.current().start()
This will have only one while loop running for all clients.
i'm building a kind of simulator that uses thrift protocol.
But when im executing multiple threads of my virtual equipments sending messages, the program breaks after a short time by receiving them, think the buffer is overloaded or something like that, or not, so i'm here asking for some help if its possible.
Here's the main pieces of my code
A class for threading:
class ThreadManager (threading.Thread):
def __init__(self, name, obj, client, layout):
threading.Thread.__init__(self)
self.name = name
self.obj = obj
self.client = client
self.layout = layout
def run(self):
print ("Starting " + self.name)
while True:
sleep(2)
self.obj.auto_gen_msg(self.client, layout=self.layout)
The method for generating messages:
def auto_gen_msg(self, client, layout='', min_delay=15, max_delay=30):
if not layout:
msg = self.gen_message(self.draw_random_model())
else:
msg = self.gen_message(layout)
wait = randint(min_delay, max_delay)
sleep(wait)
print(self.eqp_type, " delivered a message ...")
getattr(client, msg[0])(*msg[1])
The main:
def start(layout, equipment, number):
try:
host = 'localhost'
transport = TSocket.TSocket(host, port=9090)
transport = TTransport.TBufferedTransport(transport)
protocol = TCompactProtocol.TCompactProtocol(transport)
client = SuiteService.Client(protocol)
transport.open()
equips = [Equipment(equipment) for i in range(number)]
threads = [ThreadManager(i.eqp_type, i, client, layout) for i in equips]
for i in range(len(threads)):
threads[i].start()
sleep(2)
while True:
pass
transport.close()
except Thrift.TException as tx:
print ("%s " % (tx.message))
The error haunting me:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/Users/lem4fia/Documents/sg/loki/loki-thrift/loki_thrift/loki_thrift/lib/thread_manager.py", line 39, in run
self.obj.auto_gen_msg(self.client, layout=self.layout)
File "/Users/lem4fia/Documents/sg/loki/loki-thrift/loki_thrift/loki_thrift/lib/virtual.py", line 281, in auto_gen_msg
getattr(client, msg[0])(*msg[1])
File "/Users/lem4fia/Documents/sg/loki/thrift-server/thrift_server/suite/SuiteService.py", line 4895, in v1
self.send_v1(ir, ts, ch, o1, o2, o3, o4, o5, o6, o7)
File "/Users/lem4fia/Documents/sg/loki/thrift-server/thrift_server/suite/SuiteService.py", line 4899, in send_v1
self._oprot.writeMessageBegin('v1', TMessageType.CALL, self._seqid)
File "/Users/lem4fia/Documents/sg/loki/lokiv/lib/python3.6/site-packages/thrift-0.11.0-py3.6-macosx-10.6-intel.egg/thrift/protocol/TCompactProtocol.py", line 156, in writeMessageBegin
assert self.state == CLEAR
AssertionError
Curiously, it doesnt bug if instancing 2 virtual equipments in thread, but 10 virtual equipments (sometimes less than this) is sufficient to raise this error.
Can someone please gimme a light? :)
The problem there is that it seems that you have to use one diferent Transport object for each thread. This is probably related to Thrift's implementation!
Reference here : http://grokbase.com/t/thrift/user/134s16ks4m/single-connection-and-multiple-threads
As a general rule 1), Thrift is not intended to be used across multiple threads.
This is, at least to my knowledge, true for all currently supported languages.
One instance per thread will do.
1) aside from server-end things like TThreadedServer or TThreadPoolServer
I'm trying to code a simple program based on Asyncio and a Publish/Subscribe design pattern implemented with ZeroMQ. The publisher has 2 coroutines; one that listens for incoming subscriptions, and another one that publishes the value (obtained via an HTTP request) to the subscriber. The subscriber subscribes to a specific parameter (the name of a city in this case), and waits for the value (the temperature in this city).
Here is my code:
publisher.py
#!/usr/bin/env python
import json
import aiohttp
import aiozmq
import asyncio
import zmq
class Publisher:
BIND_ADDRESS = 'tcp://*:10000'
def __init__(self):
self.stream = None
self.parameter = ""
#asyncio.coroutine
def main(self):
self.stream = yield from aiozmq.create_zmq_stream(zmq.XPUB, bind=Publisher.BIND_ADDRESS)
tasks = [
asyncio.async(self.subscriptions()),
asyncio.async(self.publish())]
print("before wait")
yield from asyncio.wait(tasks)
print("after wait")
#asyncio.coroutine
def subscriptions(self):
print("Entered subscriptions coroutine")
while True:
print("New iteration of subscriptions loop")
received = yield from self.stream.read()
first_byte = received[0][0]
self.parameter = received[0][-len(received[0])+1:].decode("utf-8")
# Subscribe request
if first_byte == 1:
print("subscription request received for parameter "+self.parameter)
# Unsubscribe request
elif first_byte == 0:
print("Unsubscription request received for parameter "+self.parameter)
#asyncio.coroutine
def publish(self):
print("Entered publish coroutine")
while True:
if self.parameter:
print("New iteration of publish loop")
# Make HTTP request
url = "http://api.openweathermap.org/data/2.5/weather?q="+self.parameter
response = yield from aiohttp.request('GET', url)
assert response.status == 200
content = yield from response.read()
# Decode JSON string
decoded_json = json.loads(content.decode())
# Get parameter value
value = decoded_json["main"]["temp"]
# Publish fetched values to subscribers
message = bytearray(self.parameter+":"+str(value),"utf-8")
print(message)
pack = [message]
print("before write")
yield from self.stream.write(pack)
print("after write")
yield from asyncio.sleep(10)
test = Publisher()
loop = asyncio.get_event_loop()
loop.run_until_complete(test.main())
subscriber.py
#!/usr/bin/env python
import zmq
class Subscriber:
XSUB_CONNECT = 'tcp://localhost:10000'
def __init__(self):
self.context = zmq.Context()
self.socket = self.context.socket(zmq.XSUB)
self.socket.connect(Subscriber.XSUB_CONNECT)
def loop(self):
print(self.socket.recv())
self.socket.close()
def subscribe(self, parameter):
self.socket.send_string('\x01'+parameter)
print("Subscribed to parameter "+parameter)
def unsubscribe(self, parameter):
self.socket.send_string('\x00'+parameter)
print("Unsubscribed to parameter "+parameter)
test = Subscriber()
test.subscribe("London")
while True:
print(test.socket.recv())
And here is the output :
Subscriber side :
$ python3 subscriber.py
Subscribed to parameter London
b'London:288.15'
Publisher side :
$ python3 publisher.py
before wait
Entered subscriptions coroutine
New iteration of subscriptions loop
Entered publish coroutine
subscription request received for parameter London
New iteration of subscriptions loop
New iteration of publish loop
bytearray(b'London:288.15')
before write
And the program is stuck there.
As you can see, "before write" appears in the output and the message is sent, but "after write" doesn't appear. So, I figured that an exception was probably raised and caught somewhere in the self.stream.write(pack) call stack.
If I send a KeyboardInterrupt to the Publisher, here is what I get:
Traceback (most recent call last):
File "publisher.py", line 73, in <module>
loop.run_until_complete(test.main())
File "/usr/lib/python3.4/asyncio/base_events.py", line 304, in run_until_complete
self.run_forever()
File "/usr/lib/python3.4/asyncio/base_events.py", line 276, in run_forever
self._run_once()
File "/usr/lib/python3.4/asyncio/base_events.py", line 1136, in _run_once
event_list = self._selector.select(timeout)
File "/usr/lib/python3.4/selectors.py", line 432, in select
fd_event_list = self._epoll.poll(timeout, max_ev)
KeyboardInterrupt
Task exception was never retrieved
future: <Task finished coro=<publish() done, defined at publisher.py:43> exception=TypeError("'NoneType' object is not iterable",)>
Traceback (most recent call last):
File "/usr/lib/python3.4/asyncio/tasks.py", line 236, in _step
result = coro.send(value)
File "publisher.py", line 66, in publish
yield from self.stream.write(pack)
TypeError: 'NoneType' object is not iterable
Task was destroyed but it is pending!
task: <Task pending coro=<subscriptions() running at publisher.py:32> wait_for=<Future pending cb=[Task._wakeup()]> cb=[_wait.<locals>._on_completion() at /usr/lib/python3.4/asyncio/tasks.py:399]>
So I guess my problem actually is this error: TypeError: 'NoneType' object is not iterable, but I have no clue what's causing it.
What is going wrong here?
The issue is that you're trying to yield from the call to self.stream.write(), but stream.write isn't actually a coroutine. When you call yield from on an item, Python internally calls iter(item). In this case, the call to write() is returning None, so Python is trying to do iter(None) - hence the exception you see.
To fix it, you should just call write() like a normal function. If you want to actually wait until the write is flushed and sent to the reader, use yield from stream.drain() after you make the call to write():
print("before write")
self.stream.write(pack)
yield from self.stream.drain()
print("after write")
Also, to make sure that exception in publish get raised without needing to Ctrl+C, use asyncio.gather instead of asyncio.wait:
yield from asyncio.gather(*tasks)
With asyncio.gather, any exception thrown by a task inside tasks will be re-raised.