Develop Reference

'task() takes 0 positional arguments but 1 was given' for python asyncio function

Within my python code, I am trying to design a piece of client code that connects to a WebSockets Server every second and then prints the timestamp and the obtained value from the server in a .csv file. This is given below:
import asyncio
import websockets
import logging
import datetime
import time
starttime = time.time() # start value for timed data acquisition
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
class Timer: # class for asynchronous (non-blocking) counter
def __init__(self, interval, first_immediately, callback):
self._interval = interval
self._first_immediately = first_immediately
self._callback = callback
self._is_first_call = True
self._ok = True
self._task = asyncio.ensure_future(self._job())
print("init timer done")
async def _job(self):
try:
while self._ok:
if not self._is_first_call or not self._first_immediately:
await asyncio.sleep(self._interval)
await self._callback(self)
self._is_first_call = False
except Exception as ex:
print(ex)
def cancel(self):
self._ok = False
self._task.cancel()
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/", ping_interval=None
) as websocket:
await websocket.send(
str(1.001)
) # send a message to the websocket server
response = (
await websocket.recv()
) # wait to get a response from the server
print(response)
dataline_pv1 = (
datetime.datetime.today().isoformat()
+ ","
+ str(response)
+ ","
+ str(0)
+ "\n"
) # format and assemble data line
file_name_pv1 = (
"{:%Y%m%d}".format(datetime.datetime.today()) + "_flow.csv"
) # generate file name
with open(
file_name_pv1, "a"
) as etherm_file1: # append dataline to file
etherm_file1.write(dataline_pv1)
# asyncio.get_event_loop().run_forever(test()) # run until test() is finished while True:
timer = Timer(interval=1, first_immediately=True, callback=test)
loop = asyncio.get_event_loop()
try:
asyncio.ensure_future(test())
loop.run_forever()
except KeyboardInterrupt:
timer.cancel()
pass
finally:
print("Closing Loop")
loop.close()
When this runs, I obtain the following message of my terminal (however the code does not crash):
test() takes 0 positional arguments but 1 was given
I have seen from this question (TypeError: takes 0 positional arguments but 1 was given) that this error occurs when a Class object is not defined properly, but my error seems to be occurring outside of a class framework. In addition, the desired .csv file is produced, however only one line is printed to the file, and does not repeat every second as desired.
What am I missing here? (also I am a complete novice with asyncio programming)
UPDATE: After changing the definition of test() to async def test(timer=None), my code now runs as expected and outputs the values to a .csv file every second (roughly), but still throws up an error. Specifically:
Task exception was never retrieved
future: <Task finished coro=<test() done, defined at flowmeterclient_v2.py:36> exception=ConnectionRefusedError(111, "Connect call failed ('198.162.1.177', 80)")>
Traceback (most recent call last):
File "flowmeterclient_v2.py", line 37, in test
async with websockets.connect("ws://198.162.1.177:80/", ping_interval=None) as websocket:
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 604, in __aenter__
return await self
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 622, in __await_impl__
transport, protocol = await self._create_connection()
File "/usr/lib64/python3.6/asyncio/base_events.py", line 798, in create_connection
raise exceptions[0]
File "/usr/lib64/python3.6/asyncio/base_events.py", line 785, in create_connection
yield from self.sock_connect(sock, address)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 439, in sock_connect
return (yield from fut)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 469, in _sock_connect_cb
raise OSError(err, 'Connect call failed %s' % (address,))
ConnectionRefusedError: [Errno 111] Connect call failed ('198.162.1.177', 80)

Your Timer code passes the timer itself as the first argument in
await self._callback(self)
You may wish to change the signature of test to
async def test(timer=None):
so you can call it with or without the timer.

I think you don't really need the Timer here at all. Simply have an asyncio task that loops forever and has an asyncio.sleep() internally.
This also doesn't reconnect to the websocket server for each request, like your previous code did.
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
while True:
await websocket.send(str(1.001))
response = await websocket.recv()
print(response)
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
await asyncio.sleep(1)
asyncio.run(test())
Following up on comments, if you actually do need to reconnect for each request, you can refactor this like so:
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(
logging.INFO
) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def get_data():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
await websocket.send(str(1.001))
response = await websocket.recv()
return response
def save_data(response):
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
async def test():
while True:
response = await get_data()
save_data(response)
await asyncio.sleep(1)
asyncio.run(test())

Python Tornado KeyError When removing client from clients set

I have a Python Tornado Websocket server that stores clients in a shared set() so that I know how many clients are connected.
The challenge is that calling on_close after WebSocketClosedError raises a KeyError and the client-instance is not removed from the set of connected clients. This error has caused my server to accumulate over 1000 clients even when the active clients are only around 5.
My Code:
import tornado.iostream
import tornado.websocket
import asyncio
class SocketHandler(tornado.websocket.WebSocketHandler):
socket_active_message = {"status": "Socket Connection Active"}
waiters = set()
def initialize(self):
self.client_name = "newly_connected"
def open(self):
print('connection opened')
# https://kite.com/python/docs/tornado.websocket.WebSocketHandler.set_nodelay
self.set_nodelay(True)
SocketHandler.waiters.add(self)
def on_close(self):
print("CLOSED!", self.client_name)
SocketHandler.waiters.remove(self)
def check_origin(self, origin):
# Override the origin check if needed
return True
async def send_updates(self, message):
print('starting socket service loop')
loop_counter = 0
while True:
try:
await self.write_message({'status': 82317581})
except tornado.websocket.WebSocketClosedError:
self.on_close()
except tornado.iostream.StreamClosedError:
self.on_close()
except Exception as e:
self.on_close()
print('Exception e:', self.client_name)
await asyncio.sleep(0.05)
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message)
def run_server():
# Create tornado application and supply URL routes
webApp = tornado.web.Application(
[
(
r"/",
SocketHandler,
{},
),
]
)
application = tornado.httpserver.HTTPServer(webApp)
webApp.listen(3433)
# Start IO/Event loop
tornado.ioloop.IOLoop.instance().start()
run_server()
The Stack-trace:
Traceback (most recent call last):
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/web.py", line 1699, in _execute
result = await result
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 278, in get
await self.ws_connection.accept_connection(self)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 881, in accept_connection
await self._accept_connection(handler)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 964, in _accept_connection
await self._receive_frame_loop()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1118, in _receive_frame_loop
await self._receive_frame()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1209, in _receive_frame
await handled_future
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 658, in <lambda>
self.stream.io_loop.add_future(result, lambda f: f.result())
File "ask_So.py", line 50, in on_message
await self.send_updates(message)
File "ask_So.py", line 39, in send_updates
self.on_close()
File "ask_So.py", line 26, in on_close
SocketHandler.waiters.remove(self)
KeyError: <__main__.SocketHandler object at 0x7ffef9f25520>
I have tried moving the waiters set outside the class but it still produces the same behaviour.
To simulate WebSocketClosedError: open many browser tabs as clients and close one browser tab at a time.

It seems like self.on_close() is being called twice. Once you're calling it manually from inside send_updates() and then later, when a connection is actually closed, Tornado is also calling self.on_close(). Since the self object was already removed from the set the first time, it raises a KeyError the second time.
If you want to close the connection, just call self.close(). The self.on_close() method will be called by Tornado automatically.
Also, you can handle the exception in a try...except block inside on_close.
Update
The previous part of this answer should fix the KeyError related problem. This update is regarding why the clients are not being removed from waiters set.
So, I tested your code and found a major problem with it here:
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message) # <- This is problematic
Whenever a client sends a message, it will run self.send_updates method. So even if there's only one client that sends a message, let's say, 10 times, send_updates will also be called 10 times and, as a result, you will have 10 while loops running simultaneously!
As the number of loops increase, it ultimately blocks the server. That means Tornado has no time to run other code as it's busy juggling so many while loops. Hence, the clients from the waiters are never removed.
Solution
Instead of calling send_updates everytime a message arrives, you can call it just one time. Just have a single while loop to send updates to all clients.
I'd update the code like this:
class SocketHandler(...):
# Make it a classmethod so that it can be
# called without an instance
#classmethod
async def send_updates(cls):
print('starting socket service loop')
loop_counter = 0
while True:
for waiter in cls.waiters:
# use `waiter` instead of `self`
try:
await waiter.write_message({'status': 82317581})
...
await asyncio.sleep(0.05)
Instead of calling send_updates from on_message, you'll have to tel IOLoop to call it once:
def run_server():
...
# schedule SocketHandler.send_updates to be run
tornado.ioloop.IOLoop.current().add_callback(SocketHandler.send_updates)
tornado.ioloop.IOLoop.current().start()
This will have only one while loop running for all clients.

Multiprocessing program exits on with warning on KeyboardInterrupt (chunk = read(handle, remaining))

I constantly catch a warning when trying to close my Python program, that uses asyncio and multiprocessing (by pathos), via Ctrl + c.
There's a class that's responsible for sending messages via Telegram bot
class MyAgent:
def __init__(self, test_mode: bool = False):
token = settings.env.str('BOT_TOKEN')
self.updater = Updater(token=token, use_context=True)
self.bot = Bot(token=token)
self.chat_id = settings.CHAT_ID
self.loop = asyncio.get_event_loop()
self.messagers = []
self.fetching_tasks = []
self.test_mode = test_mode
def register_messagers(self, messagers: list):
for messager in messagers:
msgr = messager(self)
self.messagers.append(msgr)
# Need to keep fetching task instances alive in the list
self.fetching_tasks.append(msgr.create_task())
if __name__ == '__main__':
import argparse
from pathos import multiprocessing as mp
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--test", action='store_true')
ns = parser.parse_args()
agent = MyAgent(test_mode=ns.test)
agent.register_messagers([Messager1, Messager2])
# Start up TG polling in a separate process
p = mp.ProcessPool(nodes=1)
# start_polling source code https://github.com/python-telegram-bot/python-telegram-bot/blob/master/telegram/ext/updater.py#L221
p.pipe(agent.updater.start_polling)
try:
tasks = asyncio.gather(*agent.fetching_tasks)
agent.loop.run_until_complete(tasks)
except KeyboardInterrupt:
agent.loop.close()
# Shut down TG polling
p.close()
p.terminate()
p.join()
finally:
print("Bye!")
I have a couple of coroutines that run infinitely in an event loop:
class Messager1:
def __init__(self, agent):
self.agent = agent
def create_task(self):
return self.agent.loop.create_task(self._run_forever())
async def _run_forever(self):
while True:
await self.run()
await asyncio.sleep(0.1)
async def run(self):
current_time = datetime.now()
msgs = fetch_messages_from_db(current_time)
for _ in range(len(msgs)):
message = "This is coroutine 1"
if self.agent.test_mode:
print(message)
else:
self.agent.bot.send_message(chat_id=self.agent.chat_id, text=message)
await asyncio.sleep(0.001)
class Messager2:
def __init__(self, agent):
self.agent = agent
def create_task(self):
return self.agent.loop.create_task(self._run_forever())
async def _run_forever(self):
while True:
await self.run()
await asyncio.sleep(0.1)
async def run(self):
current_time = datetime.now()
msgs = fetch_messages_from_db(current_time)
for _ in range(len(msgs)):
message = "This is coroutine 1"
if self.agent.test_mode:
print(message)
else:
self.agent.bot.send_message(chat_id=self.agent.chat_id, text=message)
await asyncio.sleep(0.001)
Ignore boilerplate code for now please.
My problem is:
The code sometimes runs fine when Messagers print to console and with the asyncio sleeps I've set up: asyncio.sleep(0.1) in _run_forever and asyncio.sleep(0.001) in run. But if I try to reduce asyncio.sleep(0.1) to asyncio.sleep(0.01), then I on CTRL+C shutdown I always get a warning:
Traceback (most recent call last):
File ".venv/lib/python3.8/site-packages/multiprocess/process.py", line 313, in _bootstrap
self.run()
File ".venv/lib/python3.8/site-packages/multiprocess/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File ".venv/lib/python3.8/site-packages/multiprocess/pool.py", line 114, in worker
task = get()
File ".venv/lib/python3.8/site-packages/multiprocess/queues.py", line 359, in get
res = self._reader.recv_bytes()
File ".venv/lib/python3.8/site-packages/multiprocess/connection.py", line 219, in recv_bytes
buf = self._recv_bytes(maxlength)
File ".venv/lib/python3.8/site-packages/multiprocess/connection.py", line 382, in _recv
chunk = read(handle, remaining)
I also get this warning when not running in test mode and sending live messages via Telegram, while sleeps are unchanged. This problem seems to correlate with sleep times, amount of data fetched from DB and whether I am just printing messages to console or sending them to Telegram bot. I don't know the exact relationship, I'm assuming someone a lot more familiar with multiprocessing/asyncio in Python might point out the most likely reasons based on the fact of existence of correlation itself... I'll record the observations if necessary
I've done some googling and haven't found any particular answer. There are some discussion threads on bugs.python.org blaming a bug in GC, but they all seem to be resolved in <3.8 versions.
What might be causing this warning?
P.S. I'm aware that fetching data from disk in a coroutine is bad and won't scale, I'm planning to fix it, I just have other things to deal with currently. But comments on other design flaws are welcome.

Python asyncio: RuntimeError Non-thread-safe operation invoked on an event loop other than the current one

I have setup a Python program that relies on asyncio and socketio to transfer messages between chat participants using two different chat systems (one interface being a CRM backend and the other being a javascript chat widget on a website).
The relevant part of the code is below. I need to open a separate thread for each independent chat. Therefore, in order to open a channel to the backend, I create a separate thread and run a "long_polling" function in it, which constantly checks for new messages from the backend via pull_messages() and sends them to the widget asynchronously via socketio AsyncServer (forwarding messages from widget to backend works fine and I leave it out here):
thread = threading.Thread(target=long_poll, args=())
thread.daemon = True
thread.start()
with the long_poll() function being defined as
self.server = AsyncServer(async_mode="sanic")
async def send_aio(self, msg):
await self.server.emit(msg)
def long_poll(self):
while chat:
response = self.pull_messages(...)
if response == "/end":
chat = False
asyncio.set_event_loop(asyncio.new_event_loop())
loop = asyncio.get_event_loop()
loop.run_until_complete(self.send_aio(response))
loop.close()
When the long_poll function is executed, just after the human response is fetched, I get the following error:
ERROR asyncio - Task exception was never retrieved
future: <Task finished coro=<AsyncServer._emit_internal() done, defined at /usr/local/lib/python3.6/site-packages/socketio/asyncio_server.py:344> exception=RuntimeError('Non-thread-safe operation invoked on an event loop other than the current one',)>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/socketio/asyncio_server.py", line 354, in _emit_internal
binary=None))
File "/usr/local/lib/python3.6/site-packages/socketio/asyncio_server.py", line 365, in _send_packet
await self.eio.send(sid, encoded_packet, binary=False)
File "/usr/local/lib/python3.6/site-packages/engineio/asyncio_server.py", line 89, in send
binary=binary))
File "/usr/local/lib/python3.6/site-packages/engineio/asyncio_socket.py", line 74, in send
await self.queue.put(pkt)
File "/usr/local/lib/python3.6/asyncio/queues.py", line 141, in put
return self.put_nowait(item)
File "/usr/local/lib/python3.6/asyncio/queues.py", line 153, in put_nowait
self._wakeup_next(self._getters)
File "/usr/local/lib/python3.6/asyncio/queues.py", line 74, in _wakeup_next
waiter.set_result(None)
File "uvloop/loop.pyx", line 1251, in uvloop.loop.Loop.call_soon
File "uvloop/loop.pyx", line 644, in uvloop.loop.Loop._check_thread
RuntimeError: Non-thread-safe operation invoked on an event loop other than the current one
Strange thing is, when I test locally via docker, this error does not happen, only once I migrate to Kubernetes (images and all other settings are the same).
What I have tried so far to analyze the situation:
Put logger.info("Using thread: {}".format(threading.current_thread().name)) into the long_poll() function, just before the error occurs. It tells me code is run in Thread-n (n is always 1 locally and usually some higher integer in the Kubernetes Pod), which is different from MainThread where other parts of my code with asyncio run, therefore I thought I am safe (I am aware the above asyncio code is not thread-safe). In case you have any suggestions, let me know.
UPDATE:
As user4815162342 suggested, instead of creating and destroying the event loops within the while loop, I created a single event loop in a dedicated thread and then pass the coroutine I need to run to the loop in the thread via asyncio.run_coroutine_threadsafe(). Not sure whether I did everything correct here, but now this part of the code is blocking the rest of the program (e.g. no more messages from user to backend are possible...)
self.server = AsyncServer(async_mode="sanic")
async def send_aio(self, msg):
await self.server.emit(msg)
def long_poll(self, loop):
while chat:
response = self.pull_messages(...)
if response == "/end":
chat = False
future = asyncio.run_coroutine_threadsafe(self.send_io(response), loop)
_ = future.result()
In the main program:
...
loop = asyncio.new_event_loop()
lpt = LongPollingObject()
threading.Thread(target=lpt.long_poll, args=(loop,), daemon=True).start()
...

Combining 2 asyncio based code segments

I'm using the Autobahn asyncio system (to talk the Websocket WAMP protocol), which works fine and I can handle incoming RPC calls and pubsub.
My problem is I now have to connect TCP sockets and send information over these sockets as soon as an RPC call comes in through the Autobahn part.
The autobahn part works fine like this :
from autobahn.asyncio.component import Component, run
from asyncio import sleep
from autobahn.asyncio.wamp import ApplicationSession, ApplicationRunner
#comp.on_join
async def joined(session, details):
print("Connected to websocket")
def on_message(msg):
msg = json.loads(msg)
print(msg)
def some_rpc(with_data):
print("Doing something with the data")
return json.dumps({'status': 'OK'})
try:
session.subscribe(on_message, u'some_pubsub_topic')
session.register(some_rpc, u'some_rpc_call')
print("RPC and Pubsub initialized")
except Exception as e:
print("could not subscribe to topic: {0}".format(e))
if __name__ == "__main__":
run([comp])
However now I need to be able to connect to multiple regular TCP sockets :
class SocketClient(asyncio.Protocol):
def __init__(self, loop):
self.data = b''
self.loop = loop
def connection_made(self, transport):
self.transport = transport
print('connected')
def data_received(self, data):
print('Data received: {!r}'.format(data.decode()))
def send(self, data):
self.transport.write(data)
def connection_lost(self, exc):
print('The server closed the connection')
print('Stop the event loop')
self.loop.stop()
loop = asyncio.get_event_loop()
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
loop.run_until_complete(c)
loop.run_forever()
loop.close()
The problem is that, when I combine both and do this :
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
It barks at me and tells me :
StopIteration
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/usr/lib/python3.5/site-packages/autobahn/wamp/websocket.py", line
95, in onMessage
self._session.onMessage(msg) File "/usr/lib/python3.5/site-packages/autobahn/wamp/protocol.py", line
894, in onMessage
on_reply = txaio.as_future(endpoint.fn, *invoke_args, **invoke_kwargs) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 400, in
as_future
return create_future_error(create_failure()) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 393, in
create_future_error
reject(f, error) File "/usr/lib/python3.5/site-packages/txaio/aio.py", line 462, in reject
future.set_exception(error.value) File "/usr/lib64/python3.5/asyncio/futures.py", line 365, in set_exception
raise TypeError("StopIteration interacts badly with generators " TypeError: StopIteration interacts badly with generators and cannot be
raised into a Future
Does anyone have any idea on how to call the send function from within the RPC call function ?

In this code:
c=loop.create_connection(lambda: SocketClient(loop),
'192.168.0.219', 6773)
# [...]
def some_rpc(with_data):
c.send('test')
return json.dumps({'status': 'OK'})
create_connection is a coroutine function, so c contains a coroutine object. Such object does have a send method, but one that is entirely unrelated to sending things over the network. After calling create_connection, you probably want to get the resulting transport with something like:
transport, ignore = loop.run_until_complete(c)
and then use transport.write(), not c.send().