Develop Reference

'task() takes 0 positional arguments but 1 was given' for python asyncio function

Within my python code, I am trying to design a piece of client code that connects to a WebSockets Server every second and then prints the timestamp and the obtained value from the server in a .csv file. This is given below:
import asyncio
import websockets
import logging
import datetime
import time
starttime = time.time() # start value for timed data acquisition
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
class Timer: # class for asynchronous (non-blocking) counter
def __init__(self, interval, first_immediately, callback):
self._interval = interval
self._first_immediately = first_immediately
self._callback = callback
self._is_first_call = True
self._ok = True
self._task = asyncio.ensure_future(self._job())
print("init timer done")
async def _job(self):
try:
while self._ok:
if not self._is_first_call or not self._first_immediately:
await asyncio.sleep(self._interval)
await self._callback(self)
self._is_first_call = False
except Exception as ex:
print(ex)
def cancel(self):
self._ok = False
self._task.cancel()
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/", ping_interval=None
) as websocket:
await websocket.send(
str(1.001)
) # send a message to the websocket server
response = (
await websocket.recv()
) # wait to get a response from the server
print(response)
dataline_pv1 = (
datetime.datetime.today().isoformat()
+ ","
+ str(response)
+ ","
+ str(0)
+ "\n"
) # format and assemble data line
file_name_pv1 = (
"{:%Y%m%d}".format(datetime.datetime.today()) + "_flow.csv"
) # generate file name
with open(
file_name_pv1, "a"
) as etherm_file1: # append dataline to file
etherm_file1.write(dataline_pv1)
# asyncio.get_event_loop().run_forever(test()) # run until test() is finished while True:
timer = Timer(interval=1, first_immediately=True, callback=test)
loop = asyncio.get_event_loop()
try:
asyncio.ensure_future(test())
loop.run_forever()
except KeyboardInterrupt:
timer.cancel()
pass
finally:
print("Closing Loop")
loop.close()
When this runs, I obtain the following message of my terminal (however the code does not crash):
test() takes 0 positional arguments but 1 was given
I have seen from this question (TypeError: takes 0 positional arguments but 1 was given) that this error occurs when a Class object is not defined properly, but my error seems to be occurring outside of a class framework. In addition, the desired .csv file is produced, however only one line is printed to the file, and does not repeat every second as desired.
What am I missing here? (also I am a complete novice with asyncio programming)
UPDATE: After changing the definition of test() to async def test(timer=None), my code now runs as expected and outputs the values to a .csv file every second (roughly), but still throws up an error. Specifically:
Task exception was never retrieved
future: <Task finished coro=<test() done, defined at flowmeterclient_v2.py:36> exception=ConnectionRefusedError(111, "Connect call failed ('198.162.1.177', 80)")>
Traceback (most recent call last):
File "flowmeterclient_v2.py", line 37, in test
async with websockets.connect("ws://198.162.1.177:80/", ping_interval=None) as websocket:
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 604, in __aenter__
return await self
File "/usr/lib64/python3.6/site-packages/websockets/legacy/client.py", line 622, in __await_impl__
transport, protocol = await self._create_connection()
File "/usr/lib64/python3.6/asyncio/base_events.py", line 798, in create_connection
raise exceptions[0]
File "/usr/lib64/python3.6/asyncio/base_events.py", line 785, in create_connection
yield from self.sock_connect(sock, address)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 439, in sock_connect
return (yield from fut)
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 469, in _sock_connect_cb
raise OSError(err, 'Connect call failed %s' % (address,))
ConnectionRefusedError: [Errno 111] Connect call failed ('198.162.1.177', 80)

Your Timer code passes the timer itself as the first argument in
await self._callback(self)
You may wish to change the signature of test to
async def test(timer=None):
so you can call it with or without the timer.

I think you don't really need the Timer here at all. Simply have an asyncio task that loops forever and has an asyncio.sleep() internally.
This also doesn't reconnect to the websocket server for each request, like your previous code did.
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(logging.INFO) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def test():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
while True:
await websocket.send(str(1.001))
response = await websocket.recv()
print(response)
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
await asyncio.sleep(1)
asyncio.run(test())
Following up on comments, if you actually do need to reconnect for each request, you can refactor this like so:
import asyncio
import websockets
import logging
import datetime
logger = logging.getLogger("websockets")
logger.setLevel(
logging.INFO
) # Switch to DEBUG for full error information
logger.addHandler(logging.StreamHandler())
async def get_data():
async with websockets.connect(
"ws://198.162.1.177:80/",
ping_interval=None,
) as websocket:
await websocket.send(str(1.001))
response = await websocket.recv()
return response
def save_data(response):
now = datetime.datetime.now()
dataline_pv1 = f"{now.isoformat()},{response},0\n"
file_name_pv1 = f"{now:%Y%m%d}_flow.csv"
with open(file_name_pv1, "a") as etherm_file1:
etherm_file1.write(dataline_pv1)
async def test():
while True:
response = await get_data()
save_data(response)
await asyncio.sleep(1)
asyncio.run(test())

Python Tornado KeyError When removing client from clients set

I have a Python Tornado Websocket server that stores clients in a shared set() so that I know how many clients are connected.
The challenge is that calling on_close after WebSocketClosedError raises a KeyError and the client-instance is not removed from the set of connected clients. This error has caused my server to accumulate over 1000 clients even when the active clients are only around 5.
My Code:
import tornado.iostream
import tornado.websocket
import asyncio
class SocketHandler(tornado.websocket.WebSocketHandler):
socket_active_message = {"status": "Socket Connection Active"}
waiters = set()
def initialize(self):
self.client_name = "newly_connected"
def open(self):
print('connection opened')
# https://kite.com/python/docs/tornado.websocket.WebSocketHandler.set_nodelay
self.set_nodelay(True)
SocketHandler.waiters.add(self)
def on_close(self):
print("CLOSED!", self.client_name)
SocketHandler.waiters.remove(self)
def check_origin(self, origin):
# Override the origin check if needed
return True
async def send_updates(self, message):
print('starting socket service loop')
loop_counter = 0
while True:
try:
await self.write_message({'status': 82317581})
except tornado.websocket.WebSocketClosedError:
self.on_close()
except tornado.iostream.StreamClosedError:
self.on_close()
except Exception as e:
self.on_close()
print('Exception e:', self.client_name)
await asyncio.sleep(0.05)
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message)
def run_server():
# Create tornado application and supply URL routes
webApp = tornado.web.Application(
[
(
r"/",
SocketHandler,
{},
),
]
)
application = tornado.httpserver.HTTPServer(webApp)
webApp.listen(3433)
# Start IO/Event loop
tornado.ioloop.IOLoop.instance().start()
run_server()
The Stack-trace:
Traceback (most recent call last):
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/web.py", line 1699, in _execute
result = await result
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 278, in get
await self.ws_connection.accept_connection(self)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 881, in accept_connection
await self._accept_connection(handler)
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 964, in _accept_connection
await self._receive_frame_loop()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1118, in _receive_frame_loop
await self._receive_frame()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 1209, in _receive_frame
await handled_future
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/mnt/c/Users/EE/projects/new/venv/lib/python3.8/site-packages/tornado/websocket.py", line 658, in <lambda>
self.stream.io_loop.add_future(result, lambda f: f.result())
File "ask_So.py", line 50, in on_message
await self.send_updates(message)
File "ask_So.py", line 39, in send_updates
self.on_close()
File "ask_So.py", line 26, in on_close
SocketHandler.waiters.remove(self)
KeyError: <__main__.SocketHandler object at 0x7ffef9f25520>
I have tried moving the waiters set outside the class but it still produces the same behaviour.
To simulate WebSocketClosedError: open many browser tabs as clients and close one browser tab at a time.

It seems like self.on_close() is being called twice. Once you're calling it manually from inside send_updates() and then later, when a connection is actually closed, Tornado is also calling self.on_close(). Since the self object was already removed from the set the first time, it raises a KeyError the second time.
If you want to close the connection, just call self.close(). The self.on_close() method will be called by Tornado automatically.
Also, you can handle the exception in a try...except block inside on_close.
Update
The previous part of this answer should fix the KeyError related problem. This update is regarding why the clients are not being removed from waiters set.
So, I tested your code and found a major problem with it here:
async def on_message(self, message):
print("RECEIVED :", message)
self.client_name = message
await self.send_updates(message) # <- This is problematic
Whenever a client sends a message, it will run self.send_updates method. So even if there's only one client that sends a message, let's say, 10 times, send_updates will also be called 10 times and, as a result, you will have 10 while loops running simultaneously!
As the number of loops increase, it ultimately blocks the server. That means Tornado has no time to run other code as it's busy juggling so many while loops. Hence, the clients from the waiters are never removed.
Solution
Instead of calling send_updates everytime a message arrives, you can call it just one time. Just have a single while loop to send updates to all clients.
I'd update the code like this:
class SocketHandler(...):
# Make it a classmethod so that it can be
# called without an instance
#classmethod
async def send_updates(cls):
print('starting socket service loop')
loop_counter = 0
while True:
for waiter in cls.waiters:
# use `waiter` instead of `self`
try:
await waiter.write_message({'status': 82317581})
...
await asyncio.sleep(0.05)
Instead of calling send_updates from on_message, you'll have to tel IOLoop to call it once:
def run_server():
...
# schedule SocketHandler.send_updates to be run
tornado.ioloop.IOLoop.current().add_callback(SocketHandler.send_updates)
tornado.ioloop.IOLoop.current().start()
This will have only one while loop running for all clients.

What happens to existing awaits when WebSocket.close is called

If I open a Starlette/FastAPI WebSocket, what happens to any coroutines currently waiting to receive data from the client if I close the WebSocket from outside the coroutine? Does the call to receive raise an exception or does the coroutine sit in memory forever because it is never going to receive anything?
from fastapi import FastAPI
from fastapi.websockets import WebSocket
app = FastAPI()
open_sockets = {}
#app.websocket('/connection/')
async def connection(websocket: WebSocket):
await websocket.accept()
id = await websocket.receive_json()['id']
open_sockets[id] = websocket
while True:
data = await websocket.receive_json()
#app.post('/kill/{id}/')
async def kill(id=str):
# What does this do to the above `await websocket.receive_json()`?
await open_sockets[id].close()
del open_sockets[id]

It raises a starlette.websockets.WebSocketDisconnect with code 1006
File ".\websocket_close_test.py", line 27, in connection
data = await websocket.receive_json()
File "C:\Apps\Python38\lib\site-packages\starlette\websockets.py", line 98, in receive_json
self._raise_on_disconnect(message)
File "C:\Apps\Python38\lib\site-packages\starlette\websockets.py", line 80, in _raise_on_disconnect
raise WebSocketDisconnect(message["code"])
I did note that the .close() call blocked for a little while.
Here's the code I used to test, the second ws.send was never received.
import time
import requests
import websocket
ws = websocket.WebSocket()
ws.connect("ws://localhost:8000/connection/")
print('Sending id 0')
ws.send('{ "id": "0" }')
time.sleep(2)
print('Closing id 0')
requests.post('http://localhost:8000/kill/0/')
print('id 0 is closed')
time.sleep(2)
print('Trying to send data on closed connection')
ws.send('{ "id": "10" }')

How to make discord.py bot run forever if client.run() returns

Problem
My discord.py bot's client.run() is returning unexpectedly every few days or so followed by an error "Task was destroyed but it is pending!".
Question
Why is client.run() returning at all? And how can I change my bot to properly handle this issue and run forever?
Code (stripped out a lot to keep it simple for discussion):
import asyncio
import discord
TOKEN = 'my bot token goes here'
CHANNEL_ID = 'my channel id goes here'
async def DiscordMsgSendTask():
await client.wait_until_ready()
my_channel = discord.Object(id=CHANNEL_ID)
while not client.is_closed:
# wait a bit to prevent busy loop
await asyncio.sleep(2)
# check for and handle new event here
# if an event was handled then send a message to the channel
embed = discord.Embed(description='Event was handled')
await client.send_message(my_channel, embed=embed)
client = discord.Client()
while True:
client.loop.create_task(DiscordMsgSendTask())
try:
client.run(TOKEN)
except Exception as e:
logging.warning('Exception: ' + str(e))
client = discord.Client()
Additional Info
The bot's purpose is basically just to check for a new file in a specific directory, and then send a message to a specific discord channel to report it. This only occurs around 10 times a day, so the bot's traffic is extremely low.
In an attempt to make the bot tolerate errors/disconnects, I put it into the while True loop, which may be wrong approach.
Using Python 3.6.5 and Discord.py 0.16.12
Edit - Added traceback
Adding traceback from a previous failure earlier in the day.
2018-06-20 04:33:08 [ERROR] Task exception was never retrieved
future: <Task finished coro=<WebSocketCommonProtocol.run() done, defined at /usr/local/lib64/python3.6/site-packages/websockets/protocol.py:428> exception=ConnectionResetError(104, 'Connection reset by peer')>
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/websockets/protocol.py", line 434, in run
msg = yield from self.read_message()
File "/usr/local/lib64/python3.6/site-packages/websockets/protocol.py", line 456, in read_message
frame = yield from self.read_data_frame(max_size=self.max_size)
File "/usr/local/lib64/python3.6/site-packages/websockets/protocol.py", line 511, in read_data_frame
frame = yield from self.read_frame(max_size)
File "/usr/local/lib64/python3.6/site-packages/websockets/protocol.py", line 546, in read_frame
self.reader.readexactly, is_masked, max_size=max_size)
File "/usr/local/lib64/python3.6/site-packages/websockets/framing.py", line 86, in read_frame
data = yield from reader(2)
File "/usr/lib64/python3.6/asyncio/streams.py", line 674, in readexactly
yield from self._wait_for_data('readexactly')
File "/usr/lib64/python3.6/asyncio/streams.py", line 464, in _wait_for_data
yield from self._waiter
File "/usr/lib64/python3.6/asyncio/selector_events.py", line 723, in _read_ready
data = self._sock.recv(self.max_size)
ConnectionResetError: [Errno 104] Connection reset by peer
2018-06-20 04:33:08 [ERROR] Task was destroyed but it is pending!
task: <Task pending coro=<DiscordMsgSendTask() running at /home/bot.py:119> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x7fc99bfd7a68>()]>>
2018-06-20 04:33:08 [ERROR] Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x7fc999b59240>

You can't use the pre-defined run method since that only handles KeyboardInterrupt. You will need to create a exit strategy that closes all tasks while still leaving the loop running.
The following example will raise SystemExit (to simulate your unexpected fatal error (The RST - ConnectionError error)) when a message "die" is received by the bot. The SystemExit will get "caught" and restarts your bot. Upon KeyboardInterrupt is raised, the bot will exit successfully without error.
import asyncio
import discord
TOKEN = 'TOKEN_HERE'
client = discord.Client()
async def task():
await client.wait_until_ready()
while True:
await asyncio.sleep(1)
print('Running')
def handle_exit():
print("Handling")
client.loop.run_until_complete(client.logout())
# For python 3.9, use asyncio.all_tasks instead
for t in asyncio.Task.all_tasks(loop=client.loop):
if t.done():
t.exception()
continue
t.cancel()
try:
client.loop.run_until_complete(asyncio.wait_for(t, 5, loop=client.loop))
t.exception()
except (asyncio.InvalidStateError, asyncio.TimeoutError, asyncio.CancelledError):
pass
while True:
#client.event
async def on_message(m):
if m.content == 'die':
print("Terminating")
raise SystemExit
client.loop.create_task(task())
try:
client.loop.run_until_complete(client.start(TOKEN))
except SystemExit:
handle_exit()
except KeyboardInterrupt:
handle_exit()
client.loop.close()
print("Program ended")
break
print("Bot restarting")
client = discord.Client(loop=client.loop)
In Discord (someone types die):
die
In terminal (STDOUT):
Running
Running
Running
Terminating
Handling
Bot restarting
Running
Running
Running
Running
Running
<CTRL-C> (KeyboardInterrupt)
Handling
Program ended
Side note:
If you're still confused about your error, it's not your code to blame. Errno 104 is a server side fatal error that's usually unpreventable by the end users.

python 3.6 coroutine was never awaited

So when ever I run my program and connect to it with the echo client it gives me this error.
Starting server
Serving on ('127.0.0.1', 8881)
Exception in callback UVTransport._call_connection_made
handle: <Handle UVTransport._call_connection_made>
Traceback (most recent call last):
File "uvloop/cbhandles.pyx", line 52, in uvloop.loop.Handle._run (uvloop/loop.c:48414)
File "uvloop/handles/tcp.pyx", line 141, in uvloop.loop.TCPTransport._call_connection_made (uvloop/loop.c:80488)
File "uvloop/handles/basetransport.pyx", line 140, in uvloop.loop.UVBaseTransport._call_connection_made (uvloop/loop.c:65774)
File "uvloop/handles/basetransport.pyx", line 137, in uvloop.loop.UVBaseTransport._call_connection_made (uvloop/loop.c:65671)
AttributeError: 'coroutine' object has no attribute 'connection_made'
/home/kenton/Programming/bridal/bridal-middle/middle/lib/server.py:16:RuntimeWarning: coroutine 'handle_request' was never awaited
loop.run_forever()
As far as I know I have everything that should be awaited awaited.
Here is the code:
class Server:
def __init__(self, port):
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
server = loop.run_until_complete(self.init(loop))
print("Serving on {}".format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
print("\rclosing the server")
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
async def init(self, loop):
server = await loop.create_server(self.handle_request, '127.0.0.1', 8881)
return server
async def handle_request(self):
print(datetime.datetime.now())
reader = asyncio.StreamReader()
writer = asyncio.StreamWriter()
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
code = message.partition('-')
if code[0].startswith("1") or code[0].startswith("5"):
accounts = lib.settings.database.accounts
if code[0] == "101":
result = await self.login_101(code, accounts, writer)
if code[0] == "501":
result = await accounts.find_one({"username":code[2]})
print("looking up", code[0])
#code logic to keep asking for a valid username if one exists
if result is None:
username = code[2]
print(username, " does not exist. Creating")
writer.write(b"0")
await writer.drain()
data = await reader.read(100)
message = data.decode()
code = message.partition('-')
post = {"username":username,"password":code[0],"email":code[2]}
post_id = await accounts.insert_one(post).inserted_id
writer.write(b(post_id))
await writer.drain()
print("Closed the client socket")
writer.close()
print(datetime.datetime.now())

Regarding your error message, the actual error is:
AttributeError: 'coroutine' object has no attribute 'connection_made'
And the line below is just a warning (RuntimeWarning: coroutine 'handle_request' was never awaited).
You might be mixing asyncio.start_server with loop.create_server().
loop.create_server()'s first parameter is protocol_factory which is a callable that returns an instance of a Protocol (and not a coroutine as in your code above):
import asyncio
class MyProtocol(asyncio.Protocol):
def connection_made(self, transport):
print("Connection made", transport)
def data_received(self, data):
print("Data received", data)
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(MyProtocol, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
loop.run_forever()
See full echo server example here.