How to read and write in parallel from a asyncio reader, writer pair provided by asyncio.open_connection?
I tried asyncio.open_connection on 2 different loops like:
async def read():
reader, writer = await asyncio.open_connection('127.0.0.1', 5454)
while True:
readval = await reader.readline()
print(f"read {readval}")
async def write():
reader, writer = await asyncio.open_connection('127.0.0.1', 5454)
while True:
self.wsem.acquire()
msg = self.wq.popleft()
print("Writing " + msg)
writer.write(msg.encode())
await writer.drain()
threading.Thread(target=lambda: asyncio.run(write())).start()
threading.Thread(target=lambda: asyncio.run(read())).start()
but it seems like sometimes the write thread drains the read content from the read thread and it doesn't work well.
Then I tried sharing reader, writer between the 2 loops but it throws an exception
Exception in thread Thread-8:
Traceback (most recent call last):
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\threading.py", line 954, in _bootstrap_inner
self.run()
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\Lenovo\PycharmProjects\testape-adb-adapter\adapter\device_socket.py", line 89, in <lambda>
threading.Thread(target=lambda: asyncio.run(read())).start()
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "C:\Users\Lenovo\PycharmProjects\testape-adb-adapter\adapter\device_socket.py", line 72, in read
readval = await self.reader.readline()
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\streams.py", line 540, in readline
line = await self.readuntil(sep)
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\streams.py", line 632, in readuntil
await self._wait_for_data('readuntil')
File "C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\streams.py", line 517, in _wait_for_data
await self._waiter
RuntimeError: Task <Task pending name='Task-2' coro=<DeviceSocket.connect.<locals>.read() running at C:\Users\Lenovo\PycharmProjects\testape-adb-adapter\adapter\device_socket.py:72> cb=[_run_until_complete_cb() at C:\Users\Lenovo\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py:184]> got Future <Future pending> attached to a different loop
async def read():
connection_opened.wait()
while True:
print(f"reading")
if self.reader.at_eof():
continue
readval = await self.reader.readline()
print(f"read {readval}")
self.rq.append(readval.decode())
self.rsem.release(1)
async def write():
self.reader, self.writer = await asyncio.open_connection('127.0.0.1', 5454)
connection_opened.set()
while True:
self.wsem.acquire()
msg = self.wq.popleft()
print("Writing " + msg)
self.writer.write(msg.encode())
await self.writer.drain()
connection_opened = threading.Event()
threading.Thread(target=lambda: asyncio.run(write())).start()
threading.Thread(target=lambda: asyncio.run(read())).start()
I thought that should be a simple and rather common use case. What is the proper way to do that?
I suggest you change the thread functions into something like this:
t = self.loop.create_task(self.write)
and ends with:
loop.run_until_complete(t)
Because I'm missing the self.wq and self.wsem function and not sure what they means, I couldn't reproduce the Error message. Hope this sorts out the question for you.
Related
async def load_abit_class(message: types.Message, state: FSMContext):
async with state.proxy() as abit_data:
abit_data['class'] = int(message.text)
async with open('''D:/Code/data.json''', 'w') as file:
json.dump(abit_data, file)
await state.finish()
That's my code. I am trying to make a tg bot, and I need to write the abit_data dictionary to data.json. But my bot is going to be used by many people and that's so I have to someway write that data avoiding saving conflicts, which can be caused by two people, using the bot at the same moment. I tried to use async but I got traceback:
Task exception was never retrieved
future: <Task finished name='Task-31' coro=<Dispatcher._process_polling_updates() done, defined at C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\dispatcher.py:407> exception=AttributeError('__aenter__')>
Traceback (most recent call last):
File "C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 415, in _process_polling_updates
for responses in itertools.chain.from_iterable(await self.process_updates(updates, fast)):
File "C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 235, in process_updates
return await asyncio.gather(*tasks)
File "C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\dispatcher.py", line 256, in process_update
return await self.message_handlers.notify(update.message)
File "C:\Users\vvvpe\Desktop\Connect\venv_2\lib\site-packages\aiogram\dispatcher\handler.py", line 116, in notify
response = await handler_obj.handler(*args, **partial_data)
File "C:\Users\vvvpe\Desktop\Connect\handlers\General.py", line 77, in load_abit_class
async with open('''D:/Code/data.json''', 'w') as file:
AttributeError: __aenter__
Help, please!
I'm trying to create a feature where users get notified when its their birthday. I've been at this for weeks but can't seem to find the error.
The problem is that I keep getting an error of:
"Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "main.py", line 59, in check_for_birthdays
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/abc.py", line 1065, in send
data = await state.http.send_message(channel.id, content, tts=tts, embed=embed,
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/http.py", line 192, in request
async with self.__session.request(method, url, **kwargs) as r:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 1117, in __aenter__
self._resp = await self._coro
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 448, in _request
with timer:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/helpers.py", line 635, in __enter__
raise RuntimeError(
RuntimeError: Timeout context manager should be used inside a task"
I keep trying different things such as await asyncio.sleep(time_to_sleep) and try to get_channel() but still get errors.
The code:
async def check_for_birthdays(time_to_sleep):
while True:
from datetime import datetime
from pytz import timezone
tz = timezone('EST')
obj = datetime.now(tz)
string = str(obj).split()[1]
index = string.index(":")
hour = int(string[:index]) + 1
# user = await client.fetch_user(user_id)
# user.send("hello user, some message to user")
# if hour != (9):
# # await channel.send("checking birthdays, not 9am currently.")
# await asyncio.sleep(time_to_sleep)
# continue
# await user.send("9am check successful.")
lst_of_users_today = is_anyones_birthday()
server_id = some_server_id
guild = client.get_guild(server_id)
channel_id = some_channel_id
channel = guild.get_channel(channel_id)
for cord_user_id in lst_of_users_today:
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
time.sleep(time_to_sleep)
import asyncio
threading.Thread(target=asyncio.run, args=(check_for_birthdays(THIRTY_MINUTES*2),)).start()
If you have any suggestions to solve this problem another way, I'm open to suggestions. Basically, allow discord bot to run as normal without being interrupted and check on a specific hour every day if there is a birthday match, for which it shall notify them in the discord channel.
I am trying to run multiple discord bots with the same script at once. The way I am doing this is going down a csv file, to get the bot token and bot name. I then start my script:
import time, csv, re, random, string, sys, os, asyncio
from datetime import datetime
from threading import Thread
import discord
from dotenv import load_dotenv
from discord_webhook import DiscordWebhook, DiscordEmbed
def main(bottoken, botname):
TOKEN = (bottoken)
client = discord.Client()
#client.event
async def on_ready():
print(f'{client.user.name} has connected to Discord!')
#client.event
async def on_message(message):
#do stuff
print('Do stuff')
client.run(TOKEN)
def runtask():
if __name__ == '__main__':
with open('botinfo.csv', 'r') as f:
reader = csv.reader(f, delimiter=',')
for i, row in enumerate(reader):
Thread(target = main, args=(row[0], row[1])).start()
if __name__ == '__main__':
runtask()
But then I get this error:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/alexforman/Documents/GitHub/bot-price-monitor/frontend_multithread.py", line 14, in main
client = discord.Client()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/discord/client.py", line 216, in __init__
self.loop = asyncio.get_event_loop() if loop is None else loop
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/events.py", line 644, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-1'.
Exception in thread Thread-2:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/alexforman/Documents/GitHub/bot-price-monitor/frontend_multithread.py", line 14, in main
client = discord.Client()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/discord/client.py", line 216, in __init__
self.loop = asyncio.get_event_loop() if loop is None else loop
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/events.py", line 644, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-2'.
Does anyone have any ideas on how I could fix it?
You don't need to start a separate thread for each bot, the asyncio event loop is perfectly capable of serving all bots at once. (That's the kind of thing event loops are designed for.)
You just need to start all the bots as coroutines of the same asyncio event loop:
async def run_client(bottoken, botname):
TOKEN = (bottoken)
client = discord.Client()
#client.event
async def on_ready():
print(f'{client.user.name} has connected to Discord!')
#client.event
async def on_message(message):
#do stuff
print('Do stuff')
# can't use client.run() because it would block the event loop
# but we can use client.start(), which is a coroutine that does
# the same thing (and which is internally called by client.run)
await client.start(TOKEN)
async def main():
coros = []
with open('botinfo.csv', 'r') as f:
reader = csv.reader(f, delimiter=',')
for i, row in enumerate(reader):
coros.append(run_client(row[0], row[1]))
await asyncio.gather(*coros)
if __name__ == '__main__':
asyncio.run(main())
I am trying to build a websockets server where the client connects and sends a request of some kind, and then the server adds that request to a queue and returns something back when finished.
to simulate the process of queueing and waiting for tasks to finish, I called asyncio.sleep whenever the process is supposed to be called.
however, whenever I do that I get the following error from the client-side:
Traceback (most recent call last):
File "D:/Python/server/client1.py", line 10, in <module>
asyncio.get_event_loop().run_until_complete(message())
File "C:\ProgramData\Anaconda3\envs\server\lib\asyncio\base_events.py", line 587, in run_until_complete
return future.result()
File "D:/Python/server/client1.py", line 8, in message
print(await socket.recv())
File "C:\ProgramData\Anaconda3\envs\server\lib\site-packages\websockets\protocol.py", line 509, in recv
await self.ensure_open()
File "C:\ProgramData\Anaconda3\envs\server\lib\site-packages\websockets\protocol.py", line 812, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedOK: code = 1000 (OK), no reason
and this error from the server-side:
Task exception was never retrieved
future: <Task finished coro=<queue_dispatcher() done, defined at D:/RamNew2020/Python/server/asyncserver.py:39> exception=ConnectionClosedOK('code = 1000 (OK), no reason')>
Traceback (most recent call last):
File "D:/Python/server/asyncserver.py", line 45, in queue_dispatcher
await data_from_q.sendto_websocket("yay")
File "D:/Python/server/asyncserver.py", line 20, in sendto_websocket
await self.mywebsocket.send(message)
File "C:\ProgramData\Anaconda3\envs\server\lib\site-packages\websockets\protocol.py", line 555, in send
await self.ensure_open()
File "C:\ProgramData\Anaconda3\envs\server\lib\site-packages\websockets\protocol.py", line 803, in ensure_open
raise self.connection_closed_exc()
websockets.exceptions.ConnectionClosedOK: code = 1000 (OK), no reason
My scripts:
server.py:
import asyncio
import websockets
import datetime
import random
queue = asyncio.Queue()
async def rnd_sleep(t):
# sleep for T seconds on average
await asyncio.sleep(t * random.random() * 2)
#When a client sends a message the information is saved in a Request object,
#including the socket connection
class Request:
def __init__(self, mywebsocket, mydat):
self.mywebsocket = mywebsocket
self.mydat = mydat
self.start_time_of_request = datetime.datetime.now()
self.id = random.randint(0, 128)
async def sendto_websocket(self, message):
await self.mywebsocket.send(message)
def get_dat(self):
return self.mydat
def get_start_time(self):
return self.start_time_of_request
async def add_to_queue(websocket, path):
global queue
dat = await websocket.recv()
base64dat = Request(websocket, dat)
await queue.put(base64dat)
#problem in queue_dispatcher
async def queue_dispatcher(q):
while True:
data_from_q = await q.get()
print(data_from_q.id)
await asyncio.sleep(1) #The problem is here, whenever I remove it, it works
await data_from_q.sendto_websocket("yay")
q.task_done()
async def main():
global queue
asyncio.create_task(queue_dispatcher(queue))
pass
start_server = websockets.serve(add_to_queue, 'localhost', 5000)
asyncio.get_event_loop().run_until_complete(main())
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
client.py:
import asyncio
async def message():
async with websockets.connect("ws://localhost:5000")as socket:
msg = input("What do you want to send: ")
await socket.send(msg)
print(await socket.recv())
asyncio.get_event_loop().run_until_complete(message())
In addition, whenever I try to await any other async function, it works fine no matter how long.
This is my first attempt at using asyncio in python. Objective is to convert 40000+ htmls to jsons. Using a synchronous for loop takes about 3.5 minutes. I am interested to see the performance boost using asyncio. I am using the following code:
import glob
import json
from parsel import Selector
import asyncio
import aiofiles
async def read_html(path):
async with aiofiles.open(path, 'r') as f:
html = await f.read()
return html
async def parse_job(path):
html = await read_html(path)
sel_obj = Selector(html)
jobs = dict()
jobs['some_var'] = sel_obj.xpath('some-xpath').get()
return jobs
async def write_json(path):
job = await parse_job(path)
async with aiofiles.open(file_name.replace("html","json"), "w") as f:
await f.write(job)
async def bulk_read_and_write(files):
# this function is from realpython tutorial.
# I have little understanding of whats going on with gather()
tasks = list()
for file in files:
tasks.append(write_json(file))
await asyncio.gather(*tasks)
if __name__ == "__main__":
files = glob.glob("some_folder_path/*.html")
asyncio.run(bulk_read_and_write(files))
After a few seconds of running, I am getting the following error.
Traceback (most recent call last):
File "06_extract_jobs_async.py", line 84, in <module>
asyncio.run(bulk_read_and_write(files))
File "/anaconda3/envs/py37/lib/python3.7/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/anaconda3/envs/py37/lib/python3.7/asyncio/base_events.py", line 579, in run_until_complete
return future.result()
File "06_extract_jobs_async.py", line 78, in bulk_read_and_write
await asyncio.gather(*tasks)
File "06_extract_jobs_async.py", line 68, in write_json
job = await parse_job(path)
File "06_extract_jobs_async.py", line 35, in parse_job
html = await read_html(path)
File "06_extract_jobs_async.py", line 29, in read_html
async with aiofiles.open(path, 'r') as f:
File "/anaconda3/envs/py37/lib/python3.7/site-packages/aiofiles/base.py", line 78, in __aenter__
self._obj = yield from self._coro
File "/anaconda3/envs/py37/lib/python3.7/site-packages/aiofiles/threadpool/__init__.py", line 35, in _open
f = yield from loop.run_in_executor(executor, cb)
File "/anaconda3/envs/py37/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
OSError: [Errno 24] Too many open files: '../html_output/jobs/6706538_478752_job.html'
What is going on here? Thanks in advance
You're making async calls as fast as you can, but the process of writing a file to disk is still an effectively synchronous task. Your OS can try to perform multiple writes at once, but there is a limit. By spawning async tasks as quickly as possible, you're getting a lot of results at once, meaning a huge amount of files open for writing at the same time. As your error suggests, there is a limit.
There are lots of good threads on here about limiting concurrency with asyncio, but the easiest solution is probably asyncio-pool with a reasonable size.
Try adding a limit to the number of parallel tasks:
# ...rest of code unchanged
async def write_json(path, limiter):
with limiter:
job = await parse_job(path)
async with aiofiles.open(file_name.replace("html","json"), "w") as f:
await f.write(job)
async def bulk_read_and_write(files):
limiter = asyncio.Semaphore(1000)
tasks = []
for file in files:
tasks.append(write_json(file, limiter))
await asyncio.gather(*tasks)