It's a weird error since when I try/catch it, it prints nothings.
I'm using sanic server to asyncio.gather a bunch of images concurrently, more than 3 thousand images.
I haven't got this error when dealing with a smaller sample size.
Simplified example :
from sanic import Sanic
from sanic import response
from aiohttp import ClientSession
from asyncio import gather
app = Sanic()
#app.listener('before_server_start')
async def init(app, loop):
app.session = ClientSession(loop=loop)
#app.route('/test')
async def test(request):
data_tasks = []
#The error only happened when a large amount of images were used
for imageURL in request.json['images']:
data_tasks.append(getRaw(imageURL))
await gather(*data_tasks)
return response.text('done')
async def getRaw(url):
async with app.session.get(url) as resp:
return await resp.read()
What could this error be? If it is some kind of limitation of my host/internet, how can I avoid it?
I'm using a basic droplet from DigitalOcean with 1vCPU and 1GB RAM if that helps
Full stack error :
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/sanic/app.py", line 750, in handle_request
response = await response
File "server-sanic.py", line 53, in xlsx
await gather(*data_tasks)
File "/usr/lib/python3.5/asyncio/futures.py", line 361, in __iter__
yield self # This tells Task to wait for completion.
File "/usr/lib/python3.5/asyncio/tasks.py", line 296, in _wakeup
future.result()
File "/usr/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/usr/lib/python3.5/asyncio/tasks.py", line 241, in _step
result = coro.throw(exc)
File "server-sanic.py", line 102, in add_data_to_sheet
await add_img_to_sheet(sheet, rowIndex, colIndex, val)
File "server-sanic.py", line 114, in add_img_to_sheet
image_data = BytesIO(await getRaw(imgUrl))
File "server-sanic.py", line 138, in getRaw
async with app.session.get(url) as resp:
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 690, in __aenter__
self._resp = yield from self._coro
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 277, in _request
yield from resp.start(conn, read_until_eof)
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client_reqrep.py", line 637, in start
self._continue = None
File "/usr/local/lib/python3.5/dist-packages/aiohttp/helpers.py", line 732, in __exit__
raise asyncio.TimeoutError from None
concurrent.futures._base.TimeoutError
There is no benefit to launching a million requests at once. Limit it to 10 or whatever works and wait for those before continuing the loop.
for imageURL in request.json['images']:
data_tasks.append(getRaw(imageURL))
if len(data_tasks) > 10:
await gather(*data_tasks)
data_tasks = []
await gather(*data_tasks)
Related
I've been banging my head against my keyboard for a day now, so I'm giving up and asking for help.
I've got a working consumer using confluence kafka, but I need to make it run as a coroutine so I can get things working with FastAPI. I really wanted to try out AIOKafka for this, but for the life of me, I can't get it to work with a self-signed certificate (this is in our dev env).
Here is the working config for my confluence kafka consumer:
conf = {
"bootstrap.servers": "10.142.252.214:9093",
"group.id": "myConsumerID",
"security.protocol": "SASL_SSL",
"sasl.username": kafkaUser,
"sasl.password": kafkaPass,
"sasl.mechanisms": "PLAIN",
"enable.ssl.certificate.verification": "False",
"on_commit": commit_completed,
"heartbeat.interval.ms": "1000",
"socket.connection.setup.timeout.ms": "10000",
"auto.offset.reset": "earliest"
}
Here is the code I'm trying to use for AIOKafka
async def consume():
cert = "../foo/cert/certificate.pem"
key = "../foo/cert/key.pem"
context2 = ssl.create_default_context()
context2.load_cert_chain(certfile=cert, keyfile=key)
context2.check_hostname = False
context2.verify_mode = CERT_NONE
#context2.ssl_cafile="../foo/cert/CARoot.pem"
context2.ssl_certfile = "cert.pem"
context2.ssl_keyfile = "key.pem"
context2.ssl_password = kafkaKey
context2.ssl_keystore_type = "PEM"
consumer = AIOKafkaConsumer(
'TopicA', 'TopicB',
bootstrap_servers="10.142.252.214:9093",
group_id="myConsumerGroup",
sasl_plain_username="kafkaUser",
sasl_plain_password="kafkaPass",
sasl_mechanism="PLAIN",
security_protocol="SASL_SSL",
ssl_context=context2)
await consumer.start()
try:
# Consume messages
async for msg in consumer:
print("consumed: ", msg.topic, msg.partition, msg.offset,
msg.key, msg.value, msg.timestamp)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
When I try to run this, I just get the most cryptic errors ever and I can't make any sense on where to start trying to figure out what's wrong.
$ python test-main.py
Traceback (most recent call last):
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 375, in _on_read_task_error
read_task.result()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 518, in _read
resp = await reader.readexactly(4)
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/streams.py", line 706, in readexactly
raise exceptions.IncompleteReadError(incomplete, n)
asyncio.exceptions.IncompleteReadError: 0 bytes read on a total of 4 expected bytes
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/myUser/scripts/ansible-hello-world/test-main.py", line 165, in <module>
asyncio.run(consume())
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/base_events.py", line 646, in run_until_complete
return future.result()
File "/Users/myUser/scripts/ansible-hello-world/test-main.py", line 155, in consume
await consumer.start()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/consumer/consumer.py", line 346, in start
await self._client.bootstrap()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/client.py", line 210, in bootstrap
bootstrap_conn = await create_conn(
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 96, in create_conn
await conn.connect()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 234, in connect
await self._do_sasl_handshake()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 314, in _do_sasl_handshake
auth_bytes = await self._send_sasl_token(
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
return fut.result()
kafka.errors.KafkaConnectionError: KafkaConnectionError: Connection at 10.142.252.214:9093 closed
Unclosed AIOKafkaConsumer
consumer: <aiokafka.consumer.consumer.AIOKafkaConsumer object at 0x107338670>
import discord
from discord.ext import commands
import random
import praw
cl = commands.Bot(command_prefix = '!')
reddit = praw.Reddit(client_id = "",
client_secret = "",
username = "",
password = "",
user_agent = "")
#cl.event
async def on_ready():
print("Bot is ready, get ready to do wutever u want with it")
#cl.command()
async def meme(ctx, amount=50, subr="memes", filter="top"):
all_submission = []
subreddit = reddit.subreddit("subr")
subs = subreddit.filter(limit = amount)
for submission in subs:
all_submission.append(submission)
random_sub = random.choice(all_submission)
name = random_sub.title
url = random_sub.url
em = discord.embed(title = name)
em.set_image = url
await ctx.send(embed=em)
print("embed sent")
cl.run("")
when I was running this nothing showed up but when I debugged it and !meme in discord it was showing me this traceback error thing
It appears that you are using PRAW in an asynchronous environment.
It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.
Ignoring exception in command meme:
Traceback (most recent call last):
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "main.py", line 20, in meme
subs = subreddit.filter(limit = amount)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/base.py", line 34, in __getattr__
self._fetch()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/subreddit.py", line 584, in _fetch
data = self._fetch_data()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/subreddit.py", line 581, in _fetch_data
return self._reddit.request("GET", path, params)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/reddit.py", line 885, in request
return self._core.request(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 330, in request
return self._request_with_retries(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 228, in _request_with_retries
response, saved_exception = self._make_request(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 185, in _make_request
response = self._rate_limiter.call(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/rate_limit.py", line 33, in call
kwargs["headers"] = set_header_callback()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 283, in _set_header_callback
self._authorizer.refresh()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/auth.py", line 425, in refresh
self._request_token(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/auth.py", line 158, in _request_token
raise OAuthException(
prawcore.exceptions.OAuthException: invalid_grant error processing request
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/bot.py", line 939, in invoke
await ctx.command.invoke(ctx)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 863, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: OAuthException: invalid_grant error processing request
This is the error thing I was talking about and it is very weird like telling me to download async praw which I have never had a problem with and pls help it is needed
This doesn't appear to be a discord.py issue, it appears to be related to the praw 0auth flow.
According to the traceback, there's an authentication issue in your praw credentials. Double, triple, and quadruple check your authentication credentials and flow with the workflow and methodology in the docs
I'm trying to create a feature where users get notified when its their birthday. I've been at this for weeks but can't seem to find the error.
The problem is that I keep getting an error of:
"Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "main.py", line 59, in check_for_birthdays
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/abc.py", line 1065, in send
data = await state.http.send_message(channel.id, content, tts=tts, embed=embed,
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/http.py", line 192, in request
async with self.__session.request(method, url, **kwargs) as r:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 1117, in __aenter__
self._resp = await self._coro
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 448, in _request
with timer:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/helpers.py", line 635, in __enter__
raise RuntimeError(
RuntimeError: Timeout context manager should be used inside a task"
I keep trying different things such as await asyncio.sleep(time_to_sleep) and try to get_channel() but still get errors.
The code:
async def check_for_birthdays(time_to_sleep):
while True:
from datetime import datetime
from pytz import timezone
tz = timezone('EST')
obj = datetime.now(tz)
string = str(obj).split()[1]
index = string.index(":")
hour = int(string[:index]) + 1
# user = await client.fetch_user(user_id)
# user.send("hello user, some message to user")
# if hour != (9):
# # await channel.send("checking birthdays, not 9am currently.")
# await asyncio.sleep(time_to_sleep)
# continue
# await user.send("9am check successful.")
lst_of_users_today = is_anyones_birthday()
server_id = some_server_id
guild = client.get_guild(server_id)
channel_id = some_channel_id
channel = guild.get_channel(channel_id)
for cord_user_id in lst_of_users_today:
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
time.sleep(time_to_sleep)
import asyncio
threading.Thread(target=asyncio.run, args=(check_for_birthdays(THIRTY_MINUTES*2),)).start()
If you have any suggestions to solve this problem another way, I'm open to suggestions. Basically, allow discord bot to run as normal without being interrupted and check on a specific hour every day if there is a birthday match, for which it shall notify them in the discord channel.
I'm making a discord.py bot and I want to create a async function to send a message. However, a I want to call this function from a non-async function.
This is the discord client initialization
import requests
import json
from datetime import datetime
import discord
import asyncio
from discord.ext import commands
import firebase_admin
from firebase_admin import credentials, firestore
intents = discord.Intents.all()
client = commands.Bot(command_prefix='&', intents=intents)
This is my send_message function
async def msg(msg,channel):
await channel.send(msg)
This is the non-async function that's calling msg()
def checkStart(doc_snapshot,changes,read_time):
for change in changes:
if change.type.name == 'ADDED' or change.type.name == 'MODIFIED':
anaDoc = change.document.to_dict()
if anaDoc['start'] == True:
asyncio.run(msg('Game started!',main_channel[change.document.reference.parent.parent.id]))
print(change.document.reference.parent.parent.id)
Traceback
Thread-ConsumeBidirectionalStream caught unexpected exception Timeout context manager should be used inside a task and will exit.
Traceback (most recent call last):
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\google\api_core\bidi.py", line 657, in _thread_main
self._on_response(response)
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\google\cloud\firestore_v1\watch.py", line 462, in on_snapshot
meth(proto)
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\google\cloud\firestore_v1\watch.py", line 392, in _on_snapshot_target_change_no_change
self.push(change.read_time, change.resume_token)
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\google\cloud\firestore_v1\watch.py", line 568, in push
self._snapshot_callback(keys, appliedChanges, read_time)
File "c:\Users\Chico\Documents\newCartola\main.py", line 221, in checkStart
asyncio.run(msg('Jogo iniciado!',main_channel[change.document.reference.parent.parent.id]))
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "c:\Users\Chico\Documents\newCartola\main.py", line 213, in msg
await channel.send(msg)
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\discord\abc.py", line 904, in send
data = await state.http.send_message(channel.id, content, tts=tts, embed=embed,
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\discord\http.py", line 185, in request
async with self.__session.request(method, url, **kwargs) as r:
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__
self._resp = await self._coro
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\client.py", line 426, in _request
with timer:
File "C:\Users\Chico\AppData\Local\Programs\Python\Python39\lib\site-packages\aiohttp\helpers.py", line 579, in __enter__
raise RuntimeError('Timeout context manager should be used '
RuntimeError: Timeout context manager should be used inside a task
I'm trying to create a web app that communicates with Telegram. And trying to use Sanic web framework with Telepot. Both are asyncio based. Now I'm getting a very weird error.
This is my code:
import datetime
import telepot.aio
from sanic import Sanic
app = Sanic(__name__, load_env=False)
app.config.LOGO = ''
#app.listener('before_server_start')
async def server_init(app, loop):
app.bot = telepot.aio.Bot('anything', loop=loop)
# here we fall
await app.bot.sendMessage(
"#test",
"Wao! {}".format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),)
)
if __name__ == "__main__":
app.run(
debug=True
)
The error that I'm getting is:
[2018-01-18 22:41:43 +0200] [10996] [ERROR] Experienced exception while trying to serve
Traceback (most recent call last):
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/sanic/app.py", line 646, in run
serve(**server_settings)
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/sanic/server.py", line 588, in serve
trigger_events(before_start, loop)
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/sanic/server.py", line 496, in trigger_events
loop.run_until_complete(result)
File "uvloop/loop.pyx", line 1364, in uvloop.loop.Loop.run_until_complete
File "/home/mk/Dev/project/sanic-telepot.py", line 14, in server_init
"Wao! {}".format(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),)
File "/usr/lib/python3.6/asyncio/coroutines.py", line 109, in __next__
return self.gen.send(None)
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/telepot/aio/__init__.py", line 100, in sendMessage
return await self._api_request('sendMessage', _rectify(p))
File "/usr/lib/python3.6/asyncio/coroutines.py", line 109, in __next__
return self.gen.send(None)
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/telepot/aio/__init__.py", line 78, in _api_request
return await api.request((self._token, method, params, files), **kwargs)
File "/usr/lib/python3.6/asyncio/coroutines.py", line 109, in __next__
return self.gen.send(None)
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/telepot/aio/api.py", line 139, in request
async with fn(*args, **kwargs) as r:
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/aiohttp/client.py", line 690, in __aenter__
self._resp = yield from self._coro
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/aiohttp/client.py", line 221, in _request
with timer:
File "/home/mk/Dev/project/venv/lib/python3.6/site-packages/aiohttp/helpers.py", line 712, in __enter__
raise RuntimeError('Timeout context manager should be used '
RuntimeError: Timeout context manager should be used inside a task
Telepot inside uses aiohttp as dependency and for the HTTP calls. And the very similar code is working if I make very similar functionality with just aiohttp.web.
I'm not sure to what project this problem is related. Also, all other dependencies like redis, database connections that I connected the same approach are working perfectly.
Any suggestions how to fix it?