I've been banging my head against my keyboard for a day now, so I'm giving up and asking for help.
I've got a working consumer using confluence kafka, but I need to make it run as a coroutine so I can get things working with FastAPI. I really wanted to try out AIOKafka for this, but for the life of me, I can't get it to work with a self-signed certificate (this is in our dev env).
Here is the working config for my confluence kafka consumer:
conf = {
"bootstrap.servers": "10.142.252.214:9093",
"group.id": "myConsumerID",
"security.protocol": "SASL_SSL",
"sasl.username": kafkaUser,
"sasl.password": kafkaPass,
"sasl.mechanisms": "PLAIN",
"enable.ssl.certificate.verification": "False",
"on_commit": commit_completed,
"heartbeat.interval.ms": "1000",
"socket.connection.setup.timeout.ms": "10000",
"auto.offset.reset": "earliest"
}
Here is the code I'm trying to use for AIOKafka
async def consume():
cert = "../foo/cert/certificate.pem"
key = "../foo/cert/key.pem"
context2 = ssl.create_default_context()
context2.load_cert_chain(certfile=cert, keyfile=key)
context2.check_hostname = False
context2.verify_mode = CERT_NONE
#context2.ssl_cafile="../foo/cert/CARoot.pem"
context2.ssl_certfile = "cert.pem"
context2.ssl_keyfile = "key.pem"
context2.ssl_password = kafkaKey
context2.ssl_keystore_type = "PEM"
consumer = AIOKafkaConsumer(
'TopicA', 'TopicB',
bootstrap_servers="10.142.252.214:9093",
group_id="myConsumerGroup",
sasl_plain_username="kafkaUser",
sasl_plain_password="kafkaPass",
sasl_mechanism="PLAIN",
security_protocol="SASL_SSL",
ssl_context=context2)
await consumer.start()
try:
# Consume messages
async for msg in consumer:
print("consumed: ", msg.topic, msg.partition, msg.offset,
msg.key, msg.value, msg.timestamp)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
When I try to run this, I just get the most cryptic errors ever and I can't make any sense on where to start trying to figure out what's wrong.
$ python test-main.py
Traceback (most recent call last):
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 375, in _on_read_task_error
read_task.result()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 518, in _read
resp = await reader.readexactly(4)
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/streams.py", line 706, in readexactly
raise exceptions.IncompleteReadError(incomplete, n)
asyncio.exceptions.IncompleteReadError: 0 bytes read on a total of 4 expected bytes
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/myUser/scripts/ansible-hello-world/test-main.py", line 165, in <module>
asyncio.run(consume())
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/base_events.py", line 646, in run_until_complete
return future.result()
File "/Users/myUser/scripts/ansible-hello-world/test-main.py", line 155, in consume
await consumer.start()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/consumer/consumer.py", line 346, in start
await self._client.bootstrap()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/client.py", line 210, in bootstrap
bootstrap_conn = await create_conn(
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 96, in create_conn
await conn.connect()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 234, in connect
await self._do_sasl_handshake()
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/site-packages/aiokafka/conn.py", line 314, in _do_sasl_handshake
auth_bytes = await self._send_sasl_token(
File "/Users/myUser/.pyenv/versions/3.10.5/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
return fut.result()
kafka.errors.KafkaConnectionError: KafkaConnectionError: Connection at 10.142.252.214:9093 closed
Unclosed AIOKafkaConsumer
consumer: <aiokafka.consumer.consumer.AIOKafkaConsumer object at 0x107338670>
Related
I create a bot that notifies the user at certain times, but from time to time gives this error
dispatcher.py [LINE:390] ERROR | 2022-10-03 04:10:16,846 : Cause exception while getting updates.
Traceback (most recent call last):
File "/usr/local/lib/python3.8/dist-packages/aiogram/dispatcher/dispatcher.py", line 381, in start_polling
updates = await self.bot.get_updates(
File "/usr/local/lib/python3.8/dist-packages/aiogram/bot/bot.py", line 110, in get_updates
result = await self.request(api.Methods.GET_UPDATES, payload)
File "/usr/local/lib/python3.8/dist-packages/aiogram/bot/base.py", line 231, in request
return await api.make_request(await self.get_session(), self.server, self.__token, method, data, files,
File "/usr/local/lib/python3.8/dist-packages/aiogram/bot/api.py", line 140, in make_request
return check_result(method, response.content_type, response.status, await response.text())
File "/usr/local/lib/python3.8/dist-packages/aiogram/bot/api.py", line 128, in check_result
raise exceptions.TelegramAPIError(description)
aiogram.utils.exceptions.TelegramAPIError: Bad Gateway
I think this problem is on the telegram side itself and is solved via webhook, but I don't want to use them.
import discord
from discord.ext import commands
import random
import praw
cl = commands.Bot(command_prefix = '!')
reddit = praw.Reddit(client_id = "",
client_secret = "",
username = "",
password = "",
user_agent = "")
#cl.event
async def on_ready():
print("Bot is ready, get ready to do wutever u want with it")
#cl.command()
async def meme(ctx, amount=50, subr="memes", filter="top"):
all_submission = []
subreddit = reddit.subreddit("subr")
subs = subreddit.filter(limit = amount)
for submission in subs:
all_submission.append(submission)
random_sub = random.choice(all_submission)
name = random_sub.title
url = random_sub.url
em = discord.embed(title = name)
em.set_image = url
await ctx.send(embed=em)
print("embed sent")
cl.run("")
when I was running this nothing showed up but when I debugged it and !meme in discord it was showing me this traceback error thing
It appears that you are using PRAW in an asynchronous environment.
It is strongly recommended to use Async PRAW: https://asyncpraw.readthedocs.io.
Ignoring exception in command meme:
Traceback (most recent call last):
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "main.py", line 20, in meme
subs = subreddit.filter(limit = amount)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/base.py", line 34, in __getattr__
self._fetch()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/subreddit.py", line 584, in _fetch
data = self._fetch_data()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/models/reddit/subreddit.py", line 581, in _fetch_data
return self._reddit.request("GET", path, params)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/praw/reddit.py", line 885, in request
return self._core.request(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 330, in request
return self._request_with_retries(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 228, in _request_with_retries
response, saved_exception = self._make_request(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 185, in _make_request
response = self._rate_limiter.call(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/rate_limit.py", line 33, in call
kwargs["headers"] = set_header_callback()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/sessions.py", line 283, in _set_header_callback
self._authorizer.refresh()
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/auth.py", line 425, in refresh
self._request_token(
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/prawcore/auth.py", line 158, in _request_token
raise OAuthException(
prawcore.exceptions.OAuthException: invalid_grant error processing request
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/bot.py", line 939, in invoke
await ctx.command.invoke(ctx)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 863, in invoke
await injected(*ctx.args, **ctx.kwargs)
File "/home/runner/memes-bot/venv/lib/python3.8/site-packages/discord/ext/commands/core.py", line 94, in wrapped
raise CommandInvokeError(exc) from exc
discord.ext.commands.errors.CommandInvokeError: Command raised an exception: OAuthException: invalid_grant error processing request
This is the error thing I was talking about and it is very weird like telling me to download async praw which I have never had a problem with and pls help it is needed
This doesn't appear to be a discord.py issue, it appears to be related to the praw 0auth flow.
According to the traceback, there's an authentication issue in your praw credentials. Double, triple, and quadruple check your authentication credentials and flow with the workflow and methodology in the docs
I'm trying to connect to a ble device with bleak that uses a passkey.
async def connect(self, address):
print("Connecting to device...")
async with BleakClient(address) as client:
response = await client.connect()
print(response)
i'm using above code. The device displays the passkey to enter and windows displays the add a device message to input the code, but i get a timeout error from asyncio and never recieve the response to python.
Traceback (most recent call last):
File "C:\Users\halmelam\PycharmProjects\read_passcode\main.py", line 137, in <module>
asyncio.run(pair.connect(device.address))
File "C:\Users\halmelam\AppData\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\halmelam\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "C:\Users\halmelam\PycharmProjects\read_passcode\main.py", line 95, in connect
response = await client.connect()
File "C:\Users\halmelam\.virtualenvs\read_passcode\lib\site-packages\bleak\backends\winrt\client.py", line 249, in connect
await asyncio.wait_for(event.wait(), timeout=timeout)
File "C:\Users\halmelam\AppData\Local\Programs\Python\Python39\lib\asyncio\tasks.py", line 494, in wait_for
raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError
How do I get the response and how can i send the passkey via python?
I'm trying to create a feature where users get notified when its their birthday. I've been at this for weeks but can't seem to find the error.
The problem is that I keep getting an error of:
"Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "main.py", line 59, in check_for_birthdays
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/abc.py", line 1065, in send
data = await state.http.send_message(channel.id, content, tts=tts, embed=embed,
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/discord/http.py", line 192, in request
async with self.__session.request(method, url, **kwargs) as r:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 1117, in __aenter__
self._resp = await self._coro
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/client.py", line 448, in _request
with timer:
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/aiohttp/helpers.py", line 635, in __enter__
raise RuntimeError(
RuntimeError: Timeout context manager should be used inside a task"
I keep trying different things such as await asyncio.sleep(time_to_sleep) and try to get_channel() but still get errors.
The code:
async def check_for_birthdays(time_to_sleep):
while True:
from datetime import datetime
from pytz import timezone
tz = timezone('EST')
obj = datetime.now(tz)
string = str(obj).split()[1]
index = string.index(":")
hour = int(string[:index]) + 1
# user = await client.fetch_user(user_id)
# user.send("hello user, some message to user")
# if hour != (9):
# # await channel.send("checking birthdays, not 9am currently.")
# await asyncio.sleep(time_to_sleep)
# continue
# await user.send("9am check successful.")
lst_of_users_today = is_anyones_birthday()
server_id = some_server_id
guild = client.get_guild(server_id)
channel_id = some_channel_id
channel = guild.get_channel(channel_id)
for cord_user_id in lst_of_users_today:
await channel.send("Hey Everyone, Let's wish <#" + cord_user_id + "> Happy Birthday!!!")
time.sleep(time_to_sleep)
import asyncio
threading.Thread(target=asyncio.run, args=(check_for_birthdays(THIRTY_MINUTES*2),)).start()
If you have any suggestions to solve this problem another way, I'm open to suggestions. Basically, allow discord bot to run as normal without being interrupted and check on a specific hour every day if there is a birthday match, for which it shall notify them in the discord channel.
It's a weird error since when I try/catch it, it prints nothings.
I'm using sanic server to asyncio.gather a bunch of images concurrently, more than 3 thousand images.
I haven't got this error when dealing with a smaller sample size.
Simplified example :
from sanic import Sanic
from sanic import response
from aiohttp import ClientSession
from asyncio import gather
app = Sanic()
#app.listener('before_server_start')
async def init(app, loop):
app.session = ClientSession(loop=loop)
#app.route('/test')
async def test(request):
data_tasks = []
#The error only happened when a large amount of images were used
for imageURL in request.json['images']:
data_tasks.append(getRaw(imageURL))
await gather(*data_tasks)
return response.text('done')
async def getRaw(url):
async with app.session.get(url) as resp:
return await resp.read()
What could this error be? If it is some kind of limitation of my host/internet, how can I avoid it?
I'm using a basic droplet from DigitalOcean with 1vCPU and 1GB RAM if that helps
Full stack error :
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/sanic/app.py", line 750, in handle_request
response = await response
File "server-sanic.py", line 53, in xlsx
await gather(*data_tasks)
File "/usr/lib/python3.5/asyncio/futures.py", line 361, in __iter__
yield self # This tells Task to wait for completion.
File "/usr/lib/python3.5/asyncio/tasks.py", line 296, in _wakeup
future.result()
File "/usr/lib/python3.5/asyncio/futures.py", line 274, in result
raise self._exception
File "/usr/lib/python3.5/asyncio/tasks.py", line 241, in _step
result = coro.throw(exc)
File "server-sanic.py", line 102, in add_data_to_sheet
await add_img_to_sheet(sheet, rowIndex, colIndex, val)
File "server-sanic.py", line 114, in add_img_to_sheet
image_data = BytesIO(await getRaw(imgUrl))
File "server-sanic.py", line 138, in getRaw
async with app.session.get(url) as resp:
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 690, in __aenter__
self._resp = yield from self._coro
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client.py", line 277, in _request
yield from resp.start(conn, read_until_eof)
File "/usr/local/lib/python3.5/dist-packages/aiohttp/client_reqrep.py", line 637, in start
self._continue = None
File "/usr/local/lib/python3.5/dist-packages/aiohttp/helpers.py", line 732, in __exit__
raise asyncio.TimeoutError from None
concurrent.futures._base.TimeoutError
There is no benefit to launching a million requests at once. Limit it to 10 or whatever works and wait for those before continuing the loop.
for imageURL in request.json['images']:
data_tasks.append(getRaw(imageURL))
if len(data_tasks) > 10:
await gather(*data_tasks)
data_tasks = []
await gather(*data_tasks)