I am trying to obtain data from a redis channel by using a subscription on my client application. I am using python with asyncio and aioredis for this purpose.
I would like to use my subscription to have a variable of my main application updated when this one changes on the server, but I cannot manage to pass the data received from the subscription to my main thread.
According to aioredis website, I implemented my Subscription with:
sub = await aioredis.create_redis(
'redis://localhost')
ch1 = await sub.subscribe('channel:1')
assert isinstance(ch1, aioredis.Channel)
async def async_reader(channel, globarVar):
while await channel.wait_message():
msg = await channel.get(encoding='utf-8')
# ... process message ...
globarVar = float(msg)
print("message in {}: {}".format(channel.name, msg))
tsk1 = asyncio.ensure_future(async_reader(ch1, upToDateValue))
But I cannot get to update the global variable, I guess python pass just the current value as argument (which I expected to, but wanted to be sure).
Is there any viable option to get data out of a subscription? or to pass a reference to a shared variable or queue I could use?
You should redesign your code so you don't need a global variable. All of your processing should occur when receiving the message. However to modify a global variable you need to declare it in the function with the global keyword. You don't pass global variables around - you just use them.
Sub:
import aioredis
import asyncio
import json
gvar = 2
# Do everything you need here or call another function
# based on the message. Don't use a global variable.
async def process_message(msg):
global gvar
gvar = msg
async def async_reader(channel):
while await channel.wait_message():
j = await channel.get(encoding='utf-8')
msg = json.loads(j)
if msg == "stop":
break
print(gvar)
await process_message(msg)
print(gvar)
async def run(loop):
sub = await aioredis.create_redis('redis://localhost')
res = await sub.subscribe('channel:1')
ch1 = res[0]
assert isinstance(ch1, aioredis.Channel)
await async_reader(ch1)
await sub.unsubscribe('channel:1')
sub.close()
loop = asyncio.get_event_loop()
loop.run_until_complete( run(loop) )
loop.close()
publisher:
import asyncio
import aioredis
async def main():
pub = await aioredis.create_redis('redis://localhost')
res = await pub.publish_json('channel:1', ["Hello", "world"])
await asyncio.sleep(1)
res = await pub.publish_json('channel:1', "stop")
pub.close()
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())
Related
I need to listen tasks on 2 queues, so I wrote the code below, but it has a problem. Currently it behaves like this: if the code started when 2 queues were full, it works great. But if queues were empty one of them was, the code reads messages, but does not proccess them (does not send ack, does not do the logic). But the messages became unacked, until I stop the code. I do not see any reason to be them unacked and unprocessed.
I can't understand what is wrong with the code? May be there is another way to aggregate 2 or more queues like this?
# task_processor.py
from aio_pika import IncomingMessage
class TaskProcessor:
MAX_TASKS_PER_INSTANCE = 1
def __init__(self):
self._tasks = []
def can_accept_new_task(self) -> bool:
return len(self._tasks) < self.MAX_TASKS_PER_INSTANCE
async def process(self, message: IncomingMessage):
self._tasks.append(message)
print(message.body)
await message.ack()
self._tasks.pop()
# main.py
import asyncio
from asyncio import QueueEmpty
from typing import Callable
import aio_pika
from aio_pika import RobustQueue
from dotenv import load_dotenv
load_dotenv()
from core.logger.logger import logger
from core.services.rabbitmq.task_processor.task_processor import TaskProcessor
async def get_single_task(queue: RobustQueue):
while True:
try:
msg = await queue.get(timeout=3600)
return msg
except QueueEmpty:
await asyncio.sleep(3)
except asyncio.exceptions.TimeoutError:
logger.warning('queue timeout error')
pass
except Exception as ex:
logger.error(f"{queue} errored", exc_info=ex)
async def task_aggregator(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
while True:
if should_take_new_task_cb():
queue2, queue1 = queue1, queue2
gen1 = get_single_task(queue1)
gen2 = get_single_task(queue2)
done, _ = await asyncio.wait([gen1, gen2], return_when=asyncio.FIRST_COMPLETED)
for item in done:
result = item.result()
yield result
else:
await asyncio.sleep(1)
async def tasks(queue1: RobustQueue, queue2: RobustQueue, should_take_new_task_cb: Callable):
async for task in task_aggregator(queue1, queue2, should_take_new_task_cb):
yield task
async def main():
connection = await aio_pika.connect_robust(
f"amqp://user:password#host:port/vhost?heartbeat={180}"
)
channel1 = connection.channel()
channel2 = connection.channel()
await channel1.initialize()
await channel2.initialize()
queue1 = await channel1.get_queue('queue1')
queue2 = await channel2.get_queue('queue2')
task_processor = TaskProcessor()
task_generator = tasks(queue1, queue2, task_processor.can_accept_new_task)
while True:
if task_processor.can_accept_new_task():
task = await anext(task_generator)
await task_processor.process(task)
else:
await asyncio.sleep(1)
if __name__ == '__main__':
asyncio.run(main())
I have an idea of using an event-loop in a websocket-based server (based on websockets library), but I'm struggling to do so, since I really don't get the idea of how to use asyncio and write your own coroutines.
# let us define a dictionary of callbacks awaiting to be triggered
CALLBACKS = dict()
# let us have an infinite loop, which reads new messages from a websocket connection
async def websocket_connection(ws, path):
async for msg in ws:
# we received a message
msg = json.loads(msg)
msg_id = msg['id']
msg_data = msg['data']
# we check, if there is a callback waiting to be triggered with such id
if msg_id in CALLBACKS:
# if such id exists, we get the callback function
callback = CALLBACKS[msg_id]
# we delete the callback from the dictionary
del CALLBACKS[msg_id]
# we call it passing the data from the message in there
callback(msg_data)
else:
# we don't have such id, throw some error
raise Exception("bad id " + msg_id)
# this is the function which submits a message
async def submit(ws, data):
# it generates a random request id
from uuid import uuid4
request_id = str(uuid4())
# registers a callback, which will simply return the received data
CALLBACKS[request_id] = lambda data: # <- this is the part I don't know what it should really do to be able to return the value from the await
# after it encodes the message as json
msg = json.dumps({'id': request_id, 'data': data})
# and sends via the websocket
await ws.send(msg)
# and let us have the actual function, which will be the actual script
async def websocket_script(ws):
# this is what I would like to be able to do in the end.
# pass in commands to the websocket and await their completion
sum = int(await submit(ws, {"eval": "2+2"}))
division = float(await submit(ws, {"eval": "3/2"}))
websocket_connection and websocket_script would need to be going side by side for it to work. I suppose gather or some other async function would work. I would really like to get rid of callbacks, since this is the initial purpose of using asyncio in the first place.
How could this be done? It seems like a job for asyncio.
I think what you search is asyncio.Future if I understand you correctly.
FUTURES = dict()
async def websocket_connection(ws, path):
# ...
fut = FUTURES[msg_id]
fut.set_result(msg_data) # will "unblock" await fut below
del FUTURES[msg_id]
# ...
async def submit(ws, data):
from uuid import uuid4
request_id = str(uuid4())
fut = asyncio.Future()
FUTURES[request_id] = fut
msg = json.dumps({'id': request_id, 'data': data})
await ws.send(msg)
# ...
return (await fut)
async def websocket_script(ws):
sum = int(await submit(ws, {"eval": "2+2"}))
division = float(await submit(ws, {"eval": "3/2"}))
I am making a discord bot that will grab a json using requests from time to time, and then send the relevant information to a specific channel.
I have the following classes:
Helper, which is the discord bot itself, that runs async from the start, inside an asyncio.gather;
tasker that controls the interval which calls the class that will do the requests. It runs in a different thread so it doesn't stop the async Helper while it waits
getInfo that does the requests, store the info and should talk with Helper
I am having 2 problems right now:
While the tasker is on a different thread, every time I try to talk with Helper via getInfo it gives me the errors RuntimeError: no running event loop and RuntimeWarning: coroutine 'getInfo.discordmsg' was never awaited
If I dont run it on a different thread, however, it does work on the TestStatus: 1 but it makes Helper get stuck and stop running with TestStatus: 2
Anyway, here is the code
import requests
import asyncio
import discord
from discord.ext import commands, tasks
from datetime import datetime, timedelta
import threading
class Helper(discord.Client):
async def on_ready(self):
global discordbot, taskervar
servername = 'ServerName'
discordbot = self
self.servidores = dict()
self.canais = dict()
for i in range(len(self.guilds)):
self.servidores[self.guilds[i].name] = {}
self.servidores[self.guilds[i].name]['guild']=self.guilds[i]
servidor = self.guilds[i]
for k in range(len(servidor.channels)):
canal = servidor.channels[k]
self.canais[str(canal.name)] = canal
if 'bottalk' not in self.canais.keys():
newchan = await self.servidores[self.guilds[i].name]['guild'].create_text_channel('bottalk')
self.canais[str(newchan.name)] = newchan
self.servidores[self.guilds[i].name]['canais'] = self.canais
self.bottalk = self.get_channel(self.servidores[servername]['canais']['bottalk'].id)
await self.msg("Bot online: " + converteHora(datetime.now(),True))
print(f'{self.user} has connected to Discord!')
taskervar.startprocess()
async def msg(self, msg):
await self.bottalk.send(msg)
async def on_message(self, message):
if message.author == self.user:
return
else:
print(message)
class tasker:
def __init__(self):
global discordbot, taskervar
print('Tasker start')
taskervar = self
self.waiter = threading.Event()
self.lastupdate = datetime.now()
self.nextupdate = datetime.now()
self.thread = threading.Thread(target=self.requests)
def startprocess(self):
if not self.thread.is_alive():
self.waiter = threading.Event()
self.interval = 60*5
self.thread = threading.Thread(target=self.requests)
self.thread.start()
def requests(self):
while not self.waiter.is_set():
getInfo()
self.lastupdate = datetime.now()
self.nextupdate = datetime.now()+timedelta(seconds=self.interval)
self.waiter.wait(self.interval)
def stopprocess(self):
self.waiter.set()
class getInfo:
def __init__(self):
global discordbot, taskervar
self.requests()
async def discordmsg(self,msg):
await discordbot.msg(msg)
def requests(self):
jsondata = {"TestStatus": 1}
if jsondata['TestStatus'] == 1:
print('here')
asyncio.create_task(self.discordmsg("SOMETHING WENT WRONG"))
taskervar.stopprocess()
return
elif jsondata['TestStatus'] == 2:
print('test')
hora = converteHora(datetime.now(),True)
asyncio.create_task(self.discordmsg(str("Everything is fine but not now: " + hora )))
print('test2')
def converteHora(dateUTC, current=False):
if current:
response = (dateUTC.strftime("%d/%m/%Y, %H:%M:%S"))
else:
response = (dateutil.parser.isoparse(dateUTC)-timedelta(hours=3)).strftime("%d/%m/%Y, %H:%M:%S")
return response
async def main():
TOKEN = 'TOKEN GOES HERE'
tasker()
await asyncio.gather(
await Helper().start(TOKEN)
)
if __name__ == '__main__':
asyncio.run(main())
Your primary problem is you don't give your secondary thread access to the asyncio event loop. You can't just await and/or create_task a coroutine on a global object (One of many reasons to avoid using global objects in the first place). Here is how you could modify your code to accomplish that:
class tasker:
def __init__(self):
# ...
self.loop = asyncio.get_running_loop()
# ...
class getInfo:
#...
def requests(self):
# replace the create_tasks calls with this.
asyncio.run_coroutine_threadsafe(self.discordmsg, taskervar.loop)
This uses your global variables because I don't want to rewrite your entire program, but I still strongly recommend avoiding them and considering a re-write yourself.
All that being said, I suspect you will still have this bug:
If I dont run it on a different thread, however, it does work on the TestStatus: 1 but it makes Helper get stuck and stop running with TestStatus: 2
I can't tell what would cause this issue and I'm running into trouble reproducing this on my machine. Your code is pretty hard to read and is missing some details for reproducibility. I would imagine that is part of the reason why you didn't get an answer in the first place. I'm sure you're aware of this article but might be worth a re-visit for better practices in sharing code. https://stackoverflow.com/help/minimal-reproducible-example
I'm trying to write an async bridge between two instant messengers, let's say IRC and Telegram. Here's my code so far that doesn't work:
import asyncio
import logging
import random
async def read_from_irc(sock):
n = random.randint(1, 5)
await asyncio.sleep(n)
return "read_from_irc" + str(n)
async def read_from_telegram(sock):
n = random.randint(1, 5)
await asyncio.sleep(n)
return "read_from_telegram" + str(n)
async def main():
sock1 = await connect_to_irc()
sock2 = await connect_to_telegram()
while True:
w1 = read_from_irc(sock1)
w2 = read_from_telegram(sock2)
waits = {w1, w2}
done, pending = await asyncio.wait(waits)
print(done)
print(pending)
if w1 in done:
msg = await w1
await write_to_telegram(msg)
if w2 in done:
await write_to_irc(msg)
if __name__ == '__main__':
logging.basicConfig(level='DEBUG')
asyncio.run(main())
My problem is that in reality, read_from_irc() and read_from_telegram() are running read() functions that can block if there's no data waiting. Sometimes I want to send a message from IRC to Telegram, sometimes the other way round so I think I need to know when it's a good moment to read from IRC and when there's some data pending on Telegram. How to determine that in this example? This is the error I'm trying to avoid:
RuntimeError: readuntil() called while another coroutine is already waiting for incoming data
So, I'm implementing a Discord Bot using discord.py, and I'm trying to dynamically call functions based on commands. I was able to test dynamic function calls with exec() fine, but they seem to fall apart with the async calls needed for discord.py.
So what I'm trying to do with this example would be to call the hello function and print Hello World into discord by typing !hello in chat.
#client.event
async def on_message(message):
call = 'await ' + message.content.lower()[1:] + '(message)'
exec(call)
async def hello(message):
await client.send_message(message.channel, 'Hello World')
Unfortunately, this code doesn't seem to do anything, I'm assuming because of how exec() handles async calls. Any help would be appreciated.
Instead of exec() use globals() to get your function:
import asyncio
async def main():
s = "foo"
param = "hello"
coro = globals().get(s)
if coro:
result = await coro(param)
print("got:", result)
else:
print("unknown function:", s)
async def foo(param):
print(param)
await asyncio.sleep(0.11)
return ":-)"
loop = asyncio.get_event_loop()
response = loop.run_until_complete(main())
loop.close()
However, allowing the user to access anything in globals() might bwe dangerous, instead it would be much better to whitelist your commands, for example using:
import asyncio
my_commands = {}
def register(cmd):
my_commands[cmd.__name__] = cmd
return cmd
async def main():
s = "foo"
param = "hello"
coro = my_commands.get(s)
if coro:
result = await coro(param)
print("got:", result)
else:
print("unknown function:", s)
#register
async def foo(param):
"""I am the mighty foo command!"""
print(param)
await asyncio.sleep(0.11)
return ":-)"
loop = asyncio.get_event_loop()
response = loop.run_until_complete(main())
loop.close()
See also:
for k, v in my_commands.items():
print("{}: {}".format(k, v.__doc__ or "no docs"))