Using below code I'm attempting to start 2 infinite loops using asyncio:
async def do_job_1():
while True :
print('do_job_1')
await asyncio.sleep(5)
async def do_job_2():
while True :
print('do_job_2')
await asyncio.sleep(5)
if __name__ == '__main__':
asyncio.run(do_job_1())
asyncio.run(do_job_2())
do_job_1 blocks do_job_2, as do_job_2 never prints do_job_1. What mistake have I made ?
Ultimately I'm trying to convert kafka consumer code:
from confluent_kafka import Consumer, KafkaError
settings = {
'bootstrap.servers': 'localhost:9092',
'group.id': 'mygroup',
'client.id': 'client-1',
'enable.auto.commit': True,
'session.timeout.ms': 6000,
'default.topic.config': {'auto.offset.reset': 'smallest'}
}
c = Consumer(settings)
c.subscribe(['mytopic'])
try:
while True:
msg = c.poll(0.1)
if msg is None:
continue
elif not msg.error():
print('Received message: {0}'.format(msg.value()))
elif msg.error().code() == KafkaError._PARTITION_EOF:
print('End of partition reached {0}/{1}'
.format(msg.topic(), msg.partition()))
else:
print('Error occured: {0}'.format(msg.error().str()))
except KeyboardInterrupt:
pass
finally:
c.close()
taken from https://www.confluent.io/blog/introduction-to-apache-kafka-for-python-programmers to be concurrent so I can parallelize processing of Kafka messages.
From help(asyncio.run):
It should be used as a main entry point for asyncio programs, and should ideally only be called once.
But you can use asyncio.gather to join the tasks:
import asyncio
async def do_job_1():
while True :
print('do_job_1')
await asyncio.sleep(5)
async def do_job_2():
while True :
print('do_job_2')
await asyncio.sleep(5)
async def main():
await asyncio.gather(do_job_1(), do_job_2())
if __name__ == '__main__':
asyncio.run(main())
Related
I have a list of asyncio tasks which contains with connecting,handshaking and receiving data from a websocket. This process is running correctly but sometimes the connection of one of the websocket (or maybe all of them) is closed.
How can I detect and make a new conncetion to the closed one?
Here is the code which I use:
async def main(_id):
try:
async with websockets.connect("wss://ws.bitpin.ir/", extra_headers = request_header, timeout=10, ping_interval=None) as websocket:
await websocket.send('{"method":"sub_to_price_info"}')
recv_msg = await websocket.recv()
if recv_msg == '{"message": "sub to price info"}':
await websocket.send(json.dumps({"method":"sub_to_market","id":_id}))
recv_msg = await websocket.recv()
print(recv_msg)
counter = 1
task = asyncio.create_task(ping_func(websocket))
while True:
msg = await websocket.recv()
return_func(msg, counter, asyncio.current_task().get_name()) ## Return recieved message
counter+=1
except Exception as e:
err_log(name='Error in main function', text=str(e))
async def ping_func(websocket):
try:
while True:
await websocket.send('{"message":"PING"}')
# print('------ ping')
await asyncio.sleep(5)
except Exception as e:
err_log(name='Error in ping function', text=str(e))
def return_func(msg, counter, task_name):
if msg != '{"message": "PONG"}' and len(json.loads(msg)) <20:
print(task_name, counter, msg[:100])
else:
print(task_name, counter)
async def handler():
try:
tasks = []
for _id in symbols_id_dict.values():
tasks.append(asyncio.create_task(main(_id), name='task{}'.format(_id)))
responses = await asyncio.gather(*tasks)
except Exception as e:
err_log(name='Error in handler function', text=str(e))
try:
if __name__ == '__main__':
asyncio.run(handler())
else:
os._exit(0)
except Exception as e:
err_log(name='Error in running asyncio handler', text=str(e))
finally:
os._exit(0)
According to the below line, each task is specified with a name:
tasks.append(asyncio.create_task(main(_id), name='task{}'.format(_id)))
So each task can be detected. How can I use this feature to detect closed websocket.
try:
data = await ws.recv()
except (ConnectionClosed):
print("Connection is Closed")
data = None
print('Reconnecting')
websocket = await websockets.connect(params)
So I'm making some minigame using discord.py, and this is what I got:
asyncio.create_task(self.stream_message_timer(ctx, correct, total), name=f"stream message timer {ctx.author.id}")
while bad_count > 0:
done, pending = await asyncio.wait([
self.client.wait_for('reaction_add', check=lambda reaction, user: str(reaction.emoji) in number_emojis and reaction.message.id == discord_message.id and user == ctx.author),
self.client.wait_for('message', check=lambda m: m.author == self.client.user and m.content == f"Times up!\n{correct}/{total} tasks successful.")
], return_when=asyncio.FIRST_COMPLETED)
try:
result = done.pop().result()
except Exception as e:
raise e
for future in done:
future.exception()
for future in pending:
future.cancel()
if type(result) == discord.Message:
return False
else:
reaction = result
# process the reaction, edit a message
await ctx.send(f"You deleted all the bad messages!\n{correct+1}/{total} tasks successful.")
for task in asyncio.all_tasks():
if task.get_name() == f"stream message timer {ctx.author.id}":
task.cancel()
break
return True
async def stream_message_timer(self, ctx, correct, total):
await asyncio.sleep(5)
await ctx.send(f"Times up!\n{correct}/{total} tasks successful.") # message linked to delete_chat, change there if this is changed
return False
Basically, I'm trying to make some kind of 5 second background timer as I wait for reactions at the same time.
No, I am not looking for timeout=5
The code that I have here works, but its very hacky. I'm either waiting for a reaction from the user, or just waiting for the bot to message "Times up".
Is there a cleaner way to do this? I would like to have the timer start right before the while loop, and forcefully stop anything inside the loop and make the function return False right when the timer finishes
Also, if the function does stop, I still want some way to cancel the timer, and that timer only.
I've been using this method for quite some time and it's making my code very unorganized. Thanks.
Here’s some sort of example that is independent from discord.py:
import asyncio
import random
async def main():
asyncio.create_task(timer(), name="some task name")
# simulates waiting for user input
await asyncio.sleep(random.uniform(2, 5))
return True
async def timer():
await asyncio.sleep(5)
# somehow make this return statement stop the main() function
return False
asyncio.run(main())
wait(return_when=FIRST_COMPLETED) is the correct way to do it, but you can simplify the invocation (and subsequent cancellation) by using the return value of create_task:
async def main():
timer_task = asyncio.create_task(timer())
user_input_task = asyncio.create_task(asyncio.sleep(random.uniform(2, 5)))
await asyncio.wait([timer_task, user_input_task], return_when=asyncio.FIRST_COMPLETED)
if not timer_task.done():
timer_task.cancel()
if user_input_task.done():
# we've finished the user input
result = await user_input_task
...
else:
# we've timed out
await timer_task # to propagate exceptions, if any
...
If this pattern repeats a lot in your code base, you can easily abstract it into a utility function.
I am making a discord bot using discord.py. I want to make a command to purge all messages inside a channel every 100 seconds. Here is my code:
autodeletetime = -100
autodeletelasttime = 1
#client.command()
#commands.has_permissions(manage_messages=True)
async def autodelete(ctx):
global autodeleterunning
if autodeleterunning == False:
autodeleterunning = True
asgas = True
while asgas:
message = await ctx.send(f'All messages gonna be deleted in 100 seconds')
await message.pin()
for c in range(autodeletetime,autodeletelasttime):
okfe = abs(c)
await message.edit(content=f"All messages gonna be deleted in {okfe} seconds" )
await asyncio.sleep(1)
if c == 0:
await ctx.channel.purge(limit=9999999999999999999999999999999999999999999999999999999999999999999)
await time.sleep(1)
autodeleterunning = False
else:
await ctx.send(f'The autodelete command is already running in this server')
I want the loop to restart every 100 seconds after the purge is complete.
You should use tasks instead of commands for these kind of commands.
import discord
from discord.ext import commands, tasks
import asyncio
#tasks.loop(seconds=100)
async def autopurge(channel):
message = await channel.send(f'All messages gonna be deleted in 100 seconds')
await message.pin()
try:
await channel.purge(limit=1000)
except:
await channel.send("I could not purge messages!")
#client.group(invoke_without_command=True)
#commands.has_permissions(manage_messages=True)
async def autopurge(ctx):
await ctx.send("Please use `autopurge start` to start the loop.")
# Start the loop
#autopurge.command()
async def start(ctx):
task = autopurge.get_task()
if task and not task.done():
await ctx.send("Already running")
return
autopurge.start(ctx.channel)
# Stop the loop
#autopurge.command()
async def stop(ctx):
task = autopurge.get_task()
if task and not task.done():
autopurge.stop()
return
await ctx.send("Loop was not running")
I have a simple asyncio tcp chat server/client.
But there is an issue with, it seems to me, sending messages. Instead of receiving and sending messages in parallel, if I have 2 or more clients, they receive other messages only after they send theirs.
How to solve this problem?
import asyncio
list_of_users = {}
async def handle_echo(reader, writer):
name = await reader.read(1024)
name.decode()
print(name)
addr = writer
list_of_users[addr]
while True:
data = await reader.read(1024)
message = data.decode()
if not message:
del list_of_users[addr]
break
msg(message)
def msg(message):
for user in list_of_users:
print('send message - ', message)
user.write(message.encode())
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
async with server:
await server.serve_forever()
asyncio.run(main())
import asyncio
from aioconsole import ainput
async def tcp_echo_client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
name = input('Enter your name: ')
writer.write(name.encode())
while True:
await output_messages(writer)
await incoming_messages(reader)
async def output_messages(writer):
message = await ainput()
writer.write(message.encode())
async def incoming_messages(reader):
input_message = await reader.read(1024)
print('print incoming message', input_message)
async def main():
await tcp_echo_client()
asyncio.run(main())
The main issue was that the coroutine incoming_messages did not run in the background. Instead it was repeatedly called after output_messages finished.
I also had to make other adjustments to get the code running on Python 3.6
run incoming_messages in the background
list_of_users is now a list
no context handler for the server
no serve_forever
Try this code:
Server
import asyncio
list_of_users = []
async def handle_echo(reader, writer):
name = await reader.read(1024)
name.decode()
print(name)
addr = writer
list_of_users.append(addr)
while True:
data = await reader.read(1024)
message = data.decode()
if not message:
list_of_users.remove(addr)
break
msg(message)
def msg(message):
for user in list_of_users:
print('send message - ', message)
user.write(message.encode())
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addr = server.sockets[0].getsockname()
print(f'Serving on {addr}')
await asyncio.sleep(3600)
asyncio.ensure_future(main())
loop = asyncio.get_event_loop()
loop.run_forever()
Client
import asyncio
from aioconsole import ainput
async def tcp_echo_client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
name = input('Enter your name: ')
writer.write(name.encode())
asyncio.ensure_future(incoming_messages(reader))
while True:
await output_messages(writer)
async def output_messages(writer):
message = await ainput()
writer.write(message.encode())
async def incoming_messages(reader):
while True:
input_message = await reader.read(1024)
print('print incoming message', input_message)
async def main():
await tcp_echo_client()
asyncio.get_event_loop().run_until_complete(main())
Is there a way to use Websockets in Pyramid using Python 3.
I want to use it for live-updating tables when there are data changes on the server.
I already thought of using long-polling, but I don't think this is the best way.
Any comments or ideas?
https://github.com/housleyjk/aiopyramid works for me. See the documentation for websocket http://aiopyramid.readthedocs.io/features.html#websockets
UPD:
WebSocket server with pyramid environment.
import aiohttp
import asyncio
from aiohttp import web
from webtest import TestApp
from pyramid.config import Configurator
from pyramid.response import Response
async def websocket_handler(request):
ws = web.WebSocketResponse()
await ws.prepare(request)
while not ws.closed:
msg = await ws.receive()
if msg.tp == aiohttp.MsgType.text:
if msg.data == 'close':
await ws.close()
else:
hello = TestApp(request.app.pyramid).get('/')
ws.send_str(hello.text)
elif msg.tp == aiohttp.MsgType.close:
print('websocket connection closed')
elif msg.tp == aiohttp.MsgType.error:
print('ws connection closed with exception %s' %
ws.exception())
else:
ws.send_str('Hi')
return ws
def hello(request):
return Response('Hello world!')
async def init(loop):
app = web.Application(loop=loop)
app.router.add_route('GET', '/{name}', websocket_handler)
config = Configurator()
config.add_route('hello_world', '/')
config.add_view(hello, route_name='hello_world')
app.pyramid = config.make_wsgi_app()
srv = await loop.create_server(app.make_handler(),
'127.0.0.1', 8080)
print("Server started at http://127.0.0.1:8080")
return srv
loop = asyncio.get_event_loop()
loop.run_until_complete(init(loop))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
WebSocket client:
import asyncio
import aiohttp
session = aiohttp.ClientSession()
async def client():
ws = await session.ws_connect('http://0.0.0.0:8080/foo')
while True:
ws.send_str('Hi')
await asyncio.sleep(2)
msg = await ws.receive()
if msg.tp == aiohttp.MsgType.text:
print('MSG: ', msg)
if msg.data == 'close':
await ws.close()
break
else:
ws.send_str(msg.data + '/client')
elif msg.tp == aiohttp.MsgType.closed:
break
elif msg.tp == aiohttp.MsgType.error:
break
loop = asyncio.get_event_loop()
loop.run_until_complete(client())
loop.close()