I am trying to emulate the reading of a NFC chip by pressing the cmd key in a little python script that acts as a socket-io server. It's job is to notify a client after the cmd key has been pressed. It works, but emulate_nfc_tag() takes ages to execute after i press cmd. I suspect it has to do with how create_task works. I hope someone can spot how i could optimize this code.
from aiohttp import web
import socketio
import asyncio
from pynput import keyboard
import random
import string
sio = socketio.AsyncServer(cors_allowed_origins='*',
async_mode='aiohttp')
app = web.Application()
sio.attach(app)
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
async def emulate_nfc_tag():
print("sending out nfc id")
await sio.emit("nfc", id_generator())
def on_press(key):
if key == keyboard.Key.shift:
print("here we go")
loop.create_task(emulate_nfc_tag())
if key == keyboard.Key.esc:
# Stop listener
return False
async def index(request):
"""Serve the client-side application."""
with open('../client/dist/index.html') as f:
return web.Response(text=f.read(), content_type='text/html')
#sio.event
def connect(sid, environ):
print("connect ", sid)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
#sio.event
async def msg(sid, data):
print('message ', data)
await sio.emit("msg", "Hi back")
app.router.add_static('/assets', '../client/dist/assets')
app.router.add_get('/', index)
if __name__ == '__main__':
# web.run_app(app)
loop = asyncio.get_event_loop()
listener = keyboard.Listener(on_press=on_press)
listener.start()
# set up aiohttp - like run_app, but non-blocking
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
site = web.TCPSite(runner)
loop.run_until_complete(site.start())
# add more stuff to the loop
loop.run_forever()
The handlers that you configure for your keyboard run on a background thread and not in the thread that holds your asyncio application. You are calling the create_task() function from this handler, which is invalid because this function must be called from the asyncio thread.
Instead, you may want to try with the run_coroutine_threadsafe(), which is specifically designed for your use case in which you want to start a coroutine in the asyncio thread, but doing it from another thread.
Related
from aiohttp import web
async def handle(request):
name = request.match_info.get('name', "Anonymous")
text = "Hello, " + name
return web.Response(text=text)
app = web.Application()
app.add_routes([web.get('/', handle),
web.get('/{name}', handle)])
if __name__ == '__main__':
web.run_app(app)
The above example is from oficial documentation.
The question is how can i terminate the web.run_app(app) operations with no KeyBoardInterrupt (Ctrl+C)?
I am looking something like:
await app.shutdown()
await app.cleanup()
but i don't know where can i put this code and how can i use it.
From the source, it looks response for KeyboardInterrupt and GracefulExit:
try:
asyncio.set_event_loop(loop)
loop.run_until_complete(main_task)
except (GracefulExit, KeyboardInterrupt): # pragma: no cover
pass
finally:
_cancel_tasks({main_task}, loop)
_cancel_tasks(asyncio.all_tasks(loop), loop)
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
asyncio.set_event_loop(None)
As you don't want to stop in manual, you could raise GracefulExit in your own code, something like next:
test.py:
import signal
from aiohttp import web
from aiohttp.web_runner import GracefulExit
async def handle(request):
name = request.match_info.get('name', "Anonymous")
text = "Hello, " + name
return web.Response(text=text)
async def shutdown(request):
print("will shutdown now")
raise GracefulExit()
app = web.Application()
app.add_routes([web.get('/shutdown', shutdown),
web.get('/', handle),
web.get('/{name}', handle)])
if __name__ == '__main__':
web.run_app(app)
Once you want to close the server, you could send http://ip:8080/shutdown to server.
Additional, if you want to exit the server directly after receive a request, you may also use signal.alarm(3), then you no need to define a handler with /shutdown. This means send a alarm to program after 3 seconds after receive any request (aiohttp internal already register a signal handler with loop.add_signal_handler(signal.SIGINT, _raise_graceful_exit)):
async def handle(request):
name = request.match_info.get('name', "Anonymous")
text = "Hello, " + name
signal.alarm(3)
return web.Response(text=text)
Anyway, the way to close the server is either raise GracefulExit or send signal, but the timing to exit the server depends on your scenario.
Based on #atline answer i wrote this code:
import signal
from aiohttp import web
from aiohttp.web_runner import GracefulExit
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from multiprocessing import cpu_count
import requests
import time
async def handle(request):
name = request.match_info.get('name', "Anonymous")
text = "Hello, " + name
return web.Response(text=text)
async def shutdown(request):
print("will shutdown now")
raise GracefulExit()
app = web.Application()
app.add_routes([web.get('/shutdown', shutdown),
web.get('/', handle),
web.get('/{name}', handle)])
def init_aiohttp():
web.run_app(app, host="127.0.0.1", port="8080", ssl_context=None)
if __name__ == '__main__':
executor = ProcessPoolExecutor()
for i in range(0, 1):
aiohttp = executor.submit(init_aiohttp)
total_time = 0
while(True):
time.sleep(1)
total_time += 1
if total_time == 5:
try:
requests.get('http://127.0.0.1:8080/shutdown')
except Exception as e:
print(e)
print(total_time)
The above code:
Starts an aiohttp server.
Close the server after 5 second (fetching the shutdown url: http://127.0.0.1:8080/shutdown)
Hope that will help someone in the future.
I am running into some trouble with Azure Event Bub with Python. Below is my strater code for connection (Taken from microsoft docs)
import asyncio
from azure.eventhub.aio import EventHubConsumerClient
from azure.eventhub.extensions.checkpointstoreblobaio import BlobCheckpointStore
async def on_event(partition_context, event):
# Print the event data.
print("Received the event: \"{}\" from the partition with ID: \"{}\"".format(event.body_as_str(encoding='UTF-8'), partition_context.partition_id))
# Update the checkpoint so that the program doesn't read the events
# that it has already read when you run it next time.
await partition_context.update_checkpoint(event)
async def main():
# Create an Azure blob checkpoint store to store the checkpoints.
checkpoint_store = BlobCheckpointStore.from_connection_string("AZURE STORAGE CONNECTION STRING", "BLOB CONTAINER NAME")
# Create a consumer client for the event hub.
client = EventHubConsumerClient.from_connection_string("EVENT HUBS NAMESPACE CONNECTION STRING", consumer_group="$Default", eventhub_name="EVENT HUB NAME", checkpoint_store=checkpoint_store)
async with client:
# Call the receive method. Read from the beginning of the partition (starting_position: "-1")
await client.receive(on_event=on_event, starting_position="-1")
if __name__ == '__main__':
loop = asyncio.get_event_loop()
# Run the main method.
loop.run_until_complete(main())
Here, the receiver/consumer keeps listening. If I remove any of the awaits the consumer throws an error.
Does anyone know how to stop the consumer after running for some time like timeout).
#Abhishek
There are 2 options here :
You could stop listening when there is an inactivity for certain period time.
You could stop listening after fixed duration.
Have detailed both in below steps.
OPTION 1
You could use the max_wait_time parameter in order to stop listening in case there is no activity for certain time.
I did spin up a simple use case of the above. But you could optimize this further.
import asyncio
from azure.eventhub.aio import EventHubConsumerClient
event_hub_connection_str = '<CON_STR>'
eventhub_name = '<EventHub_NAME>'
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=eventhub_name # EventHub name should be specified if it doesn't show up in connection string.
)
#this event gets called when the message is received or Max_Wait_time is clocked
async def on_event(partition_context, event):
print(event) #Optional - to see output
#Checks whether there is any event returned None. None is returned when this event is called after the Max_Wait_time is crossed
if(event !=None):
print("Received the event: \"{}\" from the partition with ID: \"{}\"".format(event.body_as_str(encoding='UTF-8'), partition_context.partition_id))
#you can update other code like updating blob store
else:
print("Timeout is Hit")
#updating the
global receive
receive = False
async def close():
print("Closing the client.")
await consumer.close()
print("Closed")
async def main():
recv_task = asyncio.ensure_future(consumer.receive(on_event=on_event,max_wait_time=15))
while(True): # keep receiving for 3 seconds
await asyncio.sleep(3)
if(receive != True):
print("Cancelling the Task")
recv_task.cancel() # stop receiving by cancelling the task
break;
receive = True
asyncio.run(main())
asyncio.run(close())#closing the Client
With regards to the above code. If there is no activity 15 seconds the async task gets cancelled and the consumer clients gets closed. The program is eventually exited gracefully.
OPTION 2
If you are looking for a code in which the you would like to make client to listen for fixed time like 1 hour or some thing. You could refer the below code
Reference Code
event_hub_connection_str = '<>'
eventhub_name = '<>'
import asyncio
from azure.eventhub.aio import EventHubConsumerClient
consumer = EventHubConsumerClient.from_connection_string(
conn_str=event_hub_connection_str,
consumer_group='$Default',
eventhub_name=eventhub_name # EventHub name should be specified if it doesn't show up in connection string.
)
async def on_event(partition_context, event):
# Put your code here.
# If the operation is i/o intensive, async will have better performance.
print("Received event from partition: {}".format(partition_context.partition_id))
# The receive method is a coroutine which will be blocking when awaited.
# It can be executed in an async task for non-blocking behavior, and combined with the 'close' method.
async def main():
recv_task = asyncio.ensure_future(consumer.receive(on_event=on_event))
await asyncio.sleep(15) # keep receiving for 3 seconds
recv_task.cancel() # stop receiving
async def close():
print("Closing.....")
await consumer.close()
print("Closed")
asyncio.run(main())
asyncio.run(close())#closing the Client
The below code that is responsible for the client to be listening for a certain time :
recv_task =
asyncio.ensure_future(consumer.receive(on_event=on_event))
await asyncio.sleep(3) # keep receiving for 3 seconds
recv_task.cancel()
You could increase the time as per your need.
#Satya V I tried the option 2 but however I am seeing the error ,
There is no current event loop in thread 'MainThread'.
But However your code helped me in a better way . I had configured the code with Storage Account check point
import asyncio
import os
from azure.eventhub.aio import EventHubConsumerClient
from azure.eventhub.extensions.checkpointstoreblobaio import BlobCheckpointStore
CONNECTION_STR = ''
EVENTHUB_NAME = ''
STORAGE_CONNECTION_STR = ''
BLOB_CONTAINER_NAME = ""
async def on_event(partition_context, event):
print("Received event from partition: {}.".format(partition_context.partition_id))
await partition_context.update_checkpoint(event)
async def receive(client):
await client.receive(
on_event=on_event,
starting_position="-1", # "-1" is from the beginning of the partition.
)
async def main():
checkpoint_store = BlobCheckpointStore.from_connection_string(STORAGE_CONNECTION_STR, BLOB_CONTAINER_NAME)
client = EventHubConsumerClient.from_connection_string(
CONNECTION_STR,
consumer_group="$Default",
eventhub_name=EVENTHUB_NAME,
checkpoint_store=checkpoint_store, # For load-balancing and checkpoint. Leave None for no load-balancing.
)
async with client:
recv_task = asyncio.ensure_future(receive(client))
await asyncio.sleep(4) # keep receiving for 3 seconds
recv_task.cancel() # stop receiving
await client.close()
async def close():
print("Closing.....")
print("Closed")
if __name__ == '__main__':
asyncio.run(main())
asyncio.run(close())#closing the Client
I am building a pure python application with flask socket-io. Currently, I am trying to have the server emit an event to a specific client and wait for the callback before moving on.
This is my Server.py
import socketio
import eventlet
sio = socketio.Server(async_handlers=False)
app = socketio.WSGIApp(sio)
#sio.event
def connect(sid, environ):
nm = None
def namee(name):
print(name) # this has the value and is trying to assign it to nm
nonlocal nm
nm = name
sio.emit('name_', "name plz", callback=namee)
print(nm) # this shouldn't be None, but it is
print(sid, "in lobby")
#sio.event
def disconnect(sid):
print('disconnect', sid)
if __name__ == '__main__':
eventlet.wsgi.server(eventlet.listen(('', 5000)), app)
And this is my client.py
import sys
import socketio
sio = socketio.Client()
#sio.event
def connect():
print("you have connected to the server")
#sio.event
def connect_error(data):
print("The connection failed!")
#sio.event
def disconnect():
print("You have left the server")
#sio.event
def name_(data):
print("name asked for")
return "test"
def main():
sio.connect('http://localhost:5000')
print('Your sid is', sio.sid)
if __name__ == '__main__':
main()
I tried using time.sleep() but that delayed the whole process. I also tried making a while loop
while nm is None:
pass
but that kicked the client off the server and a while later, the server crashed.
You can use the call() method instead of emit().
nm = sio.call('name_', "name plz")
print(nm)
I recommend that you move this logic outside of the connect handler though. This handler is just to accept or reject the connection, it is not supposed to block for a long period of time. Do this in a regular handler that you use after the connection has been established.
Goal is to allocate a thread and wait for the callback. Single thread is going to run the while loop forever. Difficulty here is that we are not directly calling or controlling the callback and we do not know in advance how long it will takes to the remote server to invoke callback.
I've tried to look for a solution in the asyncio module using asyncio.Future but unsuccessfully.
from a_module import Server # <a_module> is fictitious
import random
import time
class App(Server):
def __init__(self):
self.response = None
def send_requests(self):
"""Send request to remote server"""
self.send_number_to_server(42) # inherited from Server
# This is going to loop forever. We should "suspend" the
# current thread, allocate a new thread to wait for the
# callback and then comeback here to return the (not None)
# response.
while self.response is None:
# Wait for the callback before terminating this method.
time.sleep(1) # seconds
return self.response
def callback(self, message):
"""Inherited form parent class 'Server'. When the request sent
with App.send_req has been processed by the remote server,
this function is invoked in the background."""
self.response = message
if __name__ == '__main__':
app = App()
response = app.send_requests()
print(response)
Since callback is "invoked in the background", Server is presumably already running a background thread. In that case, you want your main thread to run the event loop, and the server's background thread to notify you when it is done. Assuming send_number_to_server is not blocking, you could do it like this:
class App(Server):
def __init__(self):
self._loop = asyncio.get_event_loop()
async def send_requests(self):
self.send_number_to_server(42)
self._future_resp = asyncio.Future()
resp = await self._future_resp
self._future_resp = None
return resp
def callback(self, message):
# called from a different thread
self._loop.call_soon_threadsafe(self._future_resp.set_result, message)
async def main():
app = App()
response = await app.send_requests()
print(response)
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())
I have a script running where the main thread takes input from stdin and then passes it to a child thread using a queue. In the child thread I'm using asyncio coroutines to spin up a listener on a socket and wait for connections. Once a connection is made I can now send data through the listener from the main thread.
It all seems to work well enough, but since asyncio.BaseEventLoop is not thread safe am I going to run into problems?
This is my attempt to solve the problem of using a blocking library like python's cmd module with asyncio.
My code is below.
import sys
import asyncio
from time import sleep
from threading import Thread
from queue import Queue
stdin_q = Queue()
clients = {} # task -> (reader, writer)
def client_connected_handler(client_reader, client_writer):
# Start a new asyncio.Task to handle this specific client connection
task = asyncio.Task(handle_client(client_reader, client_writer))
clients[task] = (client_reader, client_writer)
def client_done(task):
# When the tasks that handles the specific client connection is done
del clients[task]
# Add the client_done callback to be run when the future becomes done
task.add_done_callback(client_done)
#asyncio.coroutine
def handle_client(client_reader, client_writer):
# Handle the requests for a specific client with a line oriented protocol
while True:
cmd = yield from get_input()
client_writer.write(cmd.encode())
data = yield from client_reader.read(1024)
print(data.decode(),end="",flush=True)
#asyncio.coroutine
def get_input():
while True:
try:
return stdin_q.get()
except:
pass
class Control:
def start(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
self.loop = asyncio.get_event_loop()
server = self.loop.run_until_complete(asyncio.start_server(client_connected_handler, '0.0.0.0', 2222))
self.loop.run_forever()
self.stop()
def stop(self):
self.loop.stop()
self.loop.close()
def fire_control():
con = Control()
con.start()
if __name__ == "__main__":
stdin_q.put("\n")
t = Thread(target=fire_control)
t.start()
sleep(2)
_cmd = ""
while _cmd.lower() != "exit":
_cmd = input("")
if _cmd == "":
_cmd = "\r\n"
stdin_q.put(_cmd)
This isn't going to work quite right, because the call to stdin_q.get() is going to block your event loop. This means that if your server has multiple clients, all of them will be completely blocked by whichever one happens to get to stdin_q.get() first, until you send data into the queue. The simplest way to get around this is use BaseEvent.loop.run_in_executor to run the stdin_q.get in a background ThreadPoolExecutor, which allows you to wait for it without blocking the event loop:
#asyncio.coroutine
def get_input():
loop = asyncio.get_event_loop()
return (yield from loop.run_in_executor(None, stdin_q.get)) # None == use default executor.
Edit (1/27/16):
There is a library called janus, which provides an asyncio-friendly, thread-safe queue implementation.
Using that library, your code would look like this (I left out unchanged parts):
...
import janus
loop = asyncio.new_event_loop()
stdin_q = janus.Queue(loop=loop)
...
#asyncio.coroutine
def get_input():
loop = asyncio.get_event_loop()
return (yield from stdin_q.async_q.get())
class Control:
def start(self):
asyncio.set_event_loop(loop)
self.loop = asyncio.get_event_loop()
server = self.loop.run_until_complete(asyncio.start_server(client_connected_handler, '0.0.0.0', 2222))
self.loop.run_forever()
self.stop()
def stop(self):
self.loop.stop()
self.loop.close()
...
if __name__ == "__main__":
stdin_q.sync_q.put("\n")
t = Thread(target=runner)
t.start()
sleep(2)
_cmd = ""
while _cmd.lower() != "exit":
_cmd = input("")
if _cmd == "":
_cmd = "\r\n"
stdin_q.sync_q.put(_cmd)