Develop Reference

Python consume RabbitMQ and run SocketIO server

Setup
I have a python application, which should consume messages from a RabbitMQ and act as a SocketIO server to a Vue2 APP. When it receives messages from RabbitMQ it should send out a message over SocketIO to the Vue2 APP. Therefore I wrote 2 classes RabbitMQHandler and SocketIOHandler. I am starting the RabbitMQHandler in a separate thread so that both the RabbitMQ consume and the wsgi server can run in parallel.
Code
import random
import threading
import socketio
import eventlet
import sys
import os
import uuid
import pika
from dotenv import load_dotenv
import logging
class RabbitMQHandler():
def __init__(self, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
self.queue_name = 'myqueue'
self.exchange_name = 'myqueue'
credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PW)
self.connection = pika.BlockingConnection(pika.ConnectionParameters(RABBITMQ_IP, 5672, '/', credentials))
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.queue_name)
self.channel.exchange_declare(exchange=self.exchange_name, exchange_type='fanout')
self.channel.queue_bind(exchange=self.exchange_name, queue=self.queue_name)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.connection.close()
def run(self, callback):
logging.info('start consuming messages...')
self.channel.basic_consume(queue=self.queue_name,auto_ack=True, on_message_callback=callback)
self.channel.start_consuming()
class SocketIOHandler():
def __init__(self):
self.id = str(uuid.uuid4())
# create a Socket.IO server
self.sio = socketio.Server(async_mode='eventlet', cors_allowed_origins='*')
# wrap with a WSGI application
self.app = socketio.WSGIApp(self.sio)
self.sio.on('connect_to_backend', self.handle_connect)
self.sio.on('random_number', self.handle_random_number)
def handle_connect(self, sid, msg):
logging.info('new socket io message')
self.emit('connect_success', {
'success': True,
})
def handle_random_number(self, sid, msg):
logging.info('handle_random_number')
self.emit('response_random_number', { 'number': random.randint(0,10)})
def emit(self, event, msg):
logging.info('socket server: {}'.format(self.id))
logging.info('sending event: "{}"'.format(event))
self.sio.emit(event, msg)
logging.info('sent event: "{}"'.format(event))
def run(self):
logging.info('start web socket on port 8765...')
eventlet.wsgi.server(eventlet.listen(('', 8765)), self.app)
def start_rabbitmq_handler(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP):
def callback(ch, method, properties, body):
logging.info('rabbitmq handler')
socketio_handler.emit('response_random_number', { 'number': random.randint(0,10)})
with RabbitMQHandler(RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP) as rabbitmq_handler:
rabbitmq_handler.run(callback=callback)
threads = []
def main():
global threads
load_dotenv()
RABBITMQ_USER = os.getenv('RABBITMQ_USER')
RABBITMQ_PW = os.getenv('RABBITMQ_PW')
RABBITMQ_IP = os.getenv('RABBITMQ_IP')
socketio_handler = SocketIOHandler()
rabbitmq_thread = threading.Thread(target=start_rabbitmq_handler, args=(socketio_handler, RABBITMQ_USER, RABBITMQ_PW, RABBITMQ_IP))
threads.append(rabbitmq_thread)
rabbitmq_thread.start()
socketio_handler.run()
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO)
logging.getLogger("pika").propagate = False
main()
except KeyboardInterrupt:
try:
for t in threads:
t.exit()
sys.exit(0)
except SystemExit:
for t in threads:
t.exit()
os._exit(0)
Problem
The Problem is, that when the RabbitMQHandler receives a message the event response_random_number does not get through to the Vue2 APP. Even though it is emited in the callback function. When I send the random_number event from the Vue2 APP to the python application I do get the response_random_number event back from the python application in the Vue2 APP.
So all connections work on their own, but not together. My guess would be, that there is some sort of threading communication error. I added the id to the SocketIOHandler class to make sure it is the same instanced object and the prints are the same.
The logs 'socket server: ...', sending event: ... and sent event: ... tell me, that the function is being called correctly.

execute a specific method into sgx using gramine

I have an application which is using gRPC, client.py and server.py , I want to use gramine in order to execute the service inside SGX.
how can I run a specific method not the whole script inside sgx using gramine?
client.py:
"""The Python implementation of the GRPC helloworld.Greeter client."""
from __future__ import print_function
import logging
import grpc
import helloworld_pb2
import helloworld_pb2_grpc
def run():
# NOTE(gRPC Python Team): .close() is possible on a channel and should be
# used in circumstances in which the with statement does not fit the needs
# of the code.
print("Will try to greet world ...")
with grpc.insecure_channel('localhost:50051') as channel:
stub = helloworld_pb2_grpc.GreeterStub(channel)
response = stub.SayHello(helloworld_pb2.HelloRequest(name='you'))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
logging.basicConfig()
run()
and server.py:
from concurrent import futures
import logging
import grpc
import helloworld_pb2
import helloworld_pb2_grpc
class Greeter(helloworld_pb2_grpc.GreeterServicer):
def SayHello(self, request, context):
return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name)
def serve():
port = '50051'
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server)
server.add_insecure_port('[::]:' + port)
server.start()
print("Server started, listening on " + port)
server.wait_for_termination()
if __name__ == '__main__':
logging.basicConfig()
serve()
let say I want to execute sayhello inside SGX when I run client.py
currently I am running gramine-sgx ./python client.py that is going to execute only client inside SGX or is it going to also run sayhello from server.py inside SGX?

Why is loop.create_task taking so long to execute?

I am trying to emulate the reading of a NFC chip by pressing the cmd key in a little python script that acts as a socket-io server. It's job is to notify a client after the cmd key has been pressed. It works, but emulate_nfc_tag() takes ages to execute after i press cmd. I suspect it has to do with how create_task works. I hope someone can spot how i could optimize this code.
from aiohttp import web
import socketio
import asyncio
from pynput import keyboard
import random
import string
sio = socketio.AsyncServer(cors_allowed_origins='*',
async_mode='aiohttp')
app = web.Application()
sio.attach(app)
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))
async def emulate_nfc_tag():
print("sending out nfc id")
await sio.emit("nfc", id_generator())
def on_press(key):
if key == keyboard.Key.shift:
print("here we go")
loop.create_task(emulate_nfc_tag())
if key == keyboard.Key.esc:
# Stop listener
return False
async def index(request):
"""Serve the client-side application."""
with open('../client/dist/index.html') as f:
return web.Response(text=f.read(), content_type='text/html')
#sio.event
def connect(sid, environ):
print("connect ", sid)
#sio.event
def disconnect(sid):
print('disconnect ', sid)
#sio.event
async def msg(sid, data):
print('message ', data)
await sio.emit("msg", "Hi back")
app.router.add_static('/assets', '../client/dist/assets')
app.router.add_get('/', index)
if __name__ == '__main__':
# web.run_app(app)
loop = asyncio.get_event_loop()
listener = keyboard.Listener(on_press=on_press)
listener.start()
# set up aiohttp - like run_app, but non-blocking
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
site = web.TCPSite(runner)
loop.run_until_complete(site.start())
# add more stuff to the loop
loop.run_forever()

The handlers that you configure for your keyboard run on a background thread and not in the thread that holds your asyncio application. You are calling the create_task() function from this handler, which is invalid because this function must be called from the asyncio thread.
Instead, you may want to try with the run_coroutine_threadsafe(), which is specifically designed for your use case in which you want to start a coroutine in the asyncio thread, but doing it from another thread.

How to call Python Tornado Websocket Server inside another Python

I would like to implement Python Tornado Websocket Server inside another Python (main) and trigger send messages when needed. The main creates two threads. One of them is for Python Server and the other is for my loop that will trigger message.
When I start server from initial, server works fine however because its endless following main files doesn't run. So I start server inside a thread but this time I receive "RuntimeError: There is no current event loop in thread 'Thread-1 (start_server)'"
Main.py
import tornadoserver
import time
from threading import Lock, Thread
class Signal:
def __init__(self):
#self.socket = tornadoserver.initiate_server()
print("start")
def start_server(self):
print("start Server")
self.socket = tornadoserver.initiate_server()
def brd(self):
print("start Broad")
i = 0
while True:
time.sleep(3)
self.socket.send(i)
i = i + 1
def job(self):
# --------Main--------
threads = []
for func in [self.start_server, self.brd, ]:
threads.append(Thread(target=func))
threads[-1].start()
for thread in threads:
thread.join()
Signal().job()
tornadoserver.py
import tornado.web
import tornado.httpserver
import tornado.ioloop
import tornado.websocket as ws
from tornado.options import define, options
import time
define('port', default=4041, help='port to listen on')
ws_clients = []
class web_socket_handler(ws.WebSocketHandler):
#classmethod
def route_urls(cls):
return [(r'/', cls, {}), ]
def simple_init(self):
self.last = time.time()
self.stop = False
def open(self):
self.simple_init()
if self not in ws_clients:
ws_clients.append(self)
print("New client connected")
self.write_message("You are connected")
def on_message(self, message):
if self in ws_clients:
print("received message {}".format(message))
self.write_message("You said {}".format(message))
self.last = time.time()
def on_close(self):
if self in ws_clients:
ws_clients.remove(self)
print("connection is closed")
self.loop.stop()
def check_origin(self, origin):
return True
def send_message(self, message):
self.write_message("You said {}".format(message))
def send(message):
for c in ws_clients:
c.write_message(message)
def initiate_server():
# create a tornado application and provide the urls
app = tornado.web.Application(web_socket_handler.route_urls())
# setup the server
server = tornado.httpserver.HTTPServer(app)
server.listen(options.port)
# start io/event loop
tornado.ioloop.IOLoop.instance().start()

Using Google I found tornado issue
Starting server in separate thread gives... RuntimeError: There is no current event loop in thread 'Thread-4' · Issue #2308 · tornadoweb/tornado
and it shows that it has to use
asyncio.set_event_loop(asyncio.new_event_loop())
to run event loop in new thread
Something like this
import asyncio
# ...
def initiate_server():
asyncio.set_event_loop(asyncio.new_event_loop()) # <---
# create a tornado application and provide the urls
app = tornado.web.Application(web_socket_handler.route_urls())
# setup the server
server = tornado.httpserver.HTTPServer(app)
server.listen(options.port)
# start io/event loop
tornado.ioloop.IOLoop.instance().start()

How to get the client's IP address from python-socketio connection and still be able to emit from anywhere in the program

I have a Pyhton server that accepts SocketIO connections using the following pyhton-socketio GitHub project. I would like to get the clients IP address when they connect. How can I get the client's IP address?
import eventlet
eventlet.monkey_patch()
import threading
from flask import Flask, render_template
from flask_socketio import SocketIO
app = Flask(__name__)
socket = SocketIO(app, logger=True, engineio_logger=True)
class Server(threading.Thread):
def __init__(self, thread_id):
threading.Thread.__init__(self)
self.threadID = thread_id
def run(self):
print("Starting " + self.name)
serve()
print("Exiting " + self.name)
def serve():
if __name__ == '__main__':
eventlet.wsgi.server(eventlet.listen(('', 8000)), app)
#socket.on('connect')
def connect():
print("IP = ") # print out client IP address here
Is there some way I can get the IP using the SocketIO instance?
I see I can get the IP by doing the following:
import socketio
sio = socketio.Server()
#sio.on('connect')
def connect(sid, environ):
print('IP->' + environ['REMOTE_ADDR'])
But this then takes away emit functionality from anywhere in the program. This means that I can then only emit from inside the above connect() function.
Is there anyway I can get the client IP and still be able to emit from anywhere in the program?

Once you get the IP address, store it a global variable, then you can do anything with that variable anywhere else
for instance:
IP = "0.0.0.0"
def connect(sid, environ):
global IP
print('IP->' + environ['REMOTE_ADDR'])
IP = environ['REMOTE_ADDR']