I'm using websockets in an python project i'm working on. The websocket is being run in an thread and given 2 queue's from the parent thread. I'm using javascript to connect to the websocket server.
I'm able to get messages from the parent thread throughself.ssi.get(True) and passed them through to the javascript websocket client.
But i'm unable to receive messages from the client. When i use zaproxy i can see the messages going through. On the websocket server i'm also able to see the packets arrive on the interface. Python throws no error and logger.setLevel(logging.DEBUG) does not show messages arriving the same way as i'm able to see messages being send.
I've been trying to solve this but i've run out of ideas to find the problem, any help is welcome.
Python websocket server:
import websockets
import logging
import asyncio
import ssl
class websocket:
def __init__(self,ssi,sso):
self.ssi = ssi
self.sso = sso
logger = logging.getLogger('websockets')
logger.setLevel(logging.DEBUG)
# logger.addHandler(logging.FileHandler('debug.log'))
logger.addHandler(logging.StreamHandler())
sslc = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
sslc.load_cert_chain(
'keys/wss.server.crt',
'keys/wss.server.key')
loop = asyncio.new_event_loop()
wsrv = websockets.serve(
self.handler,
host='0.0.0.0',
port=9000,
ssl=sslc,
loop=loop)
loop.run_until_complete(wsrv)
loop.run_forever()
async def handler(self, wss, path):
consumer_task = asyncio.ensure_future(self.consumerHandler(wss, path))
producer_task = asyncio.ensure_future(self.producerHandler(wss, path))
done, pending = await asyncio.wait(
[consumer_task, producer_task],
return_when=asyncio.FIRST_COMPLETED,)
for task in pending:
task.cancel()
async def producerHandler(self, wss, path):
while True:
msg = await self.producer()
await wss.send(str(msg))
async def consumerHandler(self, wss, path):
async for msg in wss:
await self.consumer(msg)
async def producer(self):
return self.ssi.get(True)
async def consumer(self, msg):
self.sso.put(msg.data)
Javascript client:
var ws;
function ws_init() {
ws = new WebSocket("wss://pri.local:9000/");
ws.onopen = function(e) {
output("connected");
};
ws.onmessage = function(e) {
output("i: " + e.data);
};
ws.onclose = function() {
output("disconnect");
};
ws.onerror = function(e) {
output("onerror");
console.log(e)
};
}
function onSubmit() {
var input = document.getElementById("input");
ws.send(input.value);
output("o: " + input.value);
input.value = "";
input.focus();
}
function onCloseClick() {
ws.close();
}
function output(str) {
var log = document.getElementById("log");
var escaped = str.replace(/&/, "&").replace(/</, "<").
replace(/>/, ">").replace(/"/, """); // "
log.innerHTML = escaped + "<br>" + log.innerHTML;
}
I think the issue is that you're mixing the usage of the queue library and asyncio.queue.
queue is thread safe and so is a good mechanism for communicating between threads, but it doesn't have an async API, so you're blocking the websocket thread when you call self.ssi.get(True) which prevents any of the other websocket code from running.
asyncio.queue has the API you want (you can await queue.get()), but unfortunately isn't thread safe (it's designed for use within single threaded async applications).
You may be able to use loop.run_in_executor to await the blocking queue.get(True) call. See here for an example https://carlosmaniero.github.io/asyncio-handle-blocking-functions.html
Related
I'm trying to ingest data from an external WebSocket using Websocket Client A and send those messages to WebSocket Server B for clients connected to B to view. (Reason being, I'm trying to connect multiple external WebSockets and feed their messages to a single point, if this is a bad design can someone suggest an alternative)
Running server B uvicorn server:app --reload then visit http://127.0.0.1:8000/
WebSocket Server B
from typing import List
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse
app = FastAPI()
html = """
<!DOCTYPE html>
<html>
<head>
<title>Screener</title>
</head>
<body>
<h2>Your ID: <span id="ws-id"></span></h2>
<ul id='messages'>
</ul>
<script>
var client_id = Date.now()
document.querySelector("#ws-id").textContent = client_id;
var ws = new WebSocket(`ws://localhost:8000/ws/${client_id}`);
ws.onmessage = function(event) {
var messages = document.getElementById('messages')
var message = document.createElement('li')
var content = document.createTextNode(event.data)
message.appendChild(content)
messages.appendChild(message)
};
function sendMessage(event) {
var input = document.getElementById("messageText")
ws.send(input.value)
input.value = ''
event.preventDefault()
}
</script>
</body>
</html>
"""
class ConnectionManager:
def __init__(self):
self.active_connections: List[WebSocket] = []
async def connect(self, websocket: WebSocket):
await websocket.accept()
self.active_connections.append(websocket)
def disconnect(self, websocket: WebSocket):
self.active_connections.remove(websocket)
async def send_personal_message(self, message: str, websocket: WebSocket):
await websocket.send_text(message)
async def broadcast(self, message: str):
for connection in self.active_connections:
await connection.send_text(message)
manager = ConnectionManager()
#app.get("/")
async def get():
return HTMLResponse(html)
#app.websocket("/ws/{client_id}")
async def websocket_endpoint(websocket: WebSocket, client_id: int):
await manager.connect(websocket)
try:
while True:
data = await websocket.receive_text()
await manager.send_personal_message(f"You wrote: {data}", websocket)
await manager.broadcast(f"Client #{client_id} says: {data}")
except WebSocketDisconnect:
manager.disconnect(websocket)
await manager.broadcast(f"Client #{client_id} left the chat")
WebSocket ClientA
#!/usr/bin/env python
import websocket
import _thread
import time
import asyncio
import websockets
from concurrent.futures import ThreadPoolExecutor
_EXECUTOR_ = ThreadPoolExecutor(1)
async def main_loop():
async with websockets.connect("ws://localhost:8000/ws/1") as server_ws:
await server_ws.send("main_loop")
def send_message(message):
server_ws.send(message)
def ws_message(ws, message):
loop = asyncio.get_event_loop()
loop.run_in_executor(_EXECUTOR_, send_message, message)
print("WebSocket thread: %s" % message)
def ws_open(ws):
ws.send('{"event":"subscribe", "subscription":{"name":"trade"}, "pair":["XBT/USD","XRP/USD"]}')
def ws_thread(*args):
ws = websocket.WebSocketApp("wss://ws.kraken.com/", on_open = ws_open, on_message = ws_message)
ws.run_forever()
_thread.start_new_thread(ws_thread, ())
while True:
time.sleep(5)
print("Main thread: %d" % time.time())
asyncio.run(main_loop())
What is the question?
WS is kind of a multiplex connection that allows each side (server/client or whatever) to both send and receive messages in an asynchronous way. Ingesting data from one side of the WS (in your case Client A) and consuming some related data through another API is OK - it all depends on your needs.
This approach kind of reminds me of a classic Pub/Sub pattern. Where you have an API that you ingest data through and pass it to a Queue (RabbitMQ, Google Pub/Sub service/AWS SNS) the messages ingested into the queue are being consumed by another service that parses the data and populates some DB,
Then any other client can access a third service (API) to consume the relevant data.
Nice explanation about the Pub/Sub pattern Link1, Link2
Not sure what advantage WebSockets provide in your example
I want to make a websocket server using Fastapi then get a message from a manager then broadcast it to all clients.
The messages are just text.
Here is the process:
The manager connects to server.
The clients connect to server.
The manager send a message to server and server receive the message(could send OK status or response for error handling)
The server broadcast the message to all clients(no matter to manager or not)
The manager and clients are preferred to be MQL4/5 Experts.
A simple topology of the system:
The server is using fastapi sockets
FastAPI websocket
The simple server code like The manager of system:
from typing import List
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.responses import HTMLResponse
app = FastAPI()
class ConnectionManager:
def __init__(self):
self.active_connections: List[WebSocket] = []
async def connect(self, websocket: WebSocket):
await websocket.accept()
self.active_connections.append(websocket)
def disconnect(self, websocket: WebSocket):
self.active_connections.remove(websocket)
async def send_personal_message(self, message: str, websocket: WebSocket):
await websocket.send_text(message)
async def broadcast(self, message: str):
for connection in self.active_connections:
await connection.send_text(message)
manager = ConnectionManager()
#app.get("/")
async def get():
return (f"Hi!")
#app.websocket("/ws/{client_id}")
async def websocket_endpoint(websocket: WebSocket, client_id: int):
await manager.connect(websocket)
try:
while True:
data = await websocket.receive_text()
await manager.send_personal_message(f"You wrote: {data}", websocket)
await manager.broadcast(f"Client #{client_id} says: {data}")
except WebSocketDisconnect:
manager.disconnect(websocket)
await manager.broadcast(f"Hello everyone")
The connection manager in the above code can broadcast a message to all client(in that case, it is used for disconnection handling, but it is not my case).
Also, a sample piece of code for the client in MQL5:
int socket=SocketCreate();
if(socket!=INVALID_HANDLE)
{
if(SocketConnect(socket,"127.0.0.1",9091,1000)) {
Print("Connected to "," 127.0.0.1",":",9091);
string tosend = "heartbeat";
string received = socksend(socket, tosend) ? socketreceive(socket, 10) : "";
Print(received);
string sep="_"; // A separator as a character
ushort u_sep; // The code of the separator character
string result[]; // An array to get strings
u_sep=StringGetCharacter(sep,0);
//--- Split the string to substrings
int k=StringSplit(received,u_sep,result);
Print(result[0]);
Print(result[1]);
Print(result[2]);
}
}
//---
bool socksend(int sock,string request) {
char req[];
int len=StringToCharArray(request,req)-1;
if(len<0) return(false);
return(SocketSend(sock,req,len)==len);
}
//---
string socketreceive(int sock,int timeout) {
char rsp[];
string result="";
uint len;
uint timeout_check=GetTickCount()+timeout;
do {
len=SocketIsReadable(sock);
if(len) {
int rsp_len;
rsp_len=SocketRead(sock,rsp,len,timeout);
if(rsp_len>0) {
result+=CharArrayToString(rsp,0,rsp_len);
}
}
} while((GetTickCount()<timeout_check) && !IsStopped());
return result;
}
I want to know how can I connect the manager to system the get it's message and broadcast it.
I want to know how can I make the system.
I have set up an endpoint in Express.js that can start a long "for" loop and return a response before the loop finishes (thanks to Promises which push - I believe - the function to message queue):
const express = require('express')
const app = express()
const port = 3000
const long_task = () => {
for (let i=0; i<100000; i++) {
return new Promise(r => r(console.log(i)))
}
}
app.get('/', (req, res) => {
long_task()
res.send('endpoint returned')
})
app.listen(port, () => {
console.log(`Example app listening at http://localhost:${port}`)
})
(for now it does not matter that the server will not be able to process any other requests before the loop finishes).
Now, I want to mimic this in a Python framework, preferably FastAPI, due to its async capabilities.
So far I have this:
async def async_print(value):
print(value)
async def long_task():
for i in range(10000):
await async_print(i)
#app.get('/')
async def testing_long_task():
await long_task()
return 'endpoint returned'
But, as expected, the code will pause at await long_task(), and only return a response once the loop is finished.
And if I replace await long_task() with just long_task() I will get an error that the coroutine long_task was never awaited.
Is there any way to achieve such async for loop in FastAPI without using its background tasks or task queue like Celery?
I'm writing an update to my code to send a WebSocket message to a connected web browser that it needs to update its data (charting web app).
This message needs to be sent when the code has inserted new data into the MySQL database. I will write some Javascript in the browser to go and get the update on receiving the message.
My test code:
import asyncio
#import time
import websockets
def readValues():
'''do stuff that returns the values for database'''
pass
def inserdata(val):
'''insert values into mysql'''
pass
async def ph(websocket, path):
while True:
message = 'update'
# here we receive message that the data
# has been added and need to message the
# browser to update
print('socket executed')
await websocket.send(message)
await asyncio.sleep(2)
# shouldn't be needed as message
# sent only when updated data
# inserted(every 20s)
async def main(): # maybe use this to get/write to the database etc
while True: # instead of the loop at bottom
print('main executed')
await asyncio.sleep(20)
start_server = websockets.serve(ph, '0.0.0.0', 5678)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_until_complete(main())
asyncio.get_event_loop().run_forever()
#below copied from current program
'''
while 1:
try:
a = readValues() #read valves from a function
insertdata(a) #function to write values to mysql
#some method to send the message to the web browser via -
#websocket, that it needs to get the new data
time.sleep(20) #wait and then do it again
except Exception as e:
print(e)
'''
I can send a message using the message variable.
I need the readValues and insert data functions to run continuously every 20sec regardless of what's happening with the WebSocket.
But I can't work out how to send a message to the browser from the function that updates the database. And I can't work out the best method to run the WebSocket process and the updating of the database at the same time.
I've written comments in the code to try and help you understand what I'm trying to do.
Hope you can understand, thanks, Guys.
Update: Thanks Nathan:
I changed the code and do 2 files like the below:
Server:
import asyncio
import websockets
async def ph(websocket, path):
while True:
need_update = await websocket.recv()
print('socket executed')
await websocket.send(need_update)
start_server = websockets.serve(ph, '0.0.0.0', 5678)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
process file:
import asyncio
import time
import websockets
async def main():
async with websockets.connect('ws://127.0.0.1:5678') as websocket:
while 1:
try:
#a = readValues() #read values from a function
#insertdata(a) #function to write values to mysql
await websocket.send("updated")
print('data updated')
time.sleep(20) #wait and then do it again
except Exception as e:
print(e)
asyncio.get_event_loop().run_until_complete(main())
I then ran both of these (eaxctly as shown) and opened a web browser
with this:
<!DOCTYPE html>
<html>
<head>
</head>
<body>
<h3>
Test
</h3>
<p>
<div id="log"></div>
</p>
<script>
// helper function: log message to screen
function log(msg) {
document.getElementById('log').innerText += msg + '\n';
}
// setup websocket with callbacks
var ws = new WebSocket('ws://192.168.0.224:5678/');
ws.onopen = function() {
log('CONNECT');
};
ws.onclose = function() {
log('DISCONNECT');
};
ws.onmessage = function(event) {
log('MESSAGE: ' + event.data);
};
</script>
</body>
</html>
Everything seems fine until I open the browser as above.
Then nothing comes to the browser and apart from the 'connect' result.
WebSocket connection is closed: code = 1006 (connection closed abnormally [internal]), no reason
appears on both scripts.
You need a socket connexion between the "database handler" and the socket server :
create a second script with the main loop:
async def main():
async with websockets.connect(websocket_address) as websocket:
while 1:
try:
a = readValues() #read values from a function
insertdata(a) #function to write values to mysql
await websocket.send("some token to recognize that it's the db socket")
time.sleep(20) #wait and then do it again
except Exception as e:
print(e)
asyncio.get_event_loop().run_until_complete(main())
then on the other script you could have :
USERS = set()
def register(websocket):
USERS.add(websocket)
async def ph(websocket, path):
while True:
register(websocket) #not sure if you need to place it here
need_update = await websocket.recv()
#check unique token to verify that it's the database
message = 'update'#here we receive message that the data
#has been added and need to message the
#browser to update
print('socket executed')
if USERS: # asyncio.wait doesn't accept an empty list
await asyncio.wait([user.send(message) for user in USERS])
start_server = websockets.serve(ph, '0.0.0.0', 5678)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
I am trying to implement a simple web sockets server in Python by using this module. For learning purposes, the server should reply with a reversed version of what it received. For example, if the client sends "Hello Server", the server should respond with "revreS olleH". My code is based off the documentation here
Since an example of a consumer() and producer() function/coroutine wasn't provided in the documentation, I took a stab at creating them but think I am misunderstanding something not obvious to me. The code is currently returning the string 'nothing' instead of the reversed version of what the client sent.
FYI, since the machine I am using has Python 3.4.3, the code had to be adjusted to accommodate for that version. That's why you'll see newer code commented out, for now. Lots of documentation is included too as I learn this stuff.
Now, the codez...
index.py:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#########################
# Dependencies
#########################
# asyncio
# websockets
#########################
# Modules
#########################
import asyncio
import websockets
#########################
# Functions
#########################
# async indicates an asynchronous function.
# Calling them doesn't actually run them,
# but instead a coroutine object is returned,
# which can then be passed to the event loop to be executed later on.
# Python ≥ 3.5: async def producer(reply):
#asyncio.coroutine
def producer(reply=None):
"""Sends the reply to producer_handler."""
if reply is None:
return 'nothing'
else:
return reply
# Python ≥ 3.5: async def consumer(message):
#asyncio.coroutine
def consumer(message):
"""Reverses message then sends it to the producer."""
reply = message[::-1]
#await producer(reply)
yield from producer(reply)
# async def consumer_handler(websocket):
#asyncio.coroutine
def consumer_handler(websocket):
"""Handles incoming websocket messages."""
while True:
# await calls an asynchronous function.
#message = await websocket.recv()
message = yield from websocket.recv()
# Python ≥ 3.5: await consumer(message)
yield from consumer(message)
#async def producer_handler(websocket):
#asyncio.coroutine
def producer_handler(websocket):
"""Handles outgoing websocket messages."""
while True:
#message = await producer()
message = yield from producer()
#await websocket.send(message)
yield from websocket.send(message)
#async def handler(websocket, path):
#asyncio.coroutine
def handler(websocket, path):
"""Enables reading and writing messages on the same websocket connection."""
# A Future is an object that is supposed to have a result in the future.
# ensure_future:
# schedules the execution of a coroutine object,
# wraps it in a future, then returns a Task object.
# If the argument is a Future, it is returned directly.
# Python ≥ 3.5
#consumer_task = asyncio.ensure_future(consumer_handler(websocket))
#producer_task = asyncio.ensure_future(producer_handler(websocket))
consumer_task = asyncio.async(consumer_handler(websocket))
producer_task = asyncio.async(producer_handler(websocket))
# .wait:
# wait for the Futures and coroutine objects given
# by the sequence futures to complete. Coroutines will be
# wrapped in Tasks. Returns two sets of Future: (done, pending).
#done, pending = await asyncio.wait(
done, pending = yield from asyncio.wait(
# The futures.
[consumer_task, producer_task],
# FIRST_COMPLETED: the function will return when
# any future finishes or is cancelled.
return_when=asyncio.FIRST_COMPLETED,
)
for task in pending:
task.cancel()
#########################
# Start script
#########################
def main():
# Creates a WebSocket server.
start_server = websockets.serve(handler, '127.0.0.1', 8000)
# Get the event loop for the current context.
# Run until the Future is done.
asyncio.get_event_loop().run_until_complete(start_server)
# Run until stop() is called.
asyncio.get_event_loop().run_forever()
#########################
# Script entry point.
#########################
if __name__ == '__main__':
main()
index.html:
<!DOCTYPE html>
<html>
<head>
<title>WebSocket demo</title>
</head>
<body>
<script>
// Create the websocket.
var ws = new WebSocket("ws://127.0.0.1:8000/"),
messages = document.createElement('ul');
// Called when the websocket is opened.
ws.onopen = function(event) {
ws.send('Hello Server!');
};
// Called when a message is received from server.
ws.onmessage = function(event) {
var messages = document.getElementsByTagName('ul')[0],
message = document.createElement('li'),
content = document.createTextNode(event.data);
message.appendChild(content);
messages.appendChild(message);
};
document.body.appendChild(messages);
</script>
</body>
</html>
Not completely sure on this, but I think you misinterpreted the docs. The consumer shouldn't be calling the producer.
The "Hello Server!" the HTML file sends goes through consumer_handler to consumer to producer, but the yield from statements means that the reversed string ends up back in the consumer_handler, as the result of yield from consumer(message).
On the other hand, producer_handler calls producer many times without an argument (from message = yield from producer()), which is what creates the nothing that gets sent to the HTML file. It doesn't receive the consumer's string.
Instead, there should be a queue or something where the consumer pushes to and the producer takes from, like in this example.
Thanks.