How do I write a websockets server in Python which just pushes out data on a timed interval to all connected clients without waiting for any incoming messages ?
I'm answering my own question ...
This is a working example of a Python websockets server which sends out a message to all clients every 5 seconds. I wrote this and managed to get it working as I could not find a single example of this on the web (March 2021)
Hope this helps others, and if anyone has suggestions for improvements or better solutions using packages to maybe add ssl support or subscription type services additions, please write in the comments or answers section.
import asyncio
import logging
import websockets
from websockets import WebSocketServerProtocol
import time
import threading
logging.basicConfig(level=logging.INFO)
class Server:
clients = set()
logging.info(f'starting up ...')
def __init__(self):
logging.info(f'init happened ...')
async def register(self, ws: WebSocketServerProtocol) -> None:
self.clients.add(ws)
logging.info(f'{ws.remote_address} connects')
async def unregister(self, ws: WebSocketServerProtocol) -> None:
self.clients.remove(ws)
logging.info(f'{ws.remote_address} disconnects')
async def send_to_clients(self, message: str) -> None:
if self.clients:
logging.info("trying to send")
await asyncio.wait([client.send(message) for client in self.clients])
async def ws_handler(self, ws: WebSocketServerProtocol, url: str) -> None:
await self.register(ws)
try:
await self.distribute(ws)
finally:
await self.unregister(ws)
async def distribute(self, ws: WebSocketServerProtocol) -> None:
async for message in ws:
await self.send_to_clients(message)
async def timerThread(server,counter):
counter = 0
while True:
await checkAndSend(server,counter)
print("doing " + str(counter))
time.sleep(5)
counter = counter + 1
async def checkAndSend(server,counter):
# check something
# send message
logging.info("in check and send")
await server.send_to_clients("Hi there: " + str(counter))
# helper routine to allow thread to call async function
def between_callback(server,counter):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(timerThread(server,counter))
loop.close()
# start server
server = Server()
start_server = websockets.serve(server.ws_handler,'localhost',4000)
counter = 0
# start timer thread
threading.Thread(target=between_callback,args=(server,counter,)).start()
# start main event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
To see this working, you can use this simple html file as the client and then open the inspector to see the incoming messages on the Console log.
<h1> Websocket Test </h1>
<script>
const ws = new WebSocket('ws://localhost:4000')
ws.onopen = () => {
console.log('ws opened on browser')
ws.send('hello world')
}
ws.onmessage = (message) => {
console.log(`message received`, message.data)
}
</script>
from datetime import time
import schedule
import time
import socket
import threading
alarm_time = input()
my_string = str(alarm_time)
my_new_time = my_string.format("%H:%M %Z")
EADER = 64
PORT = xxxx
SERVER = 'xxxxxxxx'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
if alarm_time is not None and alarm_time != 0:
print(f"Scheduled for: {my_new_time}")
else:
sys.exit()
def job():
client.connect(ADDR)
name_time = self.name_send + ' ' + my_date
message = name_time.encode(FORMAT)
msg_length = len(message)
send_length = str(msg_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
client.send(send_length)
client.send(message)
schedule.every().day.at(my_new_time).do(job)
while True:
schedule.run_pending()
time.sleep(1)
Not sure if this is any help tbh. But its a small tweak of a script I use with socket, subprocess etc to schedule them
Related
So I have been trying to experiment with Streams in Python and wrote the following code.
ServiceSubscription.py
class ServiceSubscription():
def __init__(self) -> None:
self.subscriber_connections = []
self.service_connections = []
self.server_listener = None
# Dictionary of service readers where key is the name of the service and the value is the reader for the service
self.service_readers = {}
"""
Create the listening server on port 7777
"""
async def initiate_server(self):
server = await asyncio.start_server(self.handle_incoming, '127.0.0.1', 7777)
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
print(f'Serving on {addrs}')
async with server:
await server.serve_forever()
"""
Handle the incoming connection based on whether the connection is from a service or suscriber
The first message sent should include either 'service:SERVICE_NAME' or 'suscriber: [SERVICE1, SERVICE2, ...]'
"""
async def handle_incoming(self, reader: StreamReader, writer: StreamWriter):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f"Received {message!r} from {addr!r}")
if ("Service:" in f"{message!r}"):
message = message[0:7]
self.service_connections.append(Connections(reader, writer, message))
service_reader = ServiceReader(reader=reader, writer=writer)
self.service_readers[message] = (service_reader)
await service_reader.broadcast()
elif ("Suscriber:" in f"{message!r}"):
message = message[0:9]
self.subscriber_connections.append(Connections(reader, writer, message))
self.service_readers[message].add_suscribers(writer)
else:
pass
class ServiceReader():
def __init__(self, reader: StreamReader, writer: StreamWriter):
self.reader = reader
self.writer = writer
self.suscribers: Writer = []
self.loop = asyncio.get_event_loop()
def stop(self):
self._stop.set()
"""
Add new subscriber's StreamWriter here
"""
def add_suscribers(self, writer: StreamWriter):
# Not sure if this will work
self.suscribers.append(writer)
"""
Read data and broadcast it to subscribed clients
"""
async def broadcast(self):
while not self.reader.at_eof():
data = await self.reader.readline()
if b'\n' in data:
print(True)
data = data.decode()
print(data)
WriterTest.py
import asyncio
from os import linesep
async def tcp_echo_client(message):
reader, writer = await asyncio.open_connection(
'127.0.0.1', 7777)
print(f'Send: {message!r}\n')
writer.write(message.encode())
await writer.drain()
while not writer.is_closing():
data = input("Type a message\n")
data = (data + "\n").encode()
writer.write(data)
await writer.drain()
writer.close()
asyncio.run(tcp_echo_client('Service: TEST'))
I ran both python ServiceSubscription.py and python WriterTest.py at the same time to simulate a client and server.
Upon running ServiceSubscription.py, it will print "Serving on ('127.0.0.1', 7777)". When WriterTest.py is executed, ServiceSubscription.py will print "Received 'Service: TEST' from ('127.0.0.1', 39923)". However, typing anything beyond that will not be printed out until WriterTest.py's connection is closed. When the connection is closed, ServiceSubcription.py prints out the remaining bytes in the buffer and also confirms that there are newlines in the data read but it is not picked up by readline as it doesn't return after encountering a newline.
The problem is here, in your WriterTest:
data = input("Type a message\n")
The input function is blocking, so in an asyncio program it blocks the event loop. All tasks are stopped until you enter something. With asyncio streams, the actual transmission of the bytes occurs in another task. Your call to the input function blocks that task, which prevents the transmission. Your server doesn't respond because nothing is actually sent.
Since this is just a test program, you have a couple of quick solutions. You could put this line:
await asyncio.sleep(1.0)
after the line await writer.drain(). This will keep the other tasks running for one second, plenty of time for the data to get transmitted.
You could, of course, replace the call to input with some hard-coded string.
Better solutions can be found at the following link:
Listen to keypress with asyncio
As a general rule, input and asyncio do not play well together.
Data is encoded alongside a break line and a couple of variables are unpacked as reader and writer objects, then meanwhile is_closing() is waiting for a closing process, that should be close(), you could try to set a conditional when message have no characters.
import asyncio
import sys
from os import linesep
async def tcp_echo_client(message):
reader, writer = await asyncio.open_connection(
'127.0.0.1', 7777)
print(f'Send: {message!r}\n')
writer.write(message.encode())
await writer.drain()
while not writer.is_closing():
await asyncio.get_event_loop().run_in_executor(None, lambda s="Type a message\n": sys.stdout.write(s+' '))
data = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline)
if len(data)==0: # empty data
break
data = (data + "\n").encode()
writer.write(data)
await writer.drain()
print('Close the connection')
writer.close()
await writer.wait_closed()
here I change while conditional to check instead if data is empty and use a different corountine from asyncio
async def broadcast(self):
while True:
data = asyncio.wait_for(self.reader.readline(), timeout=10.0)
if data is None or len(data.decode()) == 0:
print("Expected message, received None")
break
data = data.decode().rstrip().upper()
print(data)
I have started to work with Asyncio recently and I was wondering if that is possible thing to do.
I made a TCP Echo server that get "commands" from client in order to execute methods with binance.client (which is stored on external file called GetData), I've also made a client that connects to binance websocket to receive live updates.
The problem is I want to combine the TCP Server and the Client that connects to binance websocket so when the server is running it would be able to handle the streaming data and still let the server handle new client connections.
Server Code:
import asyncio
from GetData import *
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
recivedData = data.decode()
print('Data received: {!r}'.format(recivedData))
commands = recivedData.split(' ')
if commands[0] == 'ping':
result = 'pong!'
print(result)
elif commands[0] == 'buy' and commands[1].upper() in get_all_assets():
symbol = commands[1].upper()
result = f'Buying request for {symbol}'
try:
current_status = buy_asset(symbol, commands[2])
result = f"You bought: {current_status['asset_amount']} {symbol}, Timestamp: {current_status['time']}"
print(result)
# Save on firebase >>
except IndexError:
print("Please mention usdt amount.")
else:
result = 'invalid command!'
print(result)
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def main():
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerProtocol, '127.0.0.1', 8070)
print("Server is now running")
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit")
finally:
server.close()
loop.close()
if __name__ == "__main__":
asyncio.run(main())
Client code:
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from GetData import *
import os
import json
class BinanceMarketStream(WebSocketClientProtocol):
def onConnect(self, response):
print("Server connected: {0}".format(response.peer))
self.df_historicalData = get_historical_kline('BTCUSDT', '1m')
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
data = payload.decode('utf8')
os.system('cls' if os.name == 'nt' else 'clear')
json_message = json.loads(data)
candle = json_message['k']
is_candle_closed = candle['x']
currentCandle = set_to_df(candle)
self.df_historicalData = update_currentCandle(
self.df_historicalData, currentCandle)
if is_candle_closed:
newCandle = set_to_df(candle)
self.df_historicalData = append_newcandle(
self.df_historicalData, newCandle)
# Trading Algorithms goes to here...
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import asyncio
factory = WebSocketClientFactory(
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m")
factory.protocol = BinanceMarketStream
loop = asyncio.get_event_loop()
coro = loop.create_connection(
factory, "stream.binance.com", 9443, ssl=True)
loop.run_until_complete(coro)
loop.run_forever()
loop.close()
I can also add the client that connects to the TCP server if needed.
I am trying to receive messages via websocket-client module and be able to use received messages for other purposes (e.g. execute buy/sell orders based on incoming messages).
Here is what I have so far:
import websocket
import time
import json
def on_message(ws, message):
try:
current_price = json.loads(message)
print(current_price["price"]) # data type is dict.. only showing values for the key 'price'
except:
print("Please wait..")
time.sleep(1)
def on_error(ws, error):
print(error)
def on_close(ws):
print("### closed ###")
def on_open(ws):
sub_params = {'type': 'subscribe', 'product_ids': ['BTC-USD'], 'channels': ['ticker']}
ws.send(json.dumps(sub_params))
if __name__ == "__main__":
websocket.enableTrace(False)
ws = websocket.WebSocketApp("wss://ws-feed.pro.coinbase.com/",
on_open = on_open,
on_message = on_message,
on_error = on_error,
on_close = on_close)
ws.run_forever()
Running this code will print the current Bitcoin price (current_price) as they come in through its websocket feed.
What I want to do next is to be able to access that variable current_price outside of the websocket function, and I am having a difficulty here. Writing anything beyond ws.run_forever() will be ignored because the websocket event loop will never end.
So I tried running the websocket on a separate thread with 'threading' mordule:
import websocket
import json
import threading
current_price = 0
def on_message(ws, message):
global current_price
current_price = message
def on_error(ws, error):
print(error)
def on_close(ws):
print("### closed ###")
def on_open(ws):
sub_params = {'type': 'subscribe', 'product_ids': ['BTC-USD'], 'channels': ['ticker']}
ws.send(json.dumps(sub_params))
if __name__ == "__main__":
websocket.enableTrace(False)
ws = websocket.WebSocketApp("wss://ws-feed.pro.coinbase.com/",
on_open = on_open,
on_message = on_message,
on_error = on_error,
on_close = on_close)
ws_thread = threading.Thread(target = ws.run_forever)
ws_thread.start()
print(current_price)
and this returns 0. What can I do to make this work?
Not sure if this is the most appropriate answer, but found a way to make this work.
import queue
.
.
.
.
def on_message(ws, message):
current_price = message
q.put(current_price)
.
.
.
ws_thread.start()
while True:
print(q.get())
The key to is is to use functools partials to wrap the run_forever thread's callback functions. I've built a simple multithreaded demonstrator:
from websocket import WebSocketApp
import time
from threading import Thread
from queue import Queue
from functools import partial
class websocket_client:
def __init__(self):
self.responseQ = Queue() #our message response Q
#add the response Qs on by wrapping the callbacks in functools partials
self.websocket = WebSocketApp("ws://echo.websocket.events",
on_open = partial(self.on_open, responseQ=self.responseQ),
on_message = partial(self.on_message, responseQ=self.responseQ),
on_error = partial(self.on_error, responseQ=self.responseQ),
on_close = partial(self.on_close, responseQ=self.responseQ))
# start run_forever as thread. request keep alive pings every 10 seconds with 5 second timouts
self.controller = Thread(target=self.websocket.run_forever, args=(None, None, 10, 5))
self.controller.start()
def on_open(self, websocket, responseQ):
#print("Opening Connection (run_forever)")
responseQ.put("Connected") #transmit status information
def on_error(self, websocket, error, responseQ):
#print("Connection Error (run_forever):", error)
responseQ.put(error) # or error messages
def on_message(self, websocket, message, responseQ):
#print("Message Received (run_forever):", message)
responseQ.put(message)
def on_close(self, websocket, status, message, responseQ):
#print("Closing Connection (run_forever):", status, message)
responseQ.put({'status': message, 'message': message}) #send close status and message in a dictionary
if __name__ == '__main__':
client = websocket_client() # make the client
while client.responseQ.get() != "Connected": time.sleep(0.1) #block until connected
print("\nSERVER GREETING:", client.responseQ.get()) #greeting from server
while True:
message_to_send = time.strftime("%b %d %Y %H:%M:%S")
client.websocket.send(message_to_send)
message_received = client.responseQ.get()
print("Main Thread: SENT --> '%s' RECEIVED --> '%s'" %(message_to_send, message_received))
time.sleep(1)
So I have made some code that follows. It is suppose to let a server and a client communicate... but it doesn't work.
Can someone explain why, or better yet fix my code???
Server.
import time
import socket
from threading import Thread
global sS
sS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sS.bind(('', 2347))
sSconAddresses = []
sSconData = []
print(" Server : Running ... ")
sS.listen(10)
while True:
try:
cOn, aDr = sS.accept()
sSconAddresses.insert(0, str(aDr))
sSconData.insert(0, str(cOn))
time.sleep(0.3)
except:
time.sleep(0.1)
pass
def ConHandler():
for _ in sSconData:
PacketData = _.recv(700)
if not PacketData:
_.close()
else:
stringData = PacketData.decode('utf-8')
print(stringData)
sS.sendto(PacketData, _)
ConHandlerThread = Thread(target=ConHandler)
ConHandlerThread.daemon = True
ConHandlerThread.start()
Client.
import threading, time
import socket, sys
import os
global cS
cS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cS.connect(('PRIVATE', 2347))
Server = ('PRIVATE', 2347)
while True:
PacketData = input(" Client> ")
ByteData = PacketData.encode('utf-8')
cS.sendto(ByteData, Server)
It doesn't return any errors so I am confused why it doesn't work.
First of all, in your server-side code, you're having a while True before starting your thread, so it can't work.
Then, if you succeed starting your thread by moving the code, its for will see an empty list, so it will not loop, and just exit right here.
Starting from your code, here's something that works:
The client:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 2345))
while True:
packetdata = input(" Client> ")
bytedata = packetdata.encode()
client_socket.send(bytedata)
print(client_socket.recv(700).decode())
if __name__ == '__main__':
main()
The server:
import socket
from threading import Thread
from queue import Queue
def client_handler(client_socket):
while True:
data = client_socket.recv(700)
print(data)
client_socket.send("Server {}".format(data.decode()).encode())
def conn_handler(conn_queue):
while True:
conn, address = conn_queue.get()
print("Handler getting a connection from {}".format(address))
client_thread = Thread(target=client_handler, args=(conn,))
client_thread.daemon = True
client_thread.start()
def main():
print("Server: Running ...")
conn_queue = Queue()
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('', 2345))
server_socket.listen(10)
con_handler_thread = Thread(target=conn_handler, args=(conn_queue,))
con_handler_thread.daemon = True
con_handler_thread.start()
while True:
conn_queue.put(server_socket.accept())
if __name__ == '__main__':
main()
Note that this is suboptimal, starting one thread per client is not the way to go. The best way to handle this kind of situation is to keep everything in a single thread and use a select-like function to know what to do. However select is a bit limited too (like 1024 connections max, hardcoded in the libc), so the way to go is to use epoll / kqueue / whatever better than poll / select, and there's a module for this: https://docs.python.org/3/library/select.html
Yet using the select module is still the old, manual, cubersome way to express your needs, you should take a look at the coroutine based API of asyncio which enable a clear way to express the intention.
The asyncio equivalent may look like:
import asyncio
async def client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
while True:
message = input("Client> ")
writer.write(message.encode())
data = await reader.read(100)
print('Received: {}'.format(data.decode()))
loop = asyncio.get_event_loop()
loop.run_until_complete(client())
And, server side:
import asyncio
async def handle_client(reader, writer):
while True:
data = await reader.read(100)
if not data:
return
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_client, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()
I have a websocket server (python 3.x) taking requests where each is a url variable. It runs just fine except it only executes each request in serial, after one another. While the function is running it also blocks the client(s) trying to connect. Non-blocking is what i want!
Asyncronous multiprocessed threading of both websocket and subprocess function.
The ability to set the number of cores to use. This is not obligatory though.
Here's what i've got:
ANSWER (illustration and asyncio.subprocess in accepted answer)
So, I didn't get very far with this frustration. I reverted back to my original code and as it turns out, you need to sleep the function with await asyncio.sleep(.001). Now it runs perfectly fine, I tested with multiple clients at the same time and it handles it asynchronously.
import asyncio, websockets, json
async def handler(websocket, path):
print("New client connected.")
await websocket.send('CONNECTED')
try:
while True:
inbound = await websocket.recv()
if inbound is None:
break
while inbound != None:
import time
for line in range(10):
time.sleep(1)
data = {}
data['blah'] = line
await asyncio.sleep(.000001) # THIS
print(data)
await websocket.send(json.dumps(data))
await websocket.send(json.dumps({'progress': 'DONE'}))
break
except websockets.exceptions.ConnectionClosed:
print("Client disconnected.")
if __name__ == "__main__":
server = websockets.serve(handler, '0.0.0.0', 8080)
loop = asyncio.get_event_loop()
loop.run_until_complete(server)
loop.run_forever()
Update: as suggested by #udi, if you want a slow external process, the way to go is asyncio.subprocess and not subprocess. Reading from pipe with a blocking call stalls the other threads, which is what asyncio.subprocess takes care of.
time.sleep() is blocking.
Try:
# blocking_server.py
import asyncio
import time
import websockets
x = 0
async def handler(websocket, path):
global x
x += 1
client_id = x
try:
print("[#{}] Connected.".format(client_id))
n = int(await websocket.recv())
print("[#{}] Got: {}".format(client_id, n))
for i in range(1, n + 1):
print("[#{}] zzz...".format(client_id))
time.sleep(1)
print("[#{}] woke up!".format(client_id))
await asyncio.sleep(.001)
msg = "*" * i
print("[#{}] sending: {}".format(client_id, msg))
await websocket.send(msg)
msg = "bye!"
print("[#{}] sending: {}".format(client_id, msg))
await websocket.send(msg)
print("[#{}] Done.".format(client_id, msg))
except websockets.exceptions.ConnectionClosed:
print("[#{}] Disconnected!.".format(client_id))
if __name__ == "__main__":
port = 8080
server = websockets.serve(handler, '0.0.0.0', port)
print("Started server on port {}".format(port))
loop = asyncio.get_event_loop()
loop.run_until_complete(server)
loop.run_forever()
With this test client:
# test_client.py
import asyncio
import time
import websockets
async def client(client_id, n):
t0 = time.time()
async with websockets.connect('ws://localhost:8080') as websocket:
print("[#{}] > {}".format(client_id, n))
await websocket.send(str(n))
while True:
resp = await websocket.recv()
print("[#{}] < {}".format(client_id, resp))
if resp == "bye!":
break
print("[#{}] Done in {:.2f} seconds".format(client_id, time.time() - t0))
tasks = [client(i + 1, 3) for i in range(4)]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
Now compare the result when time.sleep(x) is replaced with await asyncio.sleep(x)!
If you need to run a slow external process via asyncio, try asynico.subprocess:
An example external program:
# I am `slow_writer.py`
import sys
import time
n = int(sys.argv[1])
for i in range(1, n + 1):
time.sleep(1)
print("*" * i)
with this server:
# nonblocking_server.py
import asyncio
import sys
import websockets
x = 0
async def handler(websocket, path):
global x
x += 1
client_id = x
try:
print("[#{}] Connected.".format(client_id))
n = int(await websocket.recv())
print("[#{}] Got: {}. Running subprocess..".format(client_id, n))
cmd = (sys.executable, 'slow_writer.py', str(n))
proc = await asyncio.create_subprocess_exec(
*cmd, stdout=asyncio.subprocess.PIPE)
async for data in proc.stdout:
print("[#{}] got from subprocess, sending: {}".format(
client_id, data))
await websocket.send(data.decode().strip())
return_value = await proc.wait()
print("[#{}] Subprocess done.".format(client_id))
msg = "bye!"
print("[#{}] sending: {}".format(client_id, msg))
await websocket.send(msg)
print("[#{}] Done.".format(client_id, msg))
except websockets.exceptions.ConnectionClosed:
print("[#{}] Disconnected!.".format(client_id))
if __name__ == "__main__":
if sys.platform == 'win32':
loop = asyncio.ProactorEventLoop()
asyncio.set_event_loop(loop)
port = 8080
server = websockets.serve(handler, '0.0.0.0', port)
print("Started server on port {}".format(port))
loop = asyncio.get_event_loop()
loop.run_until_complete(server)
loop.run_forever()