Develop Reference

Python Asyncio Echo TCP server connected to a websocket client

I have started to work with Asyncio recently and I was wondering if that is possible thing to do.
I made a TCP Echo server that get "commands" from client in order to execute methods with binance.client (which is stored on external file called GetData), I've also made a client that connects to binance websocket to receive live updates.
The problem is I want to combine the TCP Server and the Client that connects to binance websocket so when the server is running it would be able to handle the streaming data and still let the server handle new client connections.
Server Code:
import asyncio
from GetData import *
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
recivedData = data.decode()
print('Data received: {!r}'.format(recivedData))
commands = recivedData.split(' ')
if commands[0] == 'ping':
result = 'pong!'
print(result)
elif commands[0] == 'buy' and commands[1].upper() in get_all_assets():
symbol = commands[1].upper()
result = f'Buying request for {symbol}'
try:
current_status = buy_asset(symbol, commands[2])
result = f"You bought: {current_status['asset_amount']} {symbol}, Timestamp: {current_status['time']}"
print(result)
# Save on firebase >>
except IndexError:
print("Please mention usdt amount.")
else:
result = 'invalid command!'
print(result)
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def main():
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerProtocol, '127.0.0.1', 8070)
print("Server is now running")
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit")
finally:
server.close()
loop.close()
if __name__ == "__main__":
asyncio.run(main())
Client code:
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from GetData import *
import os
import json
class BinanceMarketStream(WebSocketClientProtocol):
def onConnect(self, response):
print("Server connected: {0}".format(response.peer))
self.df_historicalData = get_historical_kline('BTCUSDT', '1m')
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
data = payload.decode('utf8')
os.system('cls' if os.name == 'nt' else 'clear')
json_message = json.loads(data)
candle = json_message['k']
is_candle_closed = candle['x']
currentCandle = set_to_df(candle)
self.df_historicalData = update_currentCandle(
self.df_historicalData, currentCandle)
if is_candle_closed:
newCandle = set_to_df(candle)
self.df_historicalData = append_newcandle(
self.df_historicalData, newCandle)
# Trading Algorithms goes to here...
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import asyncio
factory = WebSocketClientFactory(
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m")
factory.protocol = BinanceMarketStream
loop = asyncio.get_event_loop()
coro = loop.create_connection(
factory, "stream.binance.com", 9443, ssl=True)
loop.run_until_complete(coro)
loop.run_forever()
loop.close()
I can also add the client that connects to the TCP server if needed.

Websockets Server Push in Python

How do I write a websockets server in Python which just pushes out data on a timed interval to all connected clients without waiting for any incoming messages ?

I'm answering my own question ...
This is a working example of a Python websockets server which sends out a message to all clients every 5 seconds. I wrote this and managed to get it working as I could not find a single example of this on the web (March 2021)
Hope this helps others, and if anyone has suggestions for improvements or better solutions using packages to maybe add ssl support or subscription type services additions, please write in the comments or answers section.
import asyncio
import logging
import websockets
from websockets import WebSocketServerProtocol
import time
import threading
logging.basicConfig(level=logging.INFO)
class Server:
clients = set()
logging.info(f'starting up ...')
def __init__(self):
logging.info(f'init happened ...')
async def register(self, ws: WebSocketServerProtocol) -> None:
self.clients.add(ws)
logging.info(f'{ws.remote_address} connects')
async def unregister(self, ws: WebSocketServerProtocol) -> None:
self.clients.remove(ws)
logging.info(f'{ws.remote_address} disconnects')
async def send_to_clients(self, message: str) -> None:
if self.clients:
logging.info("trying to send")
await asyncio.wait([client.send(message) for client in self.clients])
async def ws_handler(self, ws: WebSocketServerProtocol, url: str) -> None:
await self.register(ws)
try:
await self.distribute(ws)
finally:
await self.unregister(ws)
async def distribute(self, ws: WebSocketServerProtocol) -> None:
async for message in ws:
await self.send_to_clients(message)
async def timerThread(server,counter):
counter = 0
while True:
await checkAndSend(server,counter)
print("doing " + str(counter))
time.sleep(5)
counter = counter + 1
async def checkAndSend(server,counter):
# check something
# send message
logging.info("in check and send")
await server.send_to_clients("Hi there: " + str(counter))
# helper routine to allow thread to call async function
def between_callback(server,counter):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(timerThread(server,counter))
loop.close()
# start server
server = Server()
start_server = websockets.serve(server.ws_handler,'localhost',4000)
counter = 0
# start timer thread
threading.Thread(target=between_callback,args=(server,counter,)).start()
# start main event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
To see this working, you can use this simple html file as the client and then open the inspector to see the incoming messages on the Console log.
<h1> Websocket Test </h1>
<script>
const ws = new WebSocket('ws://localhost:4000')
ws.onopen = () => {
console.log('ws opened on browser')
ws.send('hello world')
}
ws.onmessage = (message) => {
console.log(`message received`, message.data)
}
</script>

from datetime import time
import schedule
import time
import socket
import threading
alarm_time = input()
my_string = str(alarm_time)
my_new_time = my_string.format("%H:%M %Z")
EADER = 64
PORT = xxxx
SERVER = 'xxxxxxxx'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
if alarm_time is not None and alarm_time != 0:
print(f"Scheduled for: {my_new_time}")
else:
sys.exit()
def job():
client.connect(ADDR)
name_time = self.name_send + ' ' + my_date
message = name_time.encode(FORMAT)
msg_length = len(message)
send_length = str(msg_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
client.send(send_length)
client.send(message)
schedule.every().day.at(my_new_time).do(job)
while True:
schedule.run_pending()
time.sleep(1)
Not sure if this is any help tbh. But its a small tweak of a script I use with socket, subprocess etc to schedule them

Interprocess communication using asyncio?

I have a set of CPU-intensive processes that once in a while depend on each other to proceed. So something like
def run():
while True:
do stuff
wake up some other process
wait for some other process to wake me up
do stuff
Within each process I'd like to use async, so that I can always have an instance of run running while others are waiting to be woken up. Looking at the asyncio docs, the only IPC option in the "High-level APIs" section that I see uses sockets. I'd much rather use a pipe, which it looks like I can perhaps do with the low-level API, but that documentation is chock full of warnings that if you're just writing an application then it's a mistake to be using it. Can someone weigh in on the idiomatic thing to do here? (And also, speed is an important factor, so if there's some less-idiomatic-but-more-performant thing I'd like to know about that option as well.)

I would like to mention the aioprocessing library, as I successfully used it in one of my projects. It provides an anync interface to the multiprocessing primitives including IPC, such as Process, Pipe, Lock, Queue and etc. It uses thread pool to do this:
...
#staticmethod
def coro_maker(func):
def coro_func(self, *args, loop=None, **kwargs):
return self.run_in_executor(
getattr(self, func), *args, loop=loop, **kwargs
)
return coro_func
But to be honest, a lot depends on the problem being solved, on what tasks are being performed concurrently, since the intensive IPC itself within the async approach is less effective than the synchronous approach due to overhead of event loop, thread pool and etc. Sometimes it is better to make all IPC operations synchronous and put it all in a separate thread. Again, it all depends on the problem and the environment. Below is a benchmark that is far from comprehensive, but it can give an approximate picture of the problem that is being solved in it (intensive exchange of buffers).
note: I wrote about the difference between a Queue and SimpleQueue here
Sync SimpleQueue: 1.4309470653533936
AioSimpleQueue: 12.32670259475708
AioQueue: 14.342737436294556
AioPipe: 11.747064590454102
subprocess pipe stream: 7.344956159591675
socket stream: 4.360717058181763
# main.py
import sys
import time
import asyncio
import aioprocessing as ap
import multiprocessing as mp
import proc
count = 5*10**4
data = b'*'*100
async def sync_simple_queue_func():
out_ = mp.SimpleQueue()
in_ = mp.SimpleQueue()
p = ap.AioProcess(target=proc.start_sync_queue_func, args=(out_, in_))
p.start()
begin_ts = time.time()
for i in range(count):
out_.put(data)
res = in_.get()
print('Sync SimpleQueue: ', time.time() - begin_ts)
out_.put(None)
async def simple_queue_func():
out_ = ap.AioSimpleQueue()
in_ = ap.AioSimpleQueue()
p = ap.AioProcess(target=proc.start_queue_func, args=(out_, in_))
p.start()
begin_ts = time.time()
for i in range(count):
await out_.coro_put(data)
res = await in_.coro_get()
print('AioSimpleQueue: ', time.time() - begin_ts)
await out_.coro_put(None)
async def queue_func():
out_ = ap.AioQueue()
in_ = ap.AioQueue()
p = ap.AioProcess(target=proc.start_queue_func, args=(out_, in_))
p.start()
begin_ts = time.time()
for i in range(count):
await out_.coro_put(data)
res = await in_.coro_get()
print('AioQueue: ', time.time() - begin_ts)
await out_.coro_put(None)
async def pipe_func():
main_, child_ = ap.AioPipe()
p = ap.AioProcess(target=proc.start_pipe_func, args=(child_,))
p.start()
begin_ts = time.time()
for i in range(count):
await main_.coro_send(data)
res = await main_.coro_recv()
print('AioPipe: ', time.time() - begin_ts)
await main_.coro_send(None)
await p.coro_join()
server = None
async def handle_child(reader, writer):
begin_ts = time.time()
for i in range(count):
writer.write(data)
res = await reader.read(len(data))
print('socket stream: ', time.time() - begin_ts)
writer.close()
async def socket_func():
global server
addr = ('127.0.0.1', 8888)
server = await asyncio.start_server(handle_child, *addr)
p = ap.AioProcess(target=proc.start_socket_func, args=(addr,))
p.start()
async with server:
await server.serve_forever()
async def subprocess_func():
prog = await asyncio.create_subprocess_shell(
'python proc.py',
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE)
begin_ts = time.time()
for i in range(count):
prog.stdin.write(data)
res = await prog.stdout.read(len(data))
print('subprocess pipe stream: ', time.time() - begin_ts)
prog.stdin.close()
async def main():
await sync_simple_queue_func()
await simple_queue_func()
await queue_func()
await pipe_func()
await subprocess_func()
await socket_func()
asyncio.run(main())
# proc.py
import asyncio
import sys
import aioprocessing as ap
async def sync_queue_func(in_, out_):
while True:
n = in_.get()
if n is None:
return
out_.put(n)
async def queue_func(in_, out_):
while True:
n = await in_.coro_get()
if n is None:
return
await out_.coro_put(n)
async def pipe_func(child):
while True:
n = await child.coro_recv()
if n is None:
return
await child.coro_send(n)
data = b'*' * 100
async def socket_func(addr):
reader, writer = await asyncio.open_connection(*addr)
while True:
n = await reader.read(len(data))
if not n:
break
writer.write(n)
def start_sync_queue_func(in_, out_):
asyncio.run(sync_queue_func(in_, out_))
def start_queue_func(in_, out_):
asyncio.run(queue_func(in_, out_))
def start_pipe_func(child):
asyncio.run(pipe_func(child))
def start_socket_func(addr):
asyncio.run(socket_func(addr))
async def connect_stdin_stdout():
loop = asyncio.get_event_loop()
reader = asyncio.StreamReader()
protocol = asyncio.StreamReaderProtocol(reader)
dummy = asyncio.Protocol()
await loop.connect_read_pipe(lambda: protocol, sys.stdin) # sets read_transport
w_transport, _ = await loop.connect_write_pipe(lambda: dummy, sys.stdout)
writer = asyncio.StreamWriter(w_transport, protocol, reader, loop)
return reader, writer
async def main():
reader, writer = await connect_stdin_stdout()
while True:
res = await reader.read(len(data))
if not res:
break
writer.write(res)
if __name__ == "__main__":
asyncio.run(main())

python asynchronous http request

I have some equipment with http interface that frequently generate infinite http page with values I want to parse and save to the database.
I started with requests:
import asyncio
import asyncpg
import requests
class node_http_mtr():
def __init__(self, ip, nsrc, ndst):
self.ip = ip
self.nsrc = nsrc
self.ndst = ndst
try:
self.data = requests.get('http://' + self.ip + '/nph-cgi_mtr?duration=-1&interval=0', stream=True, timeout=10)
except:
return
def __iter__(self):
return self
def __next__(self):
mtr = list()
try:
for chunk in self.data.iter_content(32 * (self.nsrc + self.ndst), '\n'):
# DEBUG log chunk
for line in chunk.split('\n'):
# DEBUG log line
if line.startswith('MTR'):
try:
_, io, num, val = line.split(' ')
l, r = val.split(':')[1], val.split(':')[2]
mtr.append((self.ip, io+num, l, r))
except:
# ERROR log line
pass
if len(mtr) == self.nsrc + self.ndst:
break
if len(mtr) == self.nsrc + self.ndst:
yield mtr
else:
continue
except:
# ERROR connection lost
return
async def save_to_db(data_to_save):
global pool
try:
async with pool.acquire() as conn:
await conn.execute('''INSERT INTO mtr (ip, io, l, r) VALUES %s''' % ','.join(str(row) for row in data_to_save))
finally:
await pool.release(conn)
async def remove_from_db(ip):
global pool
try:
async with pool.acquire() as conn:
await conn.execute('''DELETE FROM httpmtr WHERE ip = $1''', ip)
finally:
await pool.release(conn)
async def http_mtr_worker():
global workers_list
global loop
while True:
await asyncio.sleep(0)
for ip in list(workers_list):
data_to_save = next(workers_list[ip])
if data_to_save:
asyncio.ensure_future(save_to_db(next(data_to_save)))
await asyncio.sleep(0)
async def check_for_workers():
global workers_list
global pool
while True:
await asyncio.sleep(0)
try:
async with pool.acquire() as conn:
workers = await conn.fetch('''SELECT ip FROM httpmtr''')
finally:
await pool.release(conn)
for worker in workers:
if worker['ip'] not in list(workers_list):
workers_list[worker['ip']] = node_http_mtr(worker['ip'], 8, 8)
await asyncio.sleep(0)
print('Add worker', worker['ip'])
await asyncio.sleep(0)
ips_to_delete = set(workers_list.keys()) - set([i[0] for i in workers])
if len(ips_to_delete) != 0:
for ip in ips_to_delete:
print('Delete worker ', ip)
workers_list.pop(ip)
await asyncio.sleep(0)
async def make_db_connection():
pool = await asyncpg.create_pool(user='postgres', password='data', database='test', host='localhost', max_queries=50000, command_timeout=60)
return pool
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
pool = loop.run_until_complete(make_db_connection())
workers_list = {}
try:
asyncio.ensure_future(check_for_workers())
asyncio.ensure_future(http_mtr_worker())
loop.run_forever()
except Exception as e:
print(e)
pass
finally:
print("Closing Loop")
loop.close()
I have triggered procedure in DB which deletes all data older then 1 second, the final result with one worker in PostgreSQL is:
test=# select count(*) from mtr;
count
-------
384
(1 ёЄЁюър)
It means 384 results per second. There are 16 different kinds of data in each device, so I have 384/16 = 24 values per second. It's appropriate result. But the more workers I add the worse performance I have: with 10 workers I have 2-3 times less values. The goal is to have hundreds of workers and 24-25 values/sec.
Next I tried to do is to use aiohttp. I expected to get much better result. Hastily I wrote test code:
import asyncio
from aiohttp import ClientSession
import asyncpg
async def parse(line):
if line.startswith('MTR'):
_, io, num, val = line.split(' ')
l, r = val.split(':')[1], val.split(':')[2]
return ('ip.will.be.here', io + num, l, r)
async def run():
url = "http://10.150.20.130/nph-cgi_mtr?duration=-1&interval=0"
async with ClientSession() as session:
while True:
async with session.get(url) as response:
buffer = b''
start = False
async for line in response.content.iter_any():
if line.startswith(b'\n'):
start = True
buffer += line
elif start and line.endswith(b'\n'):
buffer += line
mtr = [await parse(line) for line in buffer.decode().split('\n')[1:-1]]
await save_to_db(mtr)
break
elif start:
buffer += line
async def make_db_connection():
pool = await asyncpg.create_pool(user='postgres', password='data', database='test', host='localhost', max_queries=50000, command_timeout=60)
return pool
async def save_to_db(data_to_save):
global pool
try:
async with pool.acquire() as conn:
await conn.execute('''INSERT INTO mtr (ip, io, l, r) VALUES %s''' % ','.join(str(row) for row in data_to_save))
finally:
await pool.release(conn)
loop = asyncio.get_event_loop()
pool = loop.run_until_complete(make_db_connection())
future = asyncio.ensure_future(run())
loop.run_until_complete(future)
And I've got this:
test=# select count(*) from mtr;
count
-------
80
(1 ёЄЁюър)
i.e. I've gotten 5 time worse performance with asynchronous requests. I'm stuck. I don't understand how to solve it.
UPDATE. Profiling didn't make the situation more clear at all.
requests:
aiohttp:
With requests the situation is more or less clear. But what the problem with async aiohttp I don't understand at all.
UPDATE 16/05/18. Finally I came back to multithreading and I got what I need - constant performance with big amount of workers. Asynchronous calls is not a panacea indeed.

Why won't this python socket chat server work correctly?

So I have made some code that follows. It is suppose to let a server and a client communicate... but it doesn't work.
Can someone explain why, or better yet fix my code???
Server.
import time
import socket
from threading import Thread
global sS
sS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sS.bind(('', 2347))
sSconAddresses = []
sSconData = []
print(" Server : Running ... ")
sS.listen(10)
while True:
try:
cOn, aDr = sS.accept()
sSconAddresses.insert(0, str(aDr))
sSconData.insert(0, str(cOn))
time.sleep(0.3)
except:
time.sleep(0.1)
pass
def ConHandler():
for _ in sSconData:
PacketData = _.recv(700)
if not PacketData:
_.close()
else:
stringData = PacketData.decode('utf-8')
print(stringData)
sS.sendto(PacketData, _)
ConHandlerThread = Thread(target=ConHandler)
ConHandlerThread.daemon = True
ConHandlerThread.start()
Client.
import threading, time
import socket, sys
import os
global cS
cS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cS.connect(('PRIVATE', 2347))
Server = ('PRIVATE', 2347)
while True:
PacketData = input(" Client> ")
ByteData = PacketData.encode('utf-8')
cS.sendto(ByteData, Server)
It doesn't return any errors so I am confused why it doesn't work.

First of all, in your server-side code, you're having a while True before starting your thread, so it can't work.
Then, if you succeed starting your thread by moving the code, its for will see an empty list, so it will not loop, and just exit right here.
Starting from your code, here's something that works:
The client:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 2345))
while True:
packetdata = input(" Client> ")
bytedata = packetdata.encode()
client_socket.send(bytedata)
print(client_socket.recv(700).decode())
if __name__ == '__main__':
main()
The server:
import socket
from threading import Thread
from queue import Queue
def client_handler(client_socket):
while True:
data = client_socket.recv(700)
print(data)
client_socket.send("Server {}".format(data.decode()).encode())
def conn_handler(conn_queue):
while True:
conn, address = conn_queue.get()
print("Handler getting a connection from {}".format(address))
client_thread = Thread(target=client_handler, args=(conn,))
client_thread.daemon = True
client_thread.start()
def main():
print("Server: Running ...")
conn_queue = Queue()
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('', 2345))
server_socket.listen(10)
con_handler_thread = Thread(target=conn_handler, args=(conn_queue,))
con_handler_thread.daemon = True
con_handler_thread.start()
while True:
conn_queue.put(server_socket.accept())
if __name__ == '__main__':
main()
Note that this is suboptimal, starting one thread per client is not the way to go. The best way to handle this kind of situation is to keep everything in a single thread and use a select-like function to know what to do. However select is a bit limited too (like 1024 connections max, hardcoded in the libc), so the way to go is to use epoll / kqueue / whatever better than poll / select, and there's a module for this: https://docs.python.org/3/library/select.html
Yet using the select module is still the old, manual, cubersome way to express your needs, you should take a look at the coroutine based API of asyncio which enable a clear way to express the intention.
The asyncio equivalent may look like:
import asyncio
async def client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
while True:
message = input("Client> ")
writer.write(message.encode())
data = await reader.read(100)
print('Received: {}'.format(data.decode()))
loop = asyncio.get_event_loop()
loop.run_until_complete(client())
And, server side:
import asyncio
async def handle_client(reader, writer):
while True:
data = await reader.read(100)
if not data:
return
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_client, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()