I have started to work with Asyncio recently and I was wondering if that is possible thing to do.
I made a TCP Echo server that get "commands" from client in order to execute methods with binance.client (which is stored on external file called GetData), I've also made a client that connects to binance websocket to receive live updates.
The problem is I want to combine the TCP Server and the Client that connects to binance websocket so when the server is running it would be able to handle the streaming data and still let the server handle new client connections.
Server Code:
import asyncio
from GetData import *
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
recivedData = data.decode()
print('Data received: {!r}'.format(recivedData))
commands = recivedData.split(' ')
if commands[0] == 'ping':
result = 'pong!'
print(result)
elif commands[0] == 'buy' and commands[1].upper() in get_all_assets():
symbol = commands[1].upper()
result = f'Buying request for {symbol}'
try:
current_status = buy_asset(symbol, commands[2])
result = f"You bought: {current_status['asset_amount']} {symbol}, Timestamp: {current_status['time']}"
print(result)
# Save on firebase >>
except IndexError:
print("Please mention usdt amount.")
else:
result = 'invalid command!'
print(result)
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def main():
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerProtocol, '127.0.0.1', 8070)
print("Server is now running")
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit")
finally:
server.close()
loop.close()
if __name__ == "__main__":
asyncio.run(main())
Client code:
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from GetData import *
import os
import json
class BinanceMarketStream(WebSocketClientProtocol):
def onConnect(self, response):
print("Server connected: {0}".format(response.peer))
self.df_historicalData = get_historical_kline('BTCUSDT', '1m')
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
data = payload.decode('utf8')
os.system('cls' if os.name == 'nt' else 'clear')
json_message = json.loads(data)
candle = json_message['k']
is_candle_closed = candle['x']
currentCandle = set_to_df(candle)
self.df_historicalData = update_currentCandle(
self.df_historicalData, currentCandle)
if is_candle_closed:
newCandle = set_to_df(candle)
self.df_historicalData = append_newcandle(
self.df_historicalData, newCandle)
# Trading Algorithms goes to here...
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import asyncio
factory = WebSocketClientFactory(
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m")
factory.protocol = BinanceMarketStream
loop = asyncio.get_event_loop()
coro = loop.create_connection(
factory, "stream.binance.com", 9443, ssl=True)
loop.run_until_complete(coro)
loop.run_forever()
loop.close()
I can also add the client that connects to the TCP server if needed.
Related
How do I write a websockets server in Python which just pushes out data on a timed interval to all connected clients without waiting for any incoming messages ?
I'm answering my own question ...
This is a working example of a Python websockets server which sends out a message to all clients every 5 seconds. I wrote this and managed to get it working as I could not find a single example of this on the web (March 2021)
Hope this helps others, and if anyone has suggestions for improvements or better solutions using packages to maybe add ssl support or subscription type services additions, please write in the comments or answers section.
import asyncio
import logging
import websockets
from websockets import WebSocketServerProtocol
import time
import threading
logging.basicConfig(level=logging.INFO)
class Server:
clients = set()
logging.info(f'starting up ...')
def __init__(self):
logging.info(f'init happened ...')
async def register(self, ws: WebSocketServerProtocol) -> None:
self.clients.add(ws)
logging.info(f'{ws.remote_address} connects')
async def unregister(self, ws: WebSocketServerProtocol) -> None:
self.clients.remove(ws)
logging.info(f'{ws.remote_address} disconnects')
async def send_to_clients(self, message: str) -> None:
if self.clients:
logging.info("trying to send")
await asyncio.wait([client.send(message) for client in self.clients])
async def ws_handler(self, ws: WebSocketServerProtocol, url: str) -> None:
await self.register(ws)
try:
await self.distribute(ws)
finally:
await self.unregister(ws)
async def distribute(self, ws: WebSocketServerProtocol) -> None:
async for message in ws:
await self.send_to_clients(message)
async def timerThread(server,counter):
counter = 0
while True:
await checkAndSend(server,counter)
print("doing " + str(counter))
time.sleep(5)
counter = counter + 1
async def checkAndSend(server,counter):
# check something
# send message
logging.info("in check and send")
await server.send_to_clients("Hi there: " + str(counter))
# helper routine to allow thread to call async function
def between_callback(server,counter):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(timerThread(server,counter))
loop.close()
# start server
server = Server()
start_server = websockets.serve(server.ws_handler,'localhost',4000)
counter = 0
# start timer thread
threading.Thread(target=between_callback,args=(server,counter,)).start()
# start main event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
To see this working, you can use this simple html file as the client and then open the inspector to see the incoming messages on the Console log.
<h1> Websocket Test </h1>
<script>
const ws = new WebSocket('ws://localhost:4000')
ws.onopen = () => {
console.log('ws opened on browser')
ws.send('hello world')
}
ws.onmessage = (message) => {
console.log(`message received`, message.data)
}
</script>
from datetime import time
import schedule
import time
import socket
import threading
alarm_time = input()
my_string = str(alarm_time)
my_new_time = my_string.format("%H:%M %Z")
EADER = 64
PORT = xxxx
SERVER = 'xxxxxxxx'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
if alarm_time is not None and alarm_time != 0:
print(f"Scheduled for: {my_new_time}")
else:
sys.exit()
def job():
client.connect(ADDR)
name_time = self.name_send + ' ' + my_date
message = name_time.encode(FORMAT)
msg_length = len(message)
send_length = str(msg_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
client.send(send_length)
client.send(message)
schedule.every().day.at(my_new_time).do(job)
while True:
schedule.run_pending()
time.sleep(1)
Not sure if this is any help tbh. But its a small tweak of a script I use with socket, subprocess etc to schedule them
So I have made some code that follows. It is suppose to let a server and a client communicate... but it doesn't work.
Can someone explain why, or better yet fix my code???
Server.
import time
import socket
from threading import Thread
global sS
sS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sS.bind(('', 2347))
sSconAddresses = []
sSconData = []
print(" Server : Running ... ")
sS.listen(10)
while True:
try:
cOn, aDr = sS.accept()
sSconAddresses.insert(0, str(aDr))
sSconData.insert(0, str(cOn))
time.sleep(0.3)
except:
time.sleep(0.1)
pass
def ConHandler():
for _ in sSconData:
PacketData = _.recv(700)
if not PacketData:
_.close()
else:
stringData = PacketData.decode('utf-8')
print(stringData)
sS.sendto(PacketData, _)
ConHandlerThread = Thread(target=ConHandler)
ConHandlerThread.daemon = True
ConHandlerThread.start()
Client.
import threading, time
import socket, sys
import os
global cS
cS = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cS.connect(('PRIVATE', 2347))
Server = ('PRIVATE', 2347)
while True:
PacketData = input(" Client> ")
ByteData = PacketData.encode('utf-8')
cS.sendto(ByteData, Server)
It doesn't return any errors so I am confused why it doesn't work.
First of all, in your server-side code, you're having a while True before starting your thread, so it can't work.
Then, if you succeed starting your thread by moving the code, its for will see an empty list, so it will not loop, and just exit right here.
Starting from your code, here's something that works:
The client:
import socket
def main():
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(('127.0.0.1', 2345))
while True:
packetdata = input(" Client> ")
bytedata = packetdata.encode()
client_socket.send(bytedata)
print(client_socket.recv(700).decode())
if __name__ == '__main__':
main()
The server:
import socket
from threading import Thread
from queue import Queue
def client_handler(client_socket):
while True:
data = client_socket.recv(700)
print(data)
client_socket.send("Server {}".format(data.decode()).encode())
def conn_handler(conn_queue):
while True:
conn, address = conn_queue.get()
print("Handler getting a connection from {}".format(address))
client_thread = Thread(target=client_handler, args=(conn,))
client_thread.daemon = True
client_thread.start()
def main():
print("Server: Running ...")
conn_queue = Queue()
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(('', 2345))
server_socket.listen(10)
con_handler_thread = Thread(target=conn_handler, args=(conn_queue,))
con_handler_thread.daemon = True
con_handler_thread.start()
while True:
conn_queue.put(server_socket.accept())
if __name__ == '__main__':
main()
Note that this is suboptimal, starting one thread per client is not the way to go. The best way to handle this kind of situation is to keep everything in a single thread and use a select-like function to know what to do. However select is a bit limited too (like 1024 connections max, hardcoded in the libc), so the way to go is to use epoll / kqueue / whatever better than poll / select, and there's a module for this: https://docs.python.org/3/library/select.html
Yet using the select module is still the old, manual, cubersome way to express your needs, you should take a look at the coroutine based API of asyncio which enable a clear way to express the intention.
The asyncio equivalent may look like:
import asyncio
async def client():
reader, writer = await asyncio.open_connection('127.0.0.1', 8888)
while True:
message = input("Client> ")
writer.write(message.encode())
data = await reader.read(100)
print('Received: {}'.format(data.decode()))
loop = asyncio.get_event_loop()
loop.run_until_complete(client())
And, server side:
import asyncio
async def handle_client(reader, writer):
while True:
data = await reader.read(100)
if not data:
return
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_client, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
print('Serving on {}'.format(server.sockets[0].getsockname()))
loop.run_forever()
This is a simple port forwarding proxy which listens on localhost and then forwards all the connection to a real proxy and sends back the received data to the user.
The server which accepts connections and listens:
class Server:
def __init__(self):
self.server = eventlet.listen(('127.0.0.1', 8080))
self.chunk_size = 8192
def forward_connection(self, fd):
print('in fwd_conn')
data = b''
while True:
print('reading data')
x = fd.readline()
if not x.strip():
break
data += x
print('Got request:', data)
client = Client('45.55.31.207', 8081, data) # Connecting to proxy
buff = client.read_from_proxy()
fd.write(buff)
fd.flush()
def serve_client(self):
pool = eventlet.GreenPool()
while True:
try:
new_sock, addr = self.server.accept()
print('accepted:', addr)
pool.spawn(self.forward_connection, new_sock.makefile('rbw'))
except (SystemExit, KeyboardInterrupt):
break
server = Server()
server.serve_client()
This part of the code connects to the 'real' proxy, receives the data and sends it back. But as commented in the read_from_proxy function, it isn't receiving any data:
class Client:
''' All connections from Server are forwarded to Client which then
forwards it to the proxy '''
def __init__(self, ip, port, request):
self.sock = socket.socket()
self.sock.connect((ip, port))
print('in client')
print('sent request:', request)
self.request = request
def read_from_proxy(self):
self.sock.sendall(self.request)
data = b''
while True:
print('reading data from proxy') # this executes
x = self.sock.recv(8192) # but this doesn't, it gets blocked here
print('got x:', x)
if not x.strip():
break
data += x
print('read all data')
return data
Where am I going wrong?
iv created a simple async client and server but im unable to get the client to reply after receiving the first time. It seems the server can send back a reply after receiving from the client but the client cant:
here is the client's session:
[mike#mike Public]$ python cl.py
buf got your stuff
dded callback ## this is a log addded to see if execution got where i wanted
and here is the server's log:
[mike#mike Public]$ python that.py
buf ehlo localhost
i was expecting some sort of ping pong effect where one send then the other then rinse lather repeat.
here is the client's code:
import socket
import fcntl, os, io, time, functools
from tornado import ioloop
class Punk(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.connect(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.sendall("ehlo localhost")
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
while True:
servrep = self.sock.recv(1024)
if not servrep:
break
result += servrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(self.reply) ### i tried this too
print "added callback"
def reply(self):
self.sock.sendall(" clent got your stuff")
if __name__ == "__main__":
bob = Punk()
bob.setup()
and here is the server:
import socket
import fcntl, os, io, time, functools
from array import array
from tornado import ioloop
class Player(object):
def __init__(self):
self.loop = ioloop.IOLoop.instance()
self.address = 'blah.sock'
self.authkey = "none"
self.sock = socket.socket(socket.AF_UNIX)
def setup(self):
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.sock.bind(self.address)
fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK)
self.sock.listen(1)
self.fd = self.sock.fileno()
self.loop.add_handler(self.fd,self.reader,self.loop.READ)
self.loop.start()
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
try:
while True:
maxrep = self.conn.recv(1024)
if not maxrep:
break
result += maxrep
self.prnt(result)
break
except Exception as e:
print "this %s happend"%e
return
def prnt(self,buf):
print "buf %s"%buf
tim = time.time() + 2
self.loop.instance().add_timeout(tim, self.reply)
#callbac = functools.partial(self.loop.add_timeout,tim,self.reply)
#self.loop.add_callback(callbac)
def reply(self):
self.conn.sendall("got your stuff")
if __name__ == "__main__":
bob = Player()
bob.setup()
i had set my sockets to nonblock mode, but i did not catch an error when accepting from
a nonblock state when there is no connection:
here:
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
self.conn, self.addr = self.sock.accept()
should be
def reader(self,fd,event):
result = b""
if event == self.loop.READ:
try:
self.conn, self.addr = self.sock.accept() # we get stuck here
self.connl.append(self.conn)
except Exception as e:
pass
I use unittests to check correctness of TCP client communication, so I need some TCP server that I can control it and that it sends to client (and when).
A simple test should look like:
server = Server("localhost", 5555)
server.start()
client.connect()
self.assertTrue("login_message" in server.received_data)
server.send_to_client(reject_messages)
self.assertTrue("login_again" in server.received_data)
time.sleep(10)
self.assertTrue("login_again_and_again" in server.newest_received_data)
server.stop()
self.assertTrue("login failed" in client.logs)
I need the full flow control, what could be used for implementing Server?
Now I try to use threaded SocketServer, but I don't have an access neither to data, nor to controlling it..
I don't know if anybody needs this, but anyway:
I used gevent server
from gevent import sleep, socket
from gevent.server import StreamServer
class Connection:
def __init__(self, host, port):
self.server = StreamServer((host, port), self.handle)
self.data = []
self.socks = []
self.pointer = 0
def handle(self, sock, address):
self.socks.append(sock)
while True:
line = sock.recv(1024)
if line:
self.data += [line]
else:
break
sock.close()
self.socks.remove(sock)
def send(self, msg):
if self.socks:
sock2send = self.socks[-1]
try:
sock2send.send(msg)
except IOError, e:
print "Can't send message '%s'! Exception:" % msg, e
else:
print "No sockets to send the message to"
def start(self):
self.server.start()
def serve_forever(self):
self.server.serve_forever()
def close(self):
self.server.stop()
for sock in self.socks:
sock.close()
def new_data(self):
newest = self.data[self.pointer:]
self.pointer = len(self.data)
return newest
And then unittest looks like this:
def testTCPClient(self):
j = lambda x: "".join(x)
server = Connection("", 5555)
server.start()
client.run()
sleep(3)
data = j(server.new_data())
self.assertTrue("login" in data)
sleep(2)
server.send("login approve")
sleep(2)
data = j(server.new_data())
self.assertTrue("after_login" in data)
server.send("logout")
sleep(2)
data = j(server.new_data())
self.assertTrue("received_logout" in data)
server.close()
self.assertTrue("disconnected" in client.logs)