So I am trying to scrape data from some websocket news site. I have a main script (not shown) that handles postprocessing and calls pool.apply_async(main_news, ()) to return data. My problem is, that it's only possible for me to return data from a message if I do ws.close() before "return", however ws.close() takes around 1 second which is too much and I feel it is unnecessary too.
What options do I have to just directly return "data" to my main script without having to wait for ws.close()?
import websocket, ssl, json, re, time
from datetime import datetime
def on_open(ws):
print("Opened connection")
def on_message(ws, message):
json_message = json.loads(message) # load json string into dict format
title = json_message["title"]
print(title,str(datetime.now())[11:-4])
global data
data_match = re.search(r"DATA", title)
if data_match:
data = data_match.group(0)
ws.close() # Takes around 1 second!!
return data
def on_error(ws, error):
print(error)
def on_close(ws, close_status_code, close_msg): # automatically restart websocket in case of error
print("on_close args:")
if close_status_code or close_msg:
print("close status code: " + str(close_status_code))
print("close message: " + str(close_msg))
print ("Retry: %s" % time.ctime())
time.sleep(3)
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
def main_news(socket = "wss://www.madnews.io/ws"):
ws = websocket.WebSocketApp(socket, on_open = on_open, on_message = on_message, on_close = on_close, on_error = on_error)
ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})
return data
if __name__ == "__main__":
main_news()
I have started to work with Asyncio recently and I was wondering if that is possible thing to do.
I made a TCP Echo server that get "commands" from client in order to execute methods with binance.client (which is stored on external file called GetData), I've also made a client that connects to binance websocket to receive live updates.
The problem is I want to combine the TCP Server and the Client that connects to binance websocket so when the server is running it would be able to handle the streaming data and still let the server handle new client connections.
Server Code:
import asyncio
from GetData import *
class EchoServerProtocol(asyncio.Protocol):
def connection_made(self, transport):
peername = transport.get_extra_info('peername')
print('Connection from {}'.format(peername))
self.transport = transport
def data_received(self, data):
recivedData = data.decode()
print('Data received: {!r}'.format(recivedData))
commands = recivedData.split(' ')
if commands[0] == 'ping':
result = 'pong!'
print(result)
elif commands[0] == 'buy' and commands[1].upper() in get_all_assets():
symbol = commands[1].upper()
result = f'Buying request for {symbol}'
try:
current_status = buy_asset(symbol, commands[2])
result = f"You bought: {current_status['asset_amount']} {symbol}, Timestamp: {current_status['time']}"
print(result)
# Save on firebase >>
except IndexError:
print("Please mention usdt amount.")
else:
result = 'invalid command!'
print(result)
self.transport.write(data)
print('Close the client socket')
self.transport.close()
def main():
loop = asyncio.get_event_loop()
coro = loop.create_server(EchoServerProtocol, '127.0.0.1', 8070)
print("Server is now running")
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
print("exit")
finally:
server.close()
loop.close()
if __name__ == "__main__":
asyncio.run(main())
Client code:
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from GetData import *
import os
import json
class BinanceMarketStream(WebSocketClientProtocol):
def onConnect(self, response):
print("Server connected: {0}".format(response.peer))
self.df_historicalData = get_historical_kline('BTCUSDT', '1m')
def onOpen(self):
print("WebSocket connection open.")
def onMessage(self, payload, isBinary):
if isBinary:
print("Binary message received: {0} bytes".format(len(payload)))
else:
data = payload.decode('utf8')
os.system('cls' if os.name == 'nt' else 'clear')
json_message = json.loads(data)
candle = json_message['k']
is_candle_closed = candle['x']
currentCandle = set_to_df(candle)
self.df_historicalData = update_currentCandle(
self.df_historicalData, currentCandle)
if is_candle_closed:
newCandle = set_to_df(candle)
self.df_historicalData = append_newcandle(
self.df_historicalData, newCandle)
# Trading Algorithms goes to here...
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
import asyncio
factory = WebSocketClientFactory(
"wss://stream.binance.com:9443/ws/btcusdt#kline_1m")
factory.protocol = BinanceMarketStream
loop = asyncio.get_event_loop()
coro = loop.create_connection(
factory, "stream.binance.com", 9443, ssl=True)
loop.run_until_complete(coro)
loop.run_forever()
loop.close()
I can also add the client that connects to the TCP server if needed.
How do I write a websockets server in Python which just pushes out data on a timed interval to all connected clients without waiting for any incoming messages ?
I'm answering my own question ...
This is a working example of a Python websockets server which sends out a message to all clients every 5 seconds. I wrote this and managed to get it working as I could not find a single example of this on the web (March 2021)
Hope this helps others, and if anyone has suggestions for improvements or better solutions using packages to maybe add ssl support or subscription type services additions, please write in the comments or answers section.
import asyncio
import logging
import websockets
from websockets import WebSocketServerProtocol
import time
import threading
logging.basicConfig(level=logging.INFO)
class Server:
clients = set()
logging.info(f'starting up ...')
def __init__(self):
logging.info(f'init happened ...')
async def register(self, ws: WebSocketServerProtocol) -> None:
self.clients.add(ws)
logging.info(f'{ws.remote_address} connects')
async def unregister(self, ws: WebSocketServerProtocol) -> None:
self.clients.remove(ws)
logging.info(f'{ws.remote_address} disconnects')
async def send_to_clients(self, message: str) -> None:
if self.clients:
logging.info("trying to send")
await asyncio.wait([client.send(message) for client in self.clients])
async def ws_handler(self, ws: WebSocketServerProtocol, url: str) -> None:
await self.register(ws)
try:
await self.distribute(ws)
finally:
await self.unregister(ws)
async def distribute(self, ws: WebSocketServerProtocol) -> None:
async for message in ws:
await self.send_to_clients(message)
async def timerThread(server,counter):
counter = 0
while True:
await checkAndSend(server,counter)
print("doing " + str(counter))
time.sleep(5)
counter = counter + 1
async def checkAndSend(server,counter):
# check something
# send message
logging.info("in check and send")
await server.send_to_clients("Hi there: " + str(counter))
# helper routine to allow thread to call async function
def between_callback(server,counter):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(timerThread(server,counter))
loop.close()
# start server
server = Server()
start_server = websockets.serve(server.ws_handler,'localhost',4000)
counter = 0
# start timer thread
threading.Thread(target=between_callback,args=(server,counter,)).start()
# start main event loop
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
To see this working, you can use this simple html file as the client and then open the inspector to see the incoming messages on the Console log.
<h1> Websocket Test </h1>
<script>
const ws = new WebSocket('ws://localhost:4000')
ws.onopen = () => {
console.log('ws opened on browser')
ws.send('hello world')
}
ws.onmessage = (message) => {
console.log(`message received`, message.data)
}
</script>
from datetime import time
import schedule
import time
import socket
import threading
alarm_time = input()
my_string = str(alarm_time)
my_new_time = my_string.format("%H:%M %Z")
EADER = 64
PORT = xxxx
SERVER = 'xxxxxxxx'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
if alarm_time is not None and alarm_time != 0:
print(f"Scheduled for: {my_new_time}")
else:
sys.exit()
def job():
client.connect(ADDR)
name_time = self.name_send + ' ' + my_date
message = name_time.encode(FORMAT)
msg_length = len(message)
send_length = str(msg_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
client.send(send_length)
client.send(message)
schedule.every().day.at(my_new_time).do(job)
while True:
schedule.run_pending()
time.sleep(1)
Not sure if this is any help tbh. But its a small tweak of a script I use with socket, subprocess etc to schedule them
I am VERY new to coding and Python and I am trying to just receive live trade data from the Bitfinex API, and filter out specific messages as they come in because it gives duplicates. I want to take these filtered messages and then output them to a csv file continuously.
Specifically, I want to save the messages titled "te" (see output from API below) because these are the trades that are executed as they are executed. The stream gives "tu" as well, which are duplicates. I want to just take the "te" and download them live into a csv for other processing and saving.
Here is my code, which is a stripped down version of one that I found online:
import websocket
import time
import sys
from datetime import datetime, timedelta, timezone
import sched, time
import json
import csv
import requests
class BitfinexWebSocketReader():
endpoint = "wss://api.bitfinex.com/ws/2"
def __init__(self):
#websocket.enableTrace(True)
self.ws = websocket.WebSocketApp(
BitfinexWebSocketReader.endpoint,
on_message = self.on_message,
on_error = self.on_error,
on_close = self.on_close
)
self.ws.on_open = self.on_open
try:
self.run()
except KeyboardInterrupt:
self.ws.close()
def run(self):
self.ws.run_forever()
print("### run ###")
pass
def on_message(self, ws, message):
print(message)
def on_error(self, ws, error):
print(error)
sys.exit()
def on_close(self, ws):
print("### closed ###")
def on_open(self, ws):
#print("### open ###")
ws.send(json.dumps({"event": "subscribe", "channel": "Trades", "symbol": "tBTCUSD"}))
if __name__=="__main__":
BitfinexWebSocketReader()
And here is an example of a couple seconds of the output:
{"event":"info","version":2,"serverId":"88c6df7e-5159-4a8e-b1c4-f08904aeeb0a","platform":{"status":1}}
{"event":"subscribed","channel":"trades","chanId":23,"symbol":"tBTCUSD","pair":"BTCUSD"}
[23,[[281534165,1534448458635,0.005,6401.5],[281534164,1534448457975,0.01999998,6401.5],[281534139,1534448438766,-0.31749096,6401.4],[281534132,1534448438051,0.005,6401.5],[281534116,1534448432624,-0.051,6401.4],[281534099,1534448425380,0.18699482,6401.5],[281534097,1534448424900,0.013558,6401.5],[281534096,1534448424718,0.0514726,6401.5],[281534083,1534448415788,0.005,6401.8],[281534080,1534448415568,-1,6400.8],[281534079,1534448415566,-1,6401.8],[281534073,1534448409395,-0.0325,6403],[281534053,1534448398108,-0.2498,6405.1],[281534048,1534448396370,-0.25,6404.9],[281534043,1534448394675,0.42406762,6400],[281534029,1534448390257,0.30000001,6400],[281534028,1534448390236,0.30000001,6400],[281534027,1534448389714,1,6400],[281534025,1534448389033,1.18922278,6400],[281534024,1534448389030,0.41523564,6399.7],[281534023,1534448389028,0.39554158,6399.7],[281534013,1534448384920,0.025,6399.7],[281534011,1534448382885,0.018794,6399.7],[281534008,1534448380817,-1.49155951,6399.6],[281534007,1534448380815,-2.5,6399.6],[281534006,1534448380813,-0.34,6399.6],[281534005,1534448380811,-0.15098794,6399.6],[281534004,1534448380808,-0.29899445,6399.6],[281534000,1534448379152,-0.005,6399.6],[281533999,1534448377821,-0.16825162,6399.6]]]
[23,"hb"]
[23,"te",[281534199,1534448478028,-0.00937287,6401.4]]
[23,"te",[281534200,1534448478031,-0.29062714,6401.4]]
[23,"te",[281534201,1534448478036,-0.30000001,6401.4]]
[23,"tu",[281534201,1534448478036,-0.30000001,6401.4]]
[23,"tu",[281534199,1534448478028,-0.00937287,6401.4]]
[23,"tu",[281534200,1534448478031,-0.29062714,6401.4]]
[23,"te",[281534204,1534448478180,-0.65915285,6401.4]]
[23,"tu",[281534204,1534448478180,-0.65915285,6401.4]]
[23,"hb"]
[23,"te",[281534224,1534448479402,-0.114,6399.9]]
[23,"tu",[281534224,1534448479402,-0.114,6399.9]]
[23,"te",[281534232,1534448480466,-0.00012512,6399.9]]
[23,"tu",[281534232,1534448480466,-0.00012512,6399.9]]
Bonus question: why does that super long first entry pop up every time I execute the code?
You can initialize some kind of data structure in the constructor, like a list() or a set() to store the desired messages and then filter them in the on_message method.
So in your constructor
def __init__(self):
#websocket.enableTrace(True)
self.ws = websocket.WebSocketApp(
BitfinexWebSocketReader.endpoint,
on_message = self.on_message,
on_error = self.on_error,
on_close = self.on_close
)
self.ws.on_open = self.on_open
self.store = []
try:
self.run()
except KeyboardInterrupt:
self.ws.close()
And in your on_message method
def on_message(self, ws, message):
if "te" in message:
self.store.append(message)
print(message)
When I trying to connect to its jupyter client in python code ,I encount a problem.
In the source code of jupyter, connection to zmq channel was established when websocket is opened
def open(self, kernel_id):
super(ZMQChannelsHandler, self).open()
try:
self.create_stream()
except web.HTTPError as e:
for channel, stream in self.channels.items():
if not stream.closed():
stream.close()
self.close()
else:
for channel, stream in self.channels.items():
//this is callback function when receive message from zqm channel
stream.on_recv_stream(self._on_zmq_reply)
while in the create_stream function, the zmq channel was established.
def create_stream(self):
km = self.kernel_manager
identity = self.session.bsession
for channel in ('shell', 'iopub', 'stdin'):
meth = getattr(km, 'connect_' + channel)
self.channels[channel] = stream = meth(self.kernel_id, identity=identity)
stream.channel = channel
... ignore no significance code
when the server receive message, on_message was invoke
def on_message(self, msg):
if not self.channels:
return
if isinstance(msg, bytes):
msg = deserialize_binary_message(msg)
else:
msg = json.loads(msg)
channel = msg.pop('channel', None)
if channel is None:
channel = 'shell'
if channel not in self.channels:
return
stream = self.channels[channel]
self.session.send(stream, msg)
At this time, zmq channel receive python code to be executed. After that, the execution result should be return, thus the function on_recv_stream above should be called and we got the result finally.
So I write the python code snippet like this:
from jupyter_client.multikernelmanager import MultiKernelManager
from jupyter_client.session import Session
from tornado import gen, web
from tornado.concurrent import Future
from tornado.ioloop import IOLoop
km = MultiKernelManager()
kernelid = km.start_kernel()
kernel =km.get_kernel(kernelid)
channel = km.connect_shell(kernelid)
print 'channel', channel
def on_reply(msg):
print 'we got return'
def on_timeout():
print("Timeout waiting for kernel_info_reply: %s", kernel_id)
kernel.session.send(channel, 'kernel_info_request')
channel.on_recv(on_reply)
Actually, I did not get the return message, that is to say,the on_reply function was not invoked. I did not what the problem is, Can anynone help me?
I solve the problem like this:
from jupyter_client.multikernelmanager import MultiKernelManager
km = MultiKernelManager()
kernelid = km.start_kernel('python2')
kn =km.get_kernel(kernelid)
kc = kn.client()
kc.start_channels()
msg_id = kc.execute('import math\nprint(math.sqrt(2))')
while True:
try:
msg = kc.get_iopub_msg()
print('\niopub msg is')
print(msg)
except Excption,e:
print(e)
break
if msg['parent_header'].get('msg_id') != msg_id:
continue
msg_type = msg['msg_type']
content = msg['content']
print('content is :')
print(content)
if msg_type == 'status':
if content['execution_state'] == 'idle':
break
else:
continue