Develop Reference

Handling high frequency updates with streamlit

Summary
I want to use streamlit to create a dashboard of all the trades (buy and sell) happening in a given market. I connect to a websocket stream to receive data of BTCUSDT from the Binance exchange. Messages are received every ~0.1s and I would like to update my dashboard in ~0.09s.
How can you handle this kind of situation where messages are delivered at high frequency? With my code, I successfully create a dashboard but it doesn't get updated fast enough. I am wondering if the dashboard is running behind.
The dashboard must display the buy and sell volumes at any moment in time as bar charts. I am also adding some metrics to show the total volume of buy and sell, as well as their change.
Steps to reproduce
My code is structured in the following way.
There is a streamer.py file, that defines a class Streamer. The Streamer object is a Websocket client. It connects to a stream, handles messages, and updates the dashboard. Whenever a new message is received, Streamer acquires a threading.Lock() and updates the pandas dataframes (one dataframe for buy orders and one dataframe for sell orders). If there are multiple orders happening at the same timestamp, it combines them by summing the corresponding volumes. Then, it releases the threading.Lock() and it creates a new thread where the update function (defined in streamer.py) is executed. The update function acquires the lock to avoid messing up with memory.
In the main.py file, streamlit's dashboard and the Streamerobject are initialized.
To reproduce the following code you need to connect to the Websocket from a region where Binance is not restricted. Since I live in the US, I must use a VPN to properly receive the data.
Code snippet:
main.py file
# main.py
import streamer
import pandas as pd
import streamlit as st # web development
import numpy as np # np mean, np random
import time # to simulate a real time data, time loop
import plotly.express as px # interactive charts
df_buy = pd.DataFrame(columns = [ 'Price', 'Quantity', 'USD Value'])
df_sell = pd.DataFrame(columns = [ 'Price', 'Quantity', 'USD Value'])
st.set_page_config(
page_title='Real-Time Data Science Dashboard',
page_icon='✅',
layout='wide'
)
# dashboard title
st.title("Real-Time / Live Data Science Dashboard")
placeholder = st.empty()
streamer.Stream(df_buy,df_sell,placeholder).connect()
streamer.py file
# streamer.py
import websocket
import json
import streamlit as st
import plotly.express as px
import pandas as pd
from threading import Thread, Lock
from streamlit.script_run_context import add_script_run_ctx
from datetime import datetime
import time
def on_close(ws, close_status_code, close_msg):
print('LOG', 'Closed orderbook client')
def update(df_buy,df_sell, placeholder, lock):
lock.acquire()
with placeholder.container():
# create three columns
kpi1, kpi2 = st.columns(2)
current_sumSellVolumes = df_sell['Quantity'].sum()
previous_sumSellVolumes = df_sell.iloc[:-1]['Quantity'].sum()
current_sumBuyVolumes = df_buy['Quantity'].sum()
previous_sumBuyVolumes = df_buy.iloc[:-1]['Quantity'].sum()
# fill in those three columns with respective metrics or KPIs
kpi2.metric(label="Sell quantity 📉", value=round(current_sumSellVolumes, 2),
delta=round(current_sumSellVolumes - previous_sumSellVolumes, 2))
kpi1.metric(label="Buy quantity 📈", value=round(current_sumBuyVolumes, 2),
delta=round(current_sumBuyVolumes - previous_sumBuyVolumes, 2))
# create two columns for charts
fig_col1, fig_col2 = st.columns(2)
with fig_col1:
st.markdown("### Buy Volumes")
fig = px.bar(data_frame=df_buy, x=df_buy.index, y='Quantity')
st.write(fig)
with fig_col2:
st.markdown("### Sell Volumes")
fig2 = px.bar(data_frame=df_sell, x=df_sell.index, y='Quantity')
st.write(fig2)
st.markdown("### Detailed Data View")
st.dataframe(df_buy)
st.dataframe(df_sell)
lock.release()
class Stream():
def __init__(self, df_buy, df_sell, placeholder):
self.symbol = 'BTCUSDT'
self.df_buy = df_buy
self.df_sell = df_sell
self.placeholder = placeholder
self.lock = Lock()
self.url = "wss://stream.binance.com:9443/ws"
self.stream = f"{self.symbol.lower()}#aggTrade"
self.times = []
def on_error(self, ws, error):
print(self.times)
print('ERROR', error)
def on_open(self, ws):
print('LOG', f'Opening WebSocket stream for {self.symbol}')
subscribe_message = {"method": "SUBSCRIBE",
"params": [self.stream],
"id": 1}
ws.send(json.dumps(subscribe_message))
def handle_message(self, message):
self.lock.acquire()
timestamp = datetime.utcfromtimestamp(int(message['T']) / 1000)
price = float(message['p'])
qty = float(message['q'])
USDvalue = price * qty
side = 'BUY' if message['m'] == False else 'SELL'
if side == 'BUY':
df = self.df_buy
else:
df = self.df_sell
if timestamp not in df.index:
df.loc[timestamp] = [price, qty, USDvalue]
else:
df.loc[df.index == timestamp, 'Quantity'] += qty
df.loc[df.index == timestamp, 'USD Value'] += USDvalue
self.lock.release()
def on_message(self, ws, message):
message = json.loads(message)
self.times.append(time.time())
if 'e' in message:
self.handle_message(message)
thr = Thread(target=update, args=(self.df_buy, self.df_sell, self.placeholder, self.lock,))
add_script_run_ctx(thr)
thr.start()
def connect(self):
print('LOG', 'Connecting to websocket')
self.ws = websocket.WebSocketApp(self.url, on_close=on_close, on_error=self.on_error,
on_open=self.on_open, on_message=self.on_message)
self.ws.run_forever()
Debug info
Streamlit version: 1.4.0
Python version: 3.10.4
OS version: MacOS 13.1
Browser version: Safari 16.2

How to sending images to unsaved numbers in pywhatkit?

My aim is to send an image file in whatsapp to various persons who are not on my contact list. Try with pywhatkit. Store the number in an excel sheet and read it by pandas. Loop the number to send the image file. The first number gets the image file successfully, and the remaining numbers WhatsApp will open but the message will not send. Also tell, me how to avoid WhatsApp open in a new tab(chrome)every time.
Here's my code
import pandas as pd
file_name = r'd:\\Recipients data.xlsx'
df = pd.read_excel(file_name)
contact = df['Contact'].tolist()
import pywhatkit
import time
from datetime import datetime
now = datetime.now()
current_time = now.strftime("%H:%M : %S")
h,m,s = current_time.split(':')
print(now)
h1 = int(h)
m1 = int(m)
s1 = int(s)
file_path = r'C:/Users/Asus/Downloads/subadev.jpeg'
for send_number in contact:
new_format = (f'"{str("+91")+str(send_number)}"')
print(new_format)
pywhatkit.sendwhats_image(new_format,file_path)
time.sleep(15)
expecting : send images to every number without open whats app every time

why does the keyboard.is_pressed() function refuse to work when pressed?

i have less than 3 months of python programming under my belt but basically, i have a program that pulls values from the CoinGecko API indefinitely and creates processes so the functions that pull the data can run independently from one another, but id like for it to break its loop after i press the specified hotkey which is set to 'q'. whenever i press the hotkey, nothing happens and the loop just keeps running. i've tried using the keyboard.read_key() function, but that just stops my program from running until i press the q button, which causes the program to run the loop once and then close. i have no idea why the is_pressed() function refuses to work and id like some help from more advanced people
Piece of Code in question:
from multiprocessing.dummy import freeze_support
from pycoingecko import CoinGeckoAPI
import time
from multiprocessing import Process
from multiprocessing import Pool
import multiprocessing
import keyboard as kb
import psutil
cg = CoinGeckoAPI()
class CGCoin:
def __init__(self, coinname, coinid):
self.coinname = coinname
self.coinid = coinid
def pulldata(self):
while True:
wishtoquit = False
if kb.is_pressed('Q'):
print('ending after this loop')
wishtoquit = True
timestarted = time.asctime()
self.prices = []
self.daychanges = []
self.volumes = []
self.marketcaps = []
self.weekchanges = []
self.highs = []
self.lows = []
self.times = []
print(f'starting {self.coinname} reading at {timestarted}')
loops = 0
maxloops = 2
while loops < maxloops:
time.sleep(15)
coin = cg.get_coin_by_id(f'{self.coinid}')
time.sleep(5)
coinvalues = coin.get('market_data')
coinprices = coinvalues.get('current_price')
coinvolumes = coinvalues.get('total_volume')
mrktcaps = coinvalues.get('market_cap')
dayhigh = coinvalues.get('high_24h')
daylow = coinvalues.get('low_24h')
daychangepercentage = coinvalues.get('price_change_percentage_24h')
weekchangepercentage = coinvalues.get('price_change_percentage_7d')
coinprice = coinprices.get('usd')
coinvolume = coinvolumes.get('usd')
coincap = mrktcaps.get('usd')
coindayhigh = dayhigh.get('usd')
coindaylow = daylow.get('usd')
timepulled = time.asctime()
self.prices.append(coinprice)
self.daychanges.append(daychangepercentage)
self.volumes.append(coinvolume)
self.marketcaps.append(coincap)
self.weekchanges.append(weekchangepercentage)
self.highs.append(coindayhigh)
self.lows.append(coindaylow)
self.times.append(timepulled)
loops = loops + 1
print(loops)
timeended = time.asctime()
})
print(f'stopping {self.coinname} reading at {timeended}')
if wishtoquit:
print('ending loops')
break
time.sleep(5)
bitcoin = CGCoin('Bitcoin', 'bitcoin')
ethereum = CGCoin('Ethereum', 'ethereum')
if __name__ == '__main__':
freeze_support()
btcpul = Process(target=bitcoin.pulldata, name=bitcoin.coinname)
btcpul.start()
if anyone has any ideas or fully-functional workarounds id really like to hear them. id be extremely grateful for any help recieved

It looks like PyPi keyboard needs root permissions on linux.
You could just do kb.on_press_key("p", lambda _: sys.exit(0)) and just do a sys.exit(0) to end the script.
If you're running this in the terminal you should just be able to press ctrl+c to interrupt its execution.
ref: How to detect key presses?

Threading with pytube while searching

This is my code, I'm trying to download a playlist from youtube.In order to prevent the app from taking too long, I use threading. But it is taking too long and I'm literally not getting any reaction form the window. Simply it's not working.Does anyone know what the problem is? (Note that I am new to python).Thank you in advance.
Here is my code:
def playlist_search():
global pl
url = playlist_link_entry.get()
try:
#Create a PlayList object
#Downloading is gonna happen with pytube
pl = Playlist(url)
except:
messagebox.showerror("Error","Either link is not functioning or connection broken".title())
#Gotta get the links to every single video
#We name it 'video_link_playlist'
#for that specific url we do our shit(get itag, size and ... thats exactly our previous code)
#We map the playlist_actual_search to pl.video_urls
with concurrent.futures.ThreadPoolExecutor() as executor:
executor.map(playlist_actual_search, pl.video_urls)
We create this function to do the search concurrently
def playlist_actual_search(pl_link):
#Saving All Links to a list to use it later
playlist_link_list.append(pl_link)
#Getting itag number is gonna happen with pafy
#Create a pafy object
video = pafy.new(pl_link)
#Get all streams from youtube
streams = video.allstreams
#Create a YouTube object to get every video individually
yt = YouTube(pl_link)
#Add yt to the yt.list
yt_list.append(yt)
#Get those streams that are normal(addaptive)
for stream in streams:
if "normal" in str(stream):
playlist_normal_stream_list.append(stream)
#Get itag value of the addaptive video
for normal_stream in playlist_normal_stream_list:
if "720" in str(normal_stream):
itag720 = normal_stream.itag
playlist_itag_value_dic["itag720"] = itag720
normal_stream = yt.streams.get_by_itag(itag720)
playlist720_video_size += normal_stream.filesize_approx
if "480" in str(normal_stream):
itag480 = normal_stream.itag
playlist_itag_value_dic["itag480"] = itag480
normal_stream = yt.streams.get_by_itag(itag480)
playlist480_video_size += normal_stream.filesize_approx
if "360" in str(normal_stream):
itag360 = normal_stream.itag
playlist_itag_value_dic["itag360"] = itag360
normal_stream = yt.streams.get_by_itag(itag360)
playlist360_video_size += normal_stream.filesize_approx

Streaming Grid Display in Jupyter Notebook

I am trying to display live price updates coming from a redis pubsub channel in a grid in Jupyter. Everytime there is a price update, the message will be added at the end of the grid. In order words, a gridview widget will be tied to a Dataframe so everytime it changes, the gridview will change. The idea is to get something like this:
I tried to do that by displaying and clearing the output. However, I am not getting a the streaming grid that gets updated in-place but rather displaying and clearing the output which is very annoying.
Here is the output widget in one jupyter cell
import ipywidgets as iw
from IPython.display import display
o = iw.Output()
def output_to_widget(df, output_widget):
output_widget.clear_output()
with output_widget:
display(df)
o
Here is the code to subscribe to redis and get handle the message
import redis, json, time
r = redis.StrictRedis(host = HOST, password = PASS, port = PORT, db = DB)
p = r.pubsub(ignore_subscribe_messages=True)
p.subscribe('QUOTES')
mdf = pd.DataFrame()
while True:
message = p.get_message()
if message:
json_msg = json.loads(message['data'])
df = pd.DataFrame([json_msg]).set_index('sym')
mdf = mdf.append(df)
output_to_widget(mdf, o)
time.sleep(0.001)

Try changing the first line of output_to_widget to output_widget.clear_output(wait = True).
https://ipython.org/ipython-doc/3/api/generated/IPython.display.html

I was able to get it to work using Streaming DataFrames from the streamz library.
Here is the class to emit the data to the streamming dataframe.
class DataEmitter:
def __init__(self, pubsub, src):
self.pubsub = pubsub
self.src = src
self.thread = None
def emit_data(self, channel):
self.pubsub.subscribe(**{channel: self._handler})
self.thread = self.pubsub.run_in_thread(sleep_time=0.001)
def stop(self):
self.pubsub.unsubscribe()
self.thread.stop()
def _handler(self, message):
json_msg = json.loads(message['data'])
df = pd.DataFrame([json_msg])
self.src.emit(df)
and here is the cell to display the streaming dataframe
r = redis.StrictRedis(host = HOST, password = PASS, port = PORT, db = DB)
p = r.pubsub(ignore_subscribe_messages=True)
source = Stream()
emitter = DataEmitter(p, source, COLUMNS)
emitter.emit_data(src='PRICE_UPDATES')
#sample for how the dataframe it's going to look like
example = pd.DataFrame({'time': [], 'sym': []})
sdf = source.to_dataframe(example=example)
sdf