Develop Reference

Python: measuring websocket queue length waiting for ws.recv()

I am consuming low latency market data and I'm trying to measure how many streams I can consume without my code slowing down due to the websocket message queue building up. My understanding is that messages are received by the websocket and queued until ws.recv() is called, which processes them one at a time in the order they were received. Under normal circumstances, my code is definitely fast enough to handle the messages, but when a burst of messages comes all at once I would imagine that the queue fills up. I would expect that the queue would only be filled up for 5 or 10 milliseconds, but it is very important that I know this. Is there a way to measure how many messages are waiting in the queue?
I'm attaching a snippet of the code I'm using for context, but the relevant part is just looping over
data = self.ws.recv()
class WebsocketClient(object):
def __init__(
self,
url=""
products=None,
message_type="subscribe",
should_print=True,
self.url = url
self.products = products
self.channels = channels
self.type = message_type
self.stop = True
self.error = None
self.ws = None
self.thread = None
self.auth = auth
self.api_key = api_key
self.api_secret = api_secret
self.api_passphrase = api_passphrase
self.should_print = should_print
def start(self):
def _go():
self._connect()
self._listen()
self._disconnect()
self.stop = False
self.on_open()
self.thread = Thread(target=_go)
self.keepalive = Thread(target=self._keepalive)
self.thread.start()
def _connect(self):
if self.products is None:
self.products = []
elif not isinstance(self.products, list):
self.products = [self.products]
if self.url[-1] == "/":
self.url = self.url[:-1]
if self.channels is None:
self.channels = [{"name": "ticker", "product_ids": [product_id for product_id in self.products]}]
sub_params = {'type': 'subscribe', 'product_ids': self.products, 'channels': self.channels}
else:
sub_params = {'type': 'subscribe', 'product_ids': self.products, 'channels': self.channels}
if self.auth:
#timestamp = int(time.time())
#message = timestamp + 'GET' + '/users/self/verify'
auth_headers = get_auth_headers('/users/self/verify','GET','')
#print(auth_headers)
sub_params['signature'] = auth_headers['CB-ACCESS-SIGN']
sub_params['key'] = auth_headers['CB-ACCESS-KEY']
sub_params['passphrase'] = auth_headers['CB-ACCESS-PASSPHRASE']
sub_params['timestamp'] = auth_headers['CB-ACCESS-TIMESTAMP']
try:
self.ws = create_connection(self.url)
self.ws.send(json.dumps(sub_params))
except:
traceback.print_exc()
self.stop = True
def _keepalive(self, interval=10):
while self.ws.connected:
self.ws.ping("keepalive")
time.sleep(interval)
def _listen(self):
self.keepalive.start()
while not self.stop:
try:
data = self.ws.recv()
msg = json.loads(data)
except ValueError as e:
self.on_error(e)
except Exception as e:
self.on_error(e)
else:
self.on_message(msg)
def _disconnect(self):
try:
if self.ws:
self.ws.close()
except WebSocketConnectionClosedException as e:
pass
finally:
self.keepalive.join()
self.on_close()
def close(self):
self.stop = True # will only disconnect after next msg recv
self._disconnect() # force disconnect so threads can join
self.thread.join()
def on_open(self):
if self.should_print:
print("-- Subscribed! --\n")
def on_close(self):
if self.should_print:
print("\n-- Socket Closed --")
def on_message(self, msg):
*** my logic ***
def on_error(self, e, data=None):
self.error = e
self.stop = True
print('{} - data: {}'.format(e, data))

You can measure length of incomming messages buffer by calling
len(self.ws.messages)
There is a background asyncio task, that reads StreamReader bytes buffer and put messages to ws.messages deque.
Messages deque is limited by max_queue parameter of client.connect method:
https://websockets.readthedocs.io/en/stable/reference/client.html#websockets.client.connect
Here are the details:
https://websockets.readthedocs.io/en/stable/topics/design.html#backpressure

How to know the name of the thread inside it

I have created multiple threads using for loop and I need to read the name of the thread inside the thread. How do I do that? I used threading.Thread.name but returns some memory address of the thread object. here is part of my code:
def catch_sharp_fall(update, context, x):
global access_token, watchlist, bot, chat_id, market_open_alert, nifty, bot_reset, stop_catch_sharp_fall, \
kite_api_key
ins_code = x
kite = KiteConnect(kite_api_key)
kite.set_access_token(access_token)
kws = KiteTicker(kite_api_key, access_token)
def on_ticks(ws, ticks):
logging.debug("Ticks: {}".format(ticks))
def on_connect(ws, response):
ws.subscribe([ins_code])
ws.set_mode(ws.MODE_FULL, [ins_code])
def on_close(ws, code, reason):
ws.stop()
kws.on_ticks = on_ticks
kws.on_connect = on_connect
kws.on_close = on_close
kws.connect(threaded=True)
while True:
def on_ticks(ws, ticks):
feed_data(ticks)
def feed_data(ticks):
list_of_threads_running = [i.name for i in threading.enumerate()]
logging.info(f'list of threads running {list_of_threads_running}')
# update.message.reply_text(f'<b>I am the stock {ins_code}</b>', parse_mode='HTML')
logging.info(f'{threading.Thread.name}')
kws.on_ticks = on_ticks
for x in watchlist:
t = threading.Thread(name=str(x), target=catch_sharp_fall, args=(update, context, x))
t.setDaemon(True)
t.start()

When you do threading.Thread.name, you're retrieving name on the Thread type itself, not on the thread object for the current thread. You need to look it up on the current thread object:
threading.current_thread().name

You can use threading.current_thread().name, t1.name and t2.name as shown below:
import threading
def test1():
print(threading.current_thread().name) # "Thread-1"
def test2():
print(threading.current_thread().name) # "test_thread"
t1 = threading.Thread(target=test1)
t2 = threading.Thread(target=test2, name="test_thread")
# Thread name
t1.start()
t2.start()
t1.join()
t2.join()
print(t1.name) # "Thread-1"
print(t2.name) # "test_thread"
print(threading.current_thread().name) # "MainThread"

Handle multiprocess in python

My code is processing some parallel perforce tasks while showing a progress bar and letting user to terminate the job whenever he wants, the problem is when user clicks the close button the thread function is not being killed but the lock is released and the main UI thread is being unlocked.
The p4.run_sync() is not terminating when Cancel button is clicked.
def P4SyncLibrary(args, que):
syncType = args[0]
view = args[1]
p4 = P4CreateConnection(disable_tmp_cleanup=True)
try:
p4.run_sync(view)
except P4Exception:
for e in p4.errors:
print "SyncError: - %s" %e
p4.disconnect()
que.put(None)
class CreateJob(QtGui.QDialog):
def __init__(self, thread, args):
QtGui.QDialog.__init__(self)
self.ui=Ui_ProgressBar()
self.ui.setupUi(self)
self.ui.cancel.clicked.connect(self.closeEvent)
self.ui.cancel.setIcon(QtGui.QIcon(QtGui.QPixmap("%s/delete.xpm" %resources)))
self.threadControl = ThreadControl(thread=thread, args=args)
self.connect(self.threadControl, QtCore.SIGNAL("__updateProgressBar(int)"), self.__updateProgressBar)
self.threadControl.finished.connect(self.closeEvent)
self.threadControl.start()
#QtCore.pyqtSlot(int)
def __updateProgressBar(self,val):
self.ui.progressBar.setValue(val)
self.setWindowTitle("Processing: {0}%".format(val))
def closeEvent(self, QCloseEvent=None):
if self.threadControl.isRunning():
self.threadControl.stop()
self.threadControl.wait()
if QCloseEvent: QtGui.QDialog.closeEvent(self, QCloseEvent)
else: self.close()
def getResults(self):
return self.threadControl.resultDict
class ThreadControl(QtCore.QThread):
stopFlag = 0
def __init__(self, thread=None, args=None):
super(ThreadControl, self).__init__()
self.args = args
self.thread = thread
self.resultDict = []
def run(self):
threads = {}
queue = multiprocessing.Queue()
for arg in self.args:
process = multiprocessing.Process(target=self.thread, args=(arg, queue))
process.start()
threads[process] = 1 ## ACTIVE thread
# WAIT TILL ALL PROCESSES COMPLETE
completedThreads = 0
total = len(threads.keys())
while completedThreads != total:
if self.stopFlag:
for t in threads.keys():
if threads[t] == 1:
t.terminate()
t.join()
threads[t] = 0
completedThreads += 1
else:
for t in threads.keys():
if self.stopFlag: break ## Process threads termination
elif threads[t] == 1 and not t.is_alive():
threads[t] = 0
completedThreads += 1
self.resultDict.append(queue.get())
self.emit(QtCore.SIGNAL('__updateProgressBar(int)'),(completedThreads*100)/total)
sleep(0.5) ## Prevent CPU from overloading
def stop(self):
self.stopFlag=1
a job is being created using instance of CreateJob
CreateJob(thread=P4SyncLibrary, args=P4Libraries).exec_()

The only solution I could give is to pass p4 object to calling thread as argument so that p4 server connection can disconnect when user wants to cancel the job.
def P4SyncLibrary(p4, args, que):
syncType = args[0]
view = args[1]
try:
p4.run_sync(view)
except P4Exception:
for e in p4.errors:
print "SyncError: - %s" %e
que.put(None)
class ThreadControl(QtCore.QThread):
...
def run(self):
threads = {}
queue = multiprocessing.Queue()
for arg in self.args:
connection = P4CreateConnection(disable_tmp_cleanup=True)
if connection.connected():
process = multiprocessing.Process(target=self.thread, args=(connection, arg, queue))
process.start()
threads[process] = {
'isAlive': True,
'connection': connection
}
# WAIT TILL ALL PROCESSES COMPLETE
completedThreads = 0
total = len(threads.keys())
while completedThreads != total:
if self._stop:
for t in threads.keys():
if threads[t]['isAlive']:
threads[t]['connection'].disconnect()
t.terminate()
t.join()
threads[t]['isAlive'] = False
completedThreads += 1
else:
for t in threads.keys():
if self._stop: break ## Process threads termination
elif threads[t]['isAlive'] and not t.is_alive():
threads[t]['connection'].disconnect()
threads[t]['isAlive'] = False
completedThreads += 1
self.results.append(queue.get())
self.emit(QtCore.SIGNAL('__updateProgressBar(int)'),(completedThreads*100)/total)
sleep(0.5) ## Prevent CPU from overloading

Run Class methods in threads (python)

I'm currently learning Python and Classes and I have a basic question, but I didn't find any answer to it. Let's say I have this dummy class
class DomainOperations:
def __init__(self, domain):
self.domain = domain
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
#resolve domain to ipv4 and save to self.domain_ip
def generate_website_thumbnail(self):
#generate website thumbnail and save the url to self.website_thumbnail
I want to run simultaneously resolve_domain and generate_website_thumbnail and when the threads are finished I want to print the IP and the thumbnail.
EDIT: I know I should use threads, maybe something like this
r = DomainOperations('google.com')
t1 = threading.Thread(target=r.resolve_domain)
t1.start()
t2 = threading.Thread(target=r.generate_website_thumbnail)
t2.start()
But should I use them outside the Class? Should I write another Class to handle Threads?
What is the right way to do that?

If you call them from the class, it is as simple as:
import threading
class DomainOperations:
def __init__(self):
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
self.domain_ip = 'foo'
def generate_website_thumbnail(self):
self.website_thumbnail= 'bar'
def run(self):
t1 = threading.Thread(target=self.resolve_domain)
t2 = threading.Thread(target=self.generate_website_thumbnail)
t1.start()
t2.start()
t1.join()
t2.join()
print(self.domain_ip, self.website_thumbnail)
if __name__ == '__main__':
d = DomainOperations()
d.run()

You can inherit Thread class in DomainOperation, in this way code would be more clean and easily understandable. you have to override a run() method.
from threading import Thread
class DomainOperations(Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.domain_ip = ''
self.website_thumbnail = ''
def resolve_domain(self):
self.domain_ip = 'foo'
def generate_website_thumbnail(self):
self.website_thumbnail= 'bar'
def run(self):
#domain will be resolved on first thread
self.resolve_domain()
#thumbnail will be resolved on second OR newly created below thread
thread2 = Thread(target=self.generate_website_thumbnail)
thread.start()
# thread1 will wait for thread2
self.join()
# thread2 will wait for thread1, if it's late.
thread2.join()
# here it will print ip and thumbnail before exiting first thread
print(self.domain_ip, self.website_thumbnail)
And you will start your threads in this way.
if __name__ == '__main__':
thread1 = DomainOperations()
thread1.start()

def post_test(tbid, line_num, response_time):
"""
:param tbid: 参数id
:return:
"""
# 请求参数
data = {'tbId': tbid, 'conditions': [{"key": "", "type": 1}], 'pageNum': 1, 'pageSize': 12}
# 请求启动时间
start = time.time()
# post请求
r = requests.post(url=url, data=json.dumps(data), headers=headers)
# 请求结束时间
end = time.time()
# 保留两位小数
finall_time = float('%.2f' % float(end - start))
text = json.loads(r.text)
# IO写入 只写入200的
with open('text6.csv', 'a', newline='') as csvfile:
if text['statusCode'] == '200':
throughput = line_num * response_time / finall_time
throughput = float('%.2f' % float(throughput))
print('the perf_counter time of %s is %s and the content is %s ,throughput is %s' % (
tbid, finall_time, json.loads(r.text), throughput))
spamwriter = csv.writer(csvfile, dialect='excel')
spamwriter.writerow([tbid] + [finall_time] + [throughput])
def start_thread(csv_name):
tbid, response_time_sort, throughput_sort = read_csv(csv_name)
print(tbid)
line_num = len(tbid)
response_times = 5
for j in range(response_times):
for i in tbid:
t = threading.Thread(target=post_test, args=(i, line_num, response_times))
t.start()
t.join()
I don't know how to call a method in a class, especially if it has initialization parameters, but you can try this method。 I'm trying to use multiple processes to solve this problem, right。

Why thread is interrupted before the changes are complete

I'm attempting to create python module for getting MAC adresses by IP addresses.
def getMACs(addressesList):
def _processArp(pkt):
spa = _inet_ntoa(pkt.spa)
if pkt.op == dpkt.arp.ARP_OP_REPLY and spa in _cache.macTable:
lock.acquire()
try:
_cache.macTable[spa] = _packedToMacStr(pkt.sha)
_cache.notFilledMacs -= 1
finally:
lock.release()
if _cache.notFilledMacs == 0:
thrd.stop()
addresses = _parseAddresses(addressesList)
_cache.registerCacheEntry("macTable", {})
_cache.registerCacheEntry("notFilledMacs", 0)
_events.arpPacket += _processArp
lock = threading.Lock()
thrd = _CaptureThread(promisc=False, timeout_ms=30, filter="arp")
thrd.start()
for addr in addresses:
if _sendArpQuery(addr):
_cache.macTable[str(addr)] = None
_cache.notFilledMacs += 1
thrd.join(125)
thrd.stop()
return _cache.macTable
if __name__ == "__main__":
macTable = getMACs([IPAddress("192.168.1.1"), IPAddress("192.168.1.3")])
_pprint.pprint(macTable)
When I run this module I get
{'192.168.1.1': '00:11:95:9E:25:B1', '192.168.1.3': None}
When I debug _processArp step by step I get
{'192.168.1.1': '00:11:95:9E:25:B1', '192.168.1.3': '00:21:63:78:98:8E'}
Class CaptureThread:
class CaptureThread(threading.Thread):
def __init__ (self, name=None, snaplen=65535, promisc=True, timeout_ms=0, immediate=False, filter=None):
threading.Thread.__init__(self)
self.__running = True
self.__name = name
self.__snaplen = snaplen
self.__promisc = promisc
self.__timeout_ms = timeout_ms
self.__immediate = immediate
self.__filter = filter
def stop(self):
self.__running = False
def run(self):
self.__pc = pcap.pcap(self.__name, self.__snaplen, self.__promisc, self.__timeout_ms, self.__immediate)
if self.__filter:
self.__pc.setfilter(self.__filter)
while self.__running:
self.__pc.dispatch(1, self.__processPacket)
def __processPacket(self, timestamp, pkt):
peth = dpkt.ethernet.Ethernet(pkt)
if isinstance(peth.data, dpkt.arp.ARP):
_events.arpPacket(peth.data)

Stupid error. As always when working with threads - because of thread synchronization.
One of my conditions for interrupting thread is "_cache.notFilledMacs == 0". In the main thread _cache.notFilledMacs did not have time to get the value of 2 when in the CaptureThread value is decreased.