I am making a File Sharing Program using sockets in python. I wanna show the transfer progress by making use of progress bar in rich. But the progress bar is not properly synced with the transfer progress
sender script-
import socket, os, time
from rich.console import Console
from rich.progress import Progress
HOST = socket.gethostbyname(socket.gethostname())
PORT = 12345
ADDR = (HOST, PORT)
BUFSIZ = 4096
FORMAT = "utf-8"
SEPARATOR = "<SEPARATOR>"
console = Console()
FILENAMES = ["file.txt", "lol.txt"]
FILSIZ = [str(os.path.getsize(x)) for x in FILENAMES]
def send():
"""main function to send files"""
console.clear()
# creating a client socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)
print(client.recv(BUFSIZ).decode(FORMAT))
# sending file data
client.send(SEPARATOR.join(FILENAMES).encode(FORMAT))
print(client.recv(BUFSIZ).decode(FORMAT))
client.send(SEPARATOR.join(FILSIZ).encode(FORMAT))
print(client.recv(BUFSIZ).decode(FORMAT))
# sending files
for idx, files in enumerate(FILENAMES):
with open(files, "rb") as f, Progress() as progress:
task = progress.add_task(f"Sending {files}", total=int(FILSIZ[idx]))
client.send(f.read(int(FILSIZ[idx])))
while not progress.finished:
progress.update(task, advance="<AMOUNT_OF_DATA_OR_CHUNKS_SENT>")
time.sleep(0.1)
f.close()
# closing connection
client.close()
send()
receiver script - https://www.toptal.com/developers/hastebin/avomadisox.py
afaik advance value must be amount of data or chunks sent(might be wrong here)... how do i calculate the amount of data sent?
Rich's progress bars are nice!
For many use-cases, the track function that wraps a Sequence or Iterable will suffice:
import time
from rich.progress import track
for i in track(range(100)):
time.sleep(0.05)
To increment progress by a variable amount at each step, use rich.progress.Progress.
This example might show something in the spirit of the original question. Just for fun, let's customize the progress bar while we're at it.
import time
import random
from rich.progress import (
BarColumn,
Progress,
SpinnerColumn,
TaskProgressColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
def process(chunks):
for chunk in chunks:
time.sleep(0.1)
yield chunk
chunks = [random.randint(1,20) for _ in range(100)]
progress_columns = (
SpinnerColumn(),
"[progress.description]{task.description}",
BarColumn(),
TaskProgressColumn(),
"Elapsed:",
TimeElapsedColumn(),
"Remaining:",
TimeRemainingColumn(),
)
with Progress(*progress_columns) as progress_bar:
task = progress_bar.add_task("[blue]Downloading...", total=sum(chunks))
for chunk in process(chunks):
progress_bar.update(task, advance=chunk)
Note: The generator process(chunks) is a generic stand-in for the file sizes in original question. This answer is mostly for the benefit of those brought here by searching on something like "python rich.progress_bar.ProgressBar example".
Your main question asks for
How to calculate the amount of data sent?
From Real Python's Socket Programming in Python (Guide):
The .send() method also behaves this way. It returns the number of bytes sent, which may be less than the size of the data passed in.
This means that you have to pass the returned int of socket.send() to the parameter advance of progress.update() function (compare your "<AMOUNT_OF_DATA_OR_CHUNKS_SENT>"):
# (1) define the transfer function returning the bytes_sent
def transfer_to_socket(from_file, to_socket, size):
bytes_to_read = int(size)
chunk = from_file.read(bytes_to_read)
# <AMOUNT_OF_DATA_OR_CHUNKS_SENT>
return to_socket.send(chunk) # may be: bytes_sent < len(chunk) < bytes_to_read
# (2) replace the progress-loop with:
while not progress.finished:
bytes_sent = transfer_to_socket(f, client, FILSIZ[idx])
progress.update(task, advance=bytes_sent)
time.sleep(0.1)
Related
I've been sketching some test code for a file transfer using a TCP socket in python (3.9 - Win10 x64). When I run the code below the NIC throughput in task manager increases by about 100Mb/s (goes back down when the socket is broken). The data rates on the hard drives and the measured rate in the code seem to indicate that the actual transfer rate of the intended data is about 11Mb/s. First the code and next some more info about what I have been trying:
Client
import socket, os, sys
from threading import Thread
from time import monotonic
file_send = r'PATH TO FILE'
stats = os.stat(file_send)
print(stats.st_size)
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client_soc:
client_soc.connect(('192.168.1.88', 6325))
client_soc.sendall(str(stats.st_size).encode('utf-8'))
client_soc.recv(10)
buffer_size = 1024 * 32
#client_soc.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, buffer_size)
with open(file_send, 'rb') as f:
bytes_read = 0
s_time = monotonic()
check_in = s_time
while bytes_read < stats.st_size:
read = f.read(1024*1024*10)
mono = monotonic()
t_diff = mono - s_time
if mono - check_in > 10:
print('{:,.0f} Kb/s'.format((bytes_read / 1024) / t_diff))
check_in = mono
client_soc.send(read)
bytes_read += len(read)
print('Done sending')
Server
import socket
from threading import Thread
class DLThread(Thread):
def __init__(self, bind_address, port, file_recv):
super(DLThread, self).__init__()
self.life_switch = True
self.bind_address = bind_address
self.port = port
self.file_recv = file_recv
def run(self) -> None:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listen_doc:
listen_doc.bind((self.bind_address, self.port))
listen_doc.listen()
print('listening')
client_conn, addr = listen_doc.accept()
with client_conn:
print('Connected')
data = client_conn.recv(1024)
size = int(data)
print('File size: {}'.format(size))
client_conn.sendall(b'k')
gotten = 0
percent_inc = 0.1
percent_chunk = percent_inc * size
thresh = percent_inc * size
with open(self.file_recv, 'wb') as f:
while gotten < size:
data = client_conn.recv(1024*1024*5)
f.write(data)
gotten += len(data)
if gotten > thresh:
print('{:.0%} Complete'.format(gotten / size))
thresh = int(gotten // percent_chunk) + percent_chunk
def pull_the_plug(self):
self.life_switch = False
self.join()
bind_addr = input('Bind Address: ')
port = int(input('port: '))
file_location = input('fileloc: ')
dl = DLThread(bind_addr, port, file_location)
dl.start()
print('Started')
dl.join()
I tried changing the buffer sizes ad using send / sendall. I also tried to have the client wait for a (redundant - I think) acknowledgement from the server at each chunk.
I took a look at the traffic in wireshark and I am seeing messages that appear to be padded with a bunch of 0s appended. I am not sure if this is a clue or not.
I can't think of why there would be so much overhead on the network interface. It feels weird just throwing all the data into the socket and letting the API do everything else but this is what many other people have done in their examples.
Sometimes with posts like this people recommend using a different tool. I just want to make it clear that I am not really trying to get this file transfer to work, I just want to understand why this is happening so I can learn something.
Although this is a very specific example of a common mistake, I'll confirm the answer here. Joachim Isaksson is correct. Task Manager reports network traffic in orders of bits/second not bytes/second. This is almost an order of magnitude different.
what learn? Pay attention to units and when things don't line up look for how the data could be related.
I have the following socket listening on my local network:
def recvall(sock):
BUFF_SIZE = 4096 # 4 KiB
fragments = []
while True:
chunk = sock.recv(BUFF_SIZE)
fragments.append(chunk)
# if the following line is removed, data is omitted
time.sleep(0.005)
if len(chunk) < BUFF_SIZE:
break
data = b''.join(fragments)
return data
def main():
pcd = o3d.geometry.PointCloud()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('192.168.0.22', 2525))
print("starting listening...")
s.listen(1)
counter = 0
while True:
clientsocket, address = s.accept()
print(f"Connection from {address} has been established!")
received_data = recvall(clientsocket)
clientsocket.send(bytes(f"response nr {counter}!", "utf-8"))
counter += 1
print(len(received_data))
if __name__ == "__main__":
main()
To this port, I'm sending byte data with a length of 172800 bytes from an app on my mobile phone.
As one can see, I'm printing the amount of data received. The amount is only correct, if I use time.sleep() as shown in the code above. If I don't use this method, only a part of the data is received.
Obviously this is some timing issue, the question is: How can I be sure to receive all the data all the time without using time.sleep() (since this is also not 100% certain to work, depending on the sleeping time set)
sock.recv() returns the data that is available. The relevant piece from the man page of recv(2) is:
The receive calls normally return any data available, up to the requested amount,
rather than waiting for receipt of the full amount requested.
In your case, time.sleep(0.005) seems to allow for all the remaining data of the message to arrive and be stored in the buffer.
There are some options to eliminate the need for time.sleep(0.005). Which one is the most appropriate depends on your needs.
If the sender sends data, but does not expect a response, you can have the sender close the socket after it sends the data, i.e., sock.close() after sock.sendall(). recv() will return an empty string that can be used to break out of the while loop on the receiver.
def recvall(sock):
BUFF_SIZE = 4096
fragments = []
while True:
chunk = sock.recv(BUFF_SIZE)
if not chunk:
break
fragments.append(chunk)
return b''.join(fragments)
If the sender sends messages of fixed length, e.g., 172800 bytes, you can use recv() in a loop until the receiver receives an entire message.
def recvall(sock, length=172800):
fragments = []
while length:
chunk = sock.recv(length)
if not chunk:
raise EOFError('socket closed')
length -= len(chunk)
fragments.append(chunk)
return b''.join(fragments)
Other options include a. adding a delimiter, e.g., a special character that cannot be part of the data, at the end of the messages that the sender sends; the receiver can then run recv() in a loop until it detects the delimiter and b. prefixing the messages on the sender with their length; the receiver will then know how many bytes to expect for each message.
I am sending 32 bytes packets every 1ms to this socket. I wish to print the data after every 40 ms. And apparently the code does that. But even when I stop sending data, I still continue to see that data is being printed.
Is it holding the data in some cache? or simply the python socket has a huge delay? Why?
The code is as follows:
## Import necessary libraries
import math
import numpy as np
import socket
import struct
import time
from synchrophasor.frame import CommandFrame
from datetime import datetime
## Configure socket for Phasor data ##
UDP_IP = "10.10.114.22"
UDP_PORT = 8208 #UDP phasor values 32 bytes (V,phi,P)
sock_ph = socket.socket(socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
sock_ph.bind((UDP_IP, UDP_PORT))
print("socket bound, waiting for data...")
while True:
raw = sock_ph.recv(32)
#print(raw)
mag = struct.unpack('d', raw[8:16])[0]
# print("mag =",mag,type(mag))
angle = struct.unpack('d', raw[16:24])[0]
# print("angle =",angle,type(angle))
header = struct.unpack('d', raw[0:8])[0]
# print("header =",header,type(header))
phasor = (mag, angle)
Vol_A=raw
VA = float(mag)
phi_A = float(angle)
VB = VA
phi_B = phi_A+(math.pi) * 2 / 3
VC = VA
phi_C = phi_A-(math.pi) * 2 / 3
time.sleep(1/25)
# pmu.send_data(phasors=[(VA,phi_A),(VB,phi_B),(VC,phi_C)],analog=[9.91],digital=[0x0001])
#time.sleep(1/config_rr)
print([(VA,phi_A),(VB,phi_B),(VC,phi_C),datetime.now()])
most programs don't want to discard unread datagrams so most OSs will buffer them for you. your case is somewhat unusual so you'd need to write code to handle this case. I'd change your code to do something like:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind(('', 8208))
# block until we read an initial packet
raw = s.recv(1024)
s.setblocking(False)
while True:
# unpack
header, mag, angle = struct.unpack('ddd', raw)
# do something with data
print(f'header={header} mag={mag} angle={angle}')
# sleep for some time
time.sleep(1/25)
# discard any packets you've received in the mean time
while True:
try:
raw = s.recv(1024)
except OSError as err:
# OS buffer is empty: we've therefore got the most recent data
if err.errno == socket.EWOULDBLOCK:
break
# something else failing, reraise the error
raise
note that Steffen Ullrich's suggestion of sending the data at the correct rate would be easier, but assumes that you have control over the sending process. the fact that you said "I am sending" suggests you do, and so would likely make a better solution
***Python code:***
import serial
import pandas as pd
import time
import re
import xlrd
from msvcrt import getch
import numpy as np
i = 0
x = 0
y = 0
df = pd.read_excel(r'C:\Users\lynchfamily\Desktop\mlglovesdata.xls')
# Read COM9
# Read from COM10 as well
# Readline() only works with a timeout (IMPORTANT)
serHC = serial.Serial('COM9', 115200,timeout=.250,parity=serial.PARITY_NONE,rtscts=1) # This is the JY
serRN = serial.Serial('COM10', 115200,timeout=.250,parity=serial.PARITY_NONE,rtscts=1) # This is the silvermate
def serialin():
# Sensor lists
sensor_names = list()
sensor_values = list()
global i
# Read a certain amount of bytes from serial and then continue
# Regular expressions for finding the proper data
while i < 6:
# print(i) for debugging
global serHC
global serRN
#searchObj = re.search(r'(A\d?\d*)?(\d*)?',serHC.read(4).decode(),re.I)
#searchObjRN = re.search(r'(A\d?\d*)?(\d*)?',serRN.read(4).decode(),re.I)
# Serial data stops while in loop
# The if statements keep the false values out of the program
#if searchObj.group(1):
sensor_names.append(serHC.read(2))
#if searchObj.group(2):
sensor_values.append(serHC.read(2))
#if searchObjRN.group(1):
sensor_names.append(serRN.read(2))
#if searchObjRN.group(2):
sensor_values.append(serRN.read(2))
i = i + 1
while 1:
# Get the key from the msvcrt module
key = getch().decode('ASCII')
# If key is pressed, do something
if key:
print(key)
# Zip them together
# Final 2D list
final_2d_list = zip(sensor_names,sensor_values)
print(list(sorted(final_2d_list)))
#vals = df.Dataframe([
#df.append(vals)
#print(sorted_array_1stdim[r])
#sensor_values = [0] * 10
# Thread for reading definition
break
# Fancy recursion
sensor_values.clear()
sensor_names.clear()
i = 0
serialin()
serialin()
Arduino Code:
// The device with green colored wires
void setup() {
Serial.begin(115200);
}
void loop() {
// It won't work with the I2C while loop for some reason. Perhaps it is getting stuck up on it
Serial.print("A4");
Serial.print(analogRead(0)); // Read the local analog signal
delay(5);
Serial.print("A5");
Serial.print(analogRead(1)); // Read the local analog signal
delay(5);
Serial.print("A6");
Serial.print(analogRead(2)); // Read the local analog signal
delay(5);
Serial.print("A7");
Serial.print(analogRead(3)); // Read the local analog signal
}
I'm trying to send analog data from sensors over through bluetooth silver mate from sparkfun, and HC-06 modules to python.
I have to read the analog data at a delay of 5 seconds between each, so that the readings aren't conflicted.
The data comes through serial ports COM9 and COM10. I know that serial in python can be blocking, that's why I attempted to read it first, and then put it in a list.
I also knows that once serial has been read through, it appears to be non-blocking. When I was using serHC.readline() and serRN.readline(), I was getting something like what I'd expect to see.
However, the data in the list were not updating according to the change in the sensors. I have to admit python is not my main programming language, so that is why I'm asking for help.
I thought maybe using multiple threads might work, but I wasn't able to get the serHC and serRN variables in the main thread.
Any help will be appreciated!!
As you have discovered it is not possible to read sequentially from serial ports: a blocking read over one port implies a loss of data simultaneous sent over the other port.
Use a thread based approach.
The following sketch should be enough to get started:
import serial
import time
import re
import threading
BYTES_TO_READ = 6
# read from serial port
def read_from_serial(board, port):
print("reading from {}: port {}".format(board, port))
payload = b''
ser = serial.Serial(port, 115200,timeout=.250, parity=serial.PARITY_NONE, rtscts=1)
bytes_count = 0
while bytes_count < BYTES_TO_READ:
read_bytes = ser.read(2)
# sum number of bytes returned (not 2), you have set the timeout on serial port
# see https://pythonhosted.org/pyserial/pyserial_api.html#serial.Serial.read
bytes_count = bytes_count + len(read_bytes)
payload = payload + read_bytes
# here you have the bytes, do your logic
# ...
print("READ from {}: [{}]".format(board, payload))
return
def main():
board = {
'JY': 'COM9',
'SILVER': 'COM10'
}
threads = []
for b in board:
t = threading.Thread(target=read_from_serial, args=(b, board[b],))
threads.append(t)
t.start()
# wait for all threads termination
for t in threads:
t.join()
main()
For learning about threading: https://pymotw.com/3/threading/
Periodic read from serials
Below a sketch for reading each TIME_PERIOD seconds.
A parte the infinite while loop around the read there is a "thread" loop with a nested try/catch block
for catching serials communication problems and retrying to connect after TIME_PERIOD.
Take it just as a starting example!
import serial
import time
import re
import threading
BYTES_TO_READ = 6
TIME_PERIOD = 5
def read_message(board, port, handle):
payload = b''
bytes_count = 0
while bytes_count < BYTES_TO_READ:
read_bytes = handle.read(2)
bytes_count = bytes_count + len(read_bytes)
payload = payload + read_bytes
# here you have the bytes, do your logic
# ...
print("READ from {}: [{}]".format(board, payload))
def serial_thread(board, port):
print("reading from {}: port {}".format(board, port))
while True:
try:
handle = serial.Serial(port, 115200,timeout=.250, parity=serial.PARITY_NONE, rtscts=1)
while True:
read_message(board, port, handle)
time.sleep(TIME_PERIOD)
except Exception as e:
print("ERROR: {}".format(e))
print("retrying in {} seconds".format(TIME_PERIOD))
handle.close()
time.sleep(TIME_PERIOD)
def main():
board = {
'JY': '/dev/ttyUSB0',
'SILVER': '/dev/ttyACM0'
}
threads = []
for b in board:
t = threading.Thread(target=serial_thread, args=(b, board[b],))
threads.append(t)
t.start()
# wait for all threads termination
for t in threads:
t.join()
main()
I am trying to read from two serial ports at once. Each connected device spits out a line of data. I read the data from each port as a list and then concatenate the list and print it out as one line.
If I read each port individually, the data updates fine. But the second I attempt to read from both, it lags up and the data stops changing in the print output. The timestamp updates fine, but the data itself is what starts to lag.
Below is my code, should I be doing some sort of threading? I am reading from an Arduino and a Teensy.
import serial
import time
serA = serial.Serial('/dev/arduino', 230400)
serT = serial.Serial('/dev/teensy', 9600)
while 1 :
timestamp = "%f" % time.time()
print(timestamp)
arduino = serA.readline().rstrip('\n')
data_listA = arduino.split('$')
teensy = serT.readline().rstrip('\n')
data_listT = teensy.split('$')
data_list = data_listA + data_listT
print(data_list)
just check to see if your serial port has bytes to read before you try to read it ...
while 1 :
timestamp = "%f" % time.time()
print(timestamp)
if serA.inWaiting(): # only read if there is something waiting to be read
arduino = serA.readline().rstrip('\n')
data_listA = arduino.split('$')
print("GOT ARDUINO:",data_listA)
if serB.inWaiting():
teensy = serT.readline().rstrip('\n')
data_listT = teensy.split('$')
print("GOT TEENSY:",data_listT)
Using inwaiting() unfortunately did not work for me. I ended up having to use threading. A basic example for people who might encounter my problem is shown below.
import serial
import Queue
import threading
queue = Queue.Queue(1000)
serA = serial.Serial('/dev/arduino', 230400)
serT = serial.Serial('/dev/teensy', 9600)
def serial_read(s):
while 1:
line = s.readline()
queue.put(line)
threadA = threading.Thread(target=serial_read, args=(serA,),).start()
threadT = threading.Thread(target=serial_read, args=(serT,),).start()
while 1:
line = queue.get(True, 1)
print line
I based my code on the last answer from this question.