Develop Reference

Speed up file transfer using socket

File transfer using socket stream is too slow. Almost 100kbps. I used the python socket module to make this code. It sends data to the client when the client sends the file name. How can I increase the speed of this thing?
Below is the server code
import socket
import os
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 9999))
server.listen()
client, addr = server.accept()
msg = client.recv(1024).decode()
file = open("Server_files/"+msg, "rb")
file_size = os.path.getsize("Server_files/"+msg)
# client.send("Received_image.png".encode())
client.send(str(file_size).encode())
data = file.read()
client.sendall(data)
client.send(b"<END>")
file.close()
client.close()
Below is the client code
import tqdm
import socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('localhost', 9999))
file_name = input("Enter the file name: ")
client.send(file_name.encode())
file_size = client.recv(1024).decode()
print(file_size)
file = open(file_name, "wb")
file_bytes = b""
done = False
progress = tqdm.tqdm(unit="B", unit_scale=True, unit_divisor=1000,
total=int(file_size))
while not done:
data = client.recv(1024)
if file_bytes[-5:] == b"<END>":
done = True
else:
file_bytes += data
progress.update(1024)
file.write(file_bytes)
file.close()
client.close()

Instead of continuously adding incoming data to file_bytes (which requires potentially a lot of RAM, and also requires Python to reallocate the buffer larger multiple times), you should just write the incoming data directly to file as you receive it.
Also it might help to increase your chunk size from 1024 to something larger, maybe 128*1024; that would allow the system calls to do more work in each call.

Here's a complete solution (Python 3.10) that allows multiple files to be downloaded rapidly. It uses socket.makefile to buffer data, with to automatically close sockets, files, and flush tqdm progress, and writes data as received for speed.
Note that TCP is not a messaged-based protocol. It sends bytes and receives bytes in the same order, but not necessarily with the same "breaks". So send('filename') and send('filesize') can be recv(1024) as 'filenamefilesize' or 'filenamefi' and 'lesize' or any other break. So define a protocol:
Server:
read filename followed by '\n'.
If received 0 bytes, connection was closed.
send filesize as a string in base 10 followed by '\n'.
send exactly "filesize" bytes of file data.
repeat.
Client:
send filename followed by '\n'
read filesize followed by '\n', convert to integer of base 10.
read exactly "filesize" bytes of file data
if done, close connection, else repeat.
server.py
import socket
import os
BLOCK = 128 << 10 # 128KB
with socket.socket() as server:
server.bind(('', 9999))
server.listen()
while True:
client, addr = server.accept()
try:
with (client,
client.makefile('rb') as rfile,
client.makefile('wb') as wfile):
while True:
filename = rfile.readline()
if not filename: break
fullname = os.path.join('Server_files', filename.decode().rstrip('\n'))
file_size = os.path.getsize(fullname)
wfile.write(f'{file_size}\n'.encode())
print(f'Sending {fullname}...')
with open(fullname, 'rb') as file:
while data := file.read(BLOCK):
wfile.write(data)
wfile.flush() # make sure anything remaining in makefile buffer is sent.
print(f' Complete ({file_size} bytes).')
except ConnectionResetError:
print('Client aborted.')
client.py
import socket
import tqdm
BLOCK = 1 << 20 # 1MB
with socket.socket() as client:
client.connect(('localhost', 9999))
with (client.makefile('rb') as rfile,
client.makefile('wb') as wfile):
while True:
file_name = input('File name (just ENTER to quit): ')
if not file_name: break
wfile.write(f'{file_name}\n'.encode())
wfile.flush() # make sure makefile buffer is fully sent
file_size = int(rfile.readline().decode())
with (tqdm.tqdm(unit='B',
unit_scale=True,
unit_divisor=1000,
total=file_size) as progress,
open(file_name, 'wb') as file):
remaining = file_size
while remaining:
data = rfile.read(BLOCK if remaining > BLOCK else remaining)
file.write(data)
progress.update(len(data))
remaining -= len(data)
Output example (note download speeds):
File name (just ENTER to quit): bigfile1
100%|██████████████████████████████████████████████████████████| 191M/191M [00:00<00:00, 706MB/s]
File name (just ENTER to quit): bigfile2
100%|██████████████████████████████████████████████████████████| 218M/218M [00:00<00:00, 718MB/s]
File name (just ENTER to quit):

sending multiple images using socket python get sent as one to client

I am capturing screenshots from the server, then sending it to the client, but the images get all sent as one big file to the client that keeps expanding in size. This only happens when i send from one machine to another (I am working on a local netwrok) but when running both client and server from my machine they work fine.
Note: for the client on the other machine, I packaged it into an exe using pyinstaller, since this machine does not have python.
server code:
host="192.168.43.79" # Set the server address to variable host
port=4446 # Sets the variable port to 4446
import time
import pyautogui
from socket import *
import os
s=socket(AF_INET, SOCK_STREAM)
s.bind((host,port))
print("Listening for connections.. ")
q,addr=s.accept()
i = 0
while True:
screenshot = pyautogui.screenshot()
screenshot.save(str(i) + ".jpg")
with open(str(i) + ".jpg", "rb") as f:
data = f.read(4096)
while data:
q.send(data)
data = f.read(4096)
q.send(b"full")
i += 1
time.sleep(0.3)
client code:
host="192.168.43.79" # Set the server address to variable host
port=4446 # Sets the variable port to 4446
from multiprocessing.reduction import recv_handle
from socket import * # Imports socket module
s=socket(AF_INET, SOCK_STREAM) # Creates a socket
s.connect((host,port))
i = 0
while True:
with open(str(i) + "s.jpg", "wb") as f:
recv_data = s.recv(4096)
while recv_data:
f.write(recv_data)
recv_data = s.recv(4096)
if(recv_data == b"full"):
break
i += 1

There various wrong assumptions here which lead to the problem you see. The wrong assumptions are:
that send(data) will write all data It might send less. You need to check the return value or use sendall to be sure.
that a single send in the sender is matched by exactly a single recv in the recipientTCP is only an unstructured byte stream. send does not add message semantics, so a single send might lead to multiple recv, multiple send might lead to a single recv etc. Specifically send("data") followed by send("full") might be recv(4096) as "datafull", thus missing your code to detect end of image.
As for why does it work on the local machine but not on the remote - the chance in the latter case is higher that send get combined together and recv as one.

As stated by Steffen Ulrich you should use sendall for sending and for receiving we create a specialized function my_recv that will repeatedly call socket.recv until the expected number of bytes have been received. Also, a 4-byte header (you can make the length greater if your file sizes warrant this) that contains a binary representation of the file length precedes the sending of the actual file data. In this way the client knows exactly how much data it should receive for each file.
Server Code
host="192.168.43.79" # Set the server address to variable host
port=4446 # Sets the variable port to 4446
import time
import pyautogui
from socket import *
import os
s=socket(AF_INET, SOCK_STREAM)
s.bind((host,port))
s.listen(1) # This should be called
print("Listening for connections.. ")
q,addr=s.accept()
i = 0
while True:
screenshot = pyautogui.screenshot()
screenshot.save(str(i) + ".jpg")
with open(str(i) + ".jpg", "rb") as f:
# Get length by positioning to end of file
image_length = f.seek(0, 2)
f.seek(0, 0) # Seek back to beginning of file
# Convert image length to a 4-byte array:
image_length_bytes = image_length.to_bytes(4, 'big')
q.sendall(image_length_bytes)
data = f.read(4096)
while len(data):
q.sendall(data)
data = f.read(4096)
i += 1
Client Code
host="192.168.43.79" # Set the server address to variable host
port=4446 # Sets the variable port to 4446
from multiprocessing.reduction import recv_handle
from socket import * # Imports socket module
s=socket(AF_INET, SOCK_STREAM) # Creates a socket
s.connect((host,port))
def my_recv(msg_length):
chunks = []
bytes_to_recv = msg_length
while bytes_to_recv:
chunk = s.recv(bytes_to_recv)
if chunk == b'':
raise RuntimeError("socket connection broken")
chunks.append(chunk)
bytes_to_recv -= len(chunk)
return b''.join(chunks)
i = 0
while True:
image_length_bytes = my_recv(4)
image_length = int.from_bytes(image_length_bytes, 'big')
with open(str(i) + "s.jpg", "wb") as f:
bytes_to_recv = image_length
while bytes_to_recv:
recv_data = my_recv(min(4096, bytes_to_recv))
f.write(recv_data)
bytes_to_recv -= len(recv_data)
i += 1

I can not get the data from my server after having sent a file thought sockets

I have a problem when I try to send an Acknowledgment after receiving data.
To resume the situation, I have a client that have a file to send to my server that is listening. When the server receive this file it returns an acknowledgment file. And here is my problem. When my server send the ACK file my client don't get it and just stand here and don't do anything else.
server.py
import socket
import signal
import os
# "all" interfaces
SERVER_HOST = "0.0.0.0"
SERVER_PORT = 8000
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {3:'HOSPITAL', 4:'RESULTS', 5:'LAB400', 6:'RESULTS',7:'20201030084524',9:'ACK', 10:'1452357', 11:'P',12:'2.2', 15:'AL', 16:'NE', 17:'CHE'}
MSA = {1:'AA', 2:'1722178'}
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE).decode()
filename, fileSize = received.split(SEPARATOR)
#convert to integer
fileSize = int(fileSize)
# remove absolute path if there is
filename = os.path.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(f".\\Files\\{filename}", "wb") as f:
while True:
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
# file transmitting is done
print(f"[+] File transfert is done")
print(f"[+] File saved")
break
# write to the file the bytes we just received
f.write(bytes_read)
myfile = open(".\\Files\\ack.hl7", "rb")
client_socket.send(myfile.read())
client_socket.close()
socket.close
client.py
import socket
import os
import random
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
# fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
fileName = f".\\ClientFiles\\{files[0]}"
filesize = os.path.getsize(fileName)
socket = socket.socket()
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
data = socket.recv(BUFFER_SIZE)
print(data)
print("[*] Closing")
socket.close()
I cannot figure out why it doesn't work
Here is my files that need to be transferred(it's some HL7 V2.5.1 btw)
Ack.hl7
MSH|^~\&|HOSPITAL|RESULTS|LAB400|RESULTS|20201030084524||ACK|1452357|P|2.2
MSA|AA|1722178
testFile.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
What I've tried so far
On my client
I've tried to recreate a socket in the case that it wasn't able to respond
try:
data = socket.recv(BUFFER_SIZE)
print(data)
except:
# recreate the socket and reconnect
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.connect(HOST, PORT)
data = socket.recv(BUFFER_SIZE)
print(data)

I could reproduce and understand your problem.
The underlying cause is that TCP is a stream protocol. That means that the only guarantee is that all the bytes that are sent from one side will the received in the same ordre by the peer. But packets mays be concatenated or splitted by any element on the network, including the protocol stack of either side.
So in server.py, this is wrong:
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE).decode()
filename, fileSize = received.split(SEPARATOR)
#convert to integer
fileSize = int(fileSize)
because received can contain the beginning of the data...
The robust way would be:
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
fileSize = bytes(itertools.takewhile(
lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
And you will have the beginning of the data (as bytes) in bytes_read and should write it immediately:
# start receiving the file from the socket and writing to the file stream
with open(f".\\Files\\{filename}", "wb") as f:
f.write(bytes_read)
while True:
...
But that is not all. Still in server.py the reading loop will not return an empty buffer until the peer closes or better shutdowns the socket.
So in client.py you must signal the end of transmission:
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR) # tells peer that nothing will be sent any more
data = socket.recv(BUFFER_SIZE)

How to send file in python over network without loosing data?

I'm trying to send a file data.txt that contains this words 'hello world' over my network using 2 files in Python, one called server.py :
import socket
import tqdm
import os
# device's IP address
SERVER_HOST = "0.0.0.0"
SERVER_PORT = 5001
# receive 4096 bytes each time
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
# create the server socket
# TCP socket
s = socket.socket()
# bind the socket to our local address
s.bind((SERVER_HOST, SERVER_PORT))
# enabling our server to accept connections
# 5 here is the number of unaccepted connections that
# the system will allow before refusing new connections
s.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept connection if there is any
client_socket, address = s.accept()
# if below code is executed, that means the sender is connected
print(f"[+] {address} is connected.")
# receive the file infos
# receive using client socket, not server socket
received = client_socket.recv(BUFFER_SIZE).decode()
filename, filesize = received.split(SEPARATOR)
# remove absolute path if there is
filename = os.path.basename(filename)
# convert to integer
filesize = int(filesize)
# start receiving the file from the socket
# and writing to the file stream
progress = tqdm.tqdm(range(filesize), f"Receiving {filename}", unit="B", unit_scale=True, unit_divisor=1024)
with open(filename, "wb") as f:
while True:
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
# nothing is received
# file transmitting is done
break
# write to the file the bytes we just received
f.write(bytes_read)
# update the progress bar
progress.update(len(bytes_read))
progress.close()
# close the client socket
client_socket.close()
# close the server socket
s.close()
and the other one called client.py :
import socket
import tqdm
import os
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096 # send 4096 bytes each time step
# the ip address or hostname of the server, the receiver
host = "192.168.1.12"
# the port, let's use 5001
port = 5001
# the name of file we want to send, make sure it exists
filename = "data.txt"
# get the file size
filesize = os.path.getsize(filename)
# create the client socket
s = socket.socket()
print(f"[+] Connecting to {host}:{port}")
s.connect((host, port))
print("[+] Connected.")
# send the filename and filesize
s.send(f"{filename}{SEPARATOR}{filesize}".encode())
# start sending the file
progress = tqdm.tqdm(range(filesize), f"Sending {filename}", unit="B", unit_scale=True, unit_divisor=1024)
with open(filename, "rb") as f:
while True:
# read the bytes from the file
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# file transmitting is done
break
# we use sendall to assure transimission in
# busy networks
s.sendall(bytes_read)
# update the progress bar
progress.update(len(bytes_read))
# close the socket
s.close()
Here is what I tried:
I checked the code over and over for any mistake. The problem is that the file is received empty with 0 bytes although it contained a string before sending.
How can I fix this problem, any help or suggestion will be much appreciated.

It seems you're running server and client in the same directory, and the server truncates the file before the client gets to read from it.

it works pefect for me with hello world but if you want to send a binary file maybe you can try base 64

Python : How to handle multiple clients and a server

I am implementing a program with a server and multiple clients. All clients send data to the server and a server check out the step of each client. If all client's steps are the same, a server sends new data to all clients to do next step. And it continues this procedure again and again.
However, when I run my program, it cannot communicate each other. Here are my code. Would you give me some hints?
client & server
#client
from socket import *
from sys import *
import time
import stat, os
import glob
# set the socket parameters
host = "localhost"
port = 21567
buf = 1024
data = ''
addr = (host, port)
UDPSock = socket(AF_INET, SOCK_DGRAM)
UDPSock.settimeout(100)
def_msg = "=== TEST ==="
#FILE = open("test.jpg", "w+")
FILE = open("data.txt","w+")
while (1):
#data, addr = UDPSock.recvfrom(buf)
print "Sending"
UDPSock.sendto(def_msg, addr)
#time.sleep(3)
data, addr = UDPSock.recvfrom(buf)
if data == 'done':
FILE.close()
break
FILE.write(data)
print "Receiving"
#time.sleep(3)
UDPSock.close()
# server program for nvt
from socket import *
import os, sys, time, glob
#import pygame
import stat
host = 'localhost'
port = 21567
buf = 1024
addr = (host, port)
print 'test server'
UDPSock = socket(AF_INET, SOCK_DGRAM)
UDPSock.bind(addr)
msg = "send txt file to all clients"
#FILE = open("cam.jpg", "r+")
FILE = open("dna.dat","r+")
sending_data = FILE.read()
FILE.close()
tmp_data = sending_data
while (1):
#UDPSock.listen(1)
#UDPSock.sendto(msg, addr)
#FILE = open("gen1000.dat","r+")
#sending_data = FILE.read()
#FILE.close()
#print 'client is at', addr
data, addr = UDPSock.recvfrom(buf)
#time.sleep(3)
print data
#msg = 'hello'
#
tmp, sending_data = sending_data[:buf-6], sending_data[buf-6:]
if len(tmp) < 1:
msg = 'done'
UDPSock.sendto(msg, addr)
print "finished"
sending_data = tmp_data
UDPSock.sendto(tmp, addr)
print "sending"
#time.sleep(3)
UDPSock.close()

A server must perform the sequence socket(), bind(), listen(), accept() (possibly repeating the accept() to service more than one client), while a client only needs the sequence socket(), connect().
Your missing listen() i saw first. Listen for connections made to the socket.
More on this here: link text

Look at this: http://heather.cs.ucdavis.edu/~matloff/Python/PyNet.pdf
It's a very good Python networking tutorial including working examples of a client and server. Now, I'm not an expert on this, but it looks to me like your code is overcomplicated. And what's the deal with all the commented-out lines?
Quote from question:
#UDPSock.listen(1)
#UDPSock.sendto(msg, addr)
#FILE = open("gen1000.dat","r+")
#sending_data = FILE.read()
#FILE.close()
End quote
Those look like some pretty important lines to me.
Also, make sure the computers are connected. From a prompt run:
ping [IP]
where [IP] is the IP address of the other machine(Note: if you're not connected to the same LAN, this becomes much harder as you might then need port forwarding and possibly static IPs).