I'm trying to send a file data.txt that contains this words 'hello world' over my network using 2 files in Python, one called server.py :
import socket
import tqdm
import os
# device's IP address
SERVER_HOST = "0.0.0.0"
SERVER_PORT = 5001
# receive 4096 bytes each time
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
# create the server socket
# TCP socket
s = socket.socket()
# bind the socket to our local address
s.bind((SERVER_HOST, SERVER_PORT))
# enabling our server to accept connections
# 5 here is the number of unaccepted connections that
# the system will allow before refusing new connections
s.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept connection if there is any
client_socket, address = s.accept()
# if below code is executed, that means the sender is connected
print(f"[+] {address} is connected.")
# receive the file infos
# receive using client socket, not server socket
received = client_socket.recv(BUFFER_SIZE).decode()
filename, filesize = received.split(SEPARATOR)
# remove absolute path if there is
filename = os.path.basename(filename)
# convert to integer
filesize = int(filesize)
# start receiving the file from the socket
# and writing to the file stream
progress = tqdm.tqdm(range(filesize), f"Receiving {filename}", unit="B", unit_scale=True, unit_divisor=1024)
with open(filename, "wb") as f:
while True:
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
# nothing is received
# file transmitting is done
break
# write to the file the bytes we just received
f.write(bytes_read)
# update the progress bar
progress.update(len(bytes_read))
progress.close()
# close the client socket
client_socket.close()
# close the server socket
s.close()
and the other one called client.py :
import socket
import tqdm
import os
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096 # send 4096 bytes each time step
# the ip address or hostname of the server, the receiver
host = "192.168.1.12"
# the port, let's use 5001
port = 5001
# the name of file we want to send, make sure it exists
filename = "data.txt"
# get the file size
filesize = os.path.getsize(filename)
# create the client socket
s = socket.socket()
print(f"[+] Connecting to {host}:{port}")
s.connect((host, port))
print("[+] Connected.")
# send the filename and filesize
s.send(f"{filename}{SEPARATOR}{filesize}".encode())
# start sending the file
progress = tqdm.tqdm(range(filesize), f"Sending {filename}", unit="B", unit_scale=True, unit_divisor=1024)
with open(filename, "rb") as f:
while True:
# read the bytes from the file
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# file transmitting is done
break
# we use sendall to assure transimission in
# busy networks
s.sendall(bytes_read)
# update the progress bar
progress.update(len(bytes_read))
# close the socket
s.close()
Here is what I tried:
I checked the code over and over for any mistake. The problem is that the file is received empty with 0 bytes although it contained a string before sending.
How can I fix this problem, any help or suggestion will be much appreciated.
It seems you're running server and client in the same directory, and the server truncates the file before the client gets to read from it.
it works pefect for me with hello world but if you want to send a binary file maybe you can try base 64
Related
This question already has answers here:
Python TCP Server receives single TCP packet split into multiple packets
(2 answers)
Sending multiple files through a TCP socket
(2 answers)
How does the python socket.recv() method know that the end of the message has been reached?
(1 answer)
How does socket recv function detects end of message
(1 answer)
How to properly use recv function for TCP chat, python3?
(1 answer)
Closed 3 days ago.
So i am trying to build a web chat app that is capable of sending and receiving files. Apparently i am doing all the right steps since i've been reading other people's code but for some reason the server side tends to fail and not read or receive the bytes of data being sent from the client side. i just can't figure out what is causing this issue since it doesn't return any errors whenever I decide to run the code, it just does not write the new file with the data being received. If some cool og programmer can help me it would mean so much.
client.py
import socket
import os
HOST = '127.0.0.1'
PORT = 65432
FORMAT = 'UTF-8'
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((HOST,PORT))
# Declare anything received as the connection status
connection_status = client.recv(1024).decode()
print(connection_status)
# Select Files
files = ["C:/Users/me/mypics/bmw bike.jpg","C:/Users/me/mypics/bmw car.jpg"]
# Files Name
files_name = [os.path.basename(files[0]), os.path.basename(files[1])]
# Files Size
files_size = [os.path.getsize(files[0]), os.path.getsize(files[1])]
# Send files
for file, file_name, file_size in zip(files, files_name, files_size):
# Send file's name and size
client.sendall(f'{file_name}:{file_size}'.encode(FORMAT))
# Open file
with open(file,'rb') as f:
# Declare chunk as 1024 bytes of data to be sent
chunk = f.read(1024)
# Active loop sending chunks of data until the file is complete
while chunk:
# Send chunk
client.sendall(chunk)
# Declare the next 1024 bytes as another chunk until there is none left
chunk = f.read(1024)
# Close file
f.close()
# Close connection
client.close()
server.py
import socket
import os
HOST = ''
PORT = 65432
FORMAT = 'UTF-8'
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((HOST, PORT))
server.listen()
print(f'Listening For Client Connections')
# Accept client connection
client, address = server.accept()
print(f'Connection Established With Client: {address}')
# Inform the client is now connected
client.sendall('You Are Now Connected!'.encode(FORMAT))
# Active loop for incoming files
while True:
# Declare anything received as the file's name and size attributes
file_attributes = client.recv(1024).decode()
# Condition to break loop in case no attributes were received
if not file_attributes:
break
# Split the attributes
file_name, file_size = file_attributes.split(':')
# Convert the file's size into integers
file_size = int(file_size)
# Open file
with open(file_name,'wb') as f:
# Active loop receiving chunks of data until the amount of bytes are completed
while file_size > 0:
# Declare chunk as to the 1024 bytes of data received from the file
chunk = client.recv(1024)
# Write the chunk of data into the new file
f.write(chunk)
# Subtract the number of bytes received from the chunk onto the file's size
file_size -= len(chunk)
# Close file
f.close()
# Close connection
client.close()
File transfer using socket stream is too slow. Almost 100kbps. I used the python socket module to make this code. It sends data to the client when the client sends the file name. How can I increase the speed of this thing?
Below is the server code
import socket
import os
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 9999))
server.listen()
client, addr = server.accept()
msg = client.recv(1024).decode()
file = open("Server_files/"+msg, "rb")
file_size = os.path.getsize("Server_files/"+msg)
# client.send("Received_image.png".encode())
client.send(str(file_size).encode())
data = file.read()
client.sendall(data)
client.send(b"<END>")
file.close()
client.close()
Below is the client code
import tqdm
import socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('localhost', 9999))
file_name = input("Enter the file name: ")
client.send(file_name.encode())
file_size = client.recv(1024).decode()
print(file_size)
file = open(file_name, "wb")
file_bytes = b""
done = False
progress = tqdm.tqdm(unit="B", unit_scale=True, unit_divisor=1000,
total=int(file_size))
while not done:
data = client.recv(1024)
if file_bytes[-5:] == b"<END>":
done = True
else:
file_bytes += data
progress.update(1024)
file.write(file_bytes)
file.close()
client.close()
Instead of continuously adding incoming data to file_bytes (which requires potentially a lot of RAM, and also requires Python to reallocate the buffer larger multiple times), you should just write the incoming data directly to file as you receive it.
Also it might help to increase your chunk size from 1024 to something larger, maybe 128*1024; that would allow the system calls to do more work in each call.
Here's a complete solution (Python 3.10) that allows multiple files to be downloaded rapidly. It uses socket.makefile to buffer data, with to automatically close sockets, files, and flush tqdm progress, and writes data as received for speed.
Note that TCP is not a messaged-based protocol. It sends bytes and receives bytes in the same order, but not necessarily with the same "breaks". So send('filename') and send('filesize') can be recv(1024) as 'filenamefilesize' or 'filenamefi' and 'lesize' or any other break. So define a protocol:
Server:
read filename followed by '\n'.
If received 0 bytes, connection was closed.
send filesize as a string in base 10 followed by '\n'.
send exactly "filesize" bytes of file data.
repeat.
Client:
send filename followed by '\n'
read filesize followed by '\n', convert to integer of base 10.
read exactly "filesize" bytes of file data
if done, close connection, else repeat.
server.py
import socket
import os
BLOCK = 128 << 10 # 128KB
with socket.socket() as server:
server.bind(('', 9999))
server.listen()
while True:
client, addr = server.accept()
try:
with (client,
client.makefile('rb') as rfile,
client.makefile('wb') as wfile):
while True:
filename = rfile.readline()
if not filename: break
fullname = os.path.join('Server_files', filename.decode().rstrip('\n'))
file_size = os.path.getsize(fullname)
wfile.write(f'{file_size}\n'.encode())
print(f'Sending {fullname}...')
with open(fullname, 'rb') as file:
while data := file.read(BLOCK):
wfile.write(data)
wfile.flush() # make sure anything remaining in makefile buffer is sent.
print(f' Complete ({file_size} bytes).')
except ConnectionResetError:
print('Client aborted.')
client.py
import socket
import tqdm
BLOCK = 1 << 20 # 1MB
with socket.socket() as client:
client.connect(('localhost', 9999))
with (client.makefile('rb') as rfile,
client.makefile('wb') as wfile):
while True:
file_name = input('File name (just ENTER to quit): ')
if not file_name: break
wfile.write(f'{file_name}\n'.encode())
wfile.flush() # make sure makefile buffer is fully sent
file_size = int(rfile.readline().decode())
with (tqdm.tqdm(unit='B',
unit_scale=True,
unit_divisor=1000,
total=file_size) as progress,
open(file_name, 'wb') as file):
remaining = file_size
while remaining:
data = rfile.read(BLOCK if remaining > BLOCK else remaining)
file.write(data)
progress.update(len(data))
remaining -= len(data)
Output example (note download speeds):
File name (just ENTER to quit): bigfile1
100%|██████████████████████████████████████████████████████████| 191M/191M [00:00<00:00, 706MB/s]
File name (just ENTER to quit): bigfile2
100%|██████████████████████████████████████████████████████████| 218M/218M [00:00<00:00, 718MB/s]
File name (just ENTER to quit):
I have one simple question: Can I sniff the mirrored packets as if I am accepting python socket?
I am receiving a file on Server A by using get_file_by_socket.py :
import socket
import tqdm
import os
import hashlib
import time
SERVER_HOST = "192.168.1.1"
SERVER_PORT = 5201
counter = 1
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
s = socket.socket()
s.bind((SERVER_HOST, SERVER_PORT))
s.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
client_socket, address = s.accept()
print("client_scoket = ",client_socket,address)
print(f"[+] {address} is connected")
received = client_socket.recv(BUFFER_SIZE).decode()
filename,filesize = received.split(SEPARATOR)
filename = os.path.basename(filename)
filesize = int(filesize)
file_hash = hashlib.md5()
progress = tqdm.tqdm(range(filesize), f"Receiving {filename}", unit="B",unit_scale=True, unit_divisor=1024)
with open(filename,"wb") as f:
while True:
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
break
f.write(bytes_read)
file_hash.update(bytes_read)
print(f"{counter}. Bytes_read={bytes_read}")
#print(f"{counter}. ")
counter = counter + 1
time.sleep(0.001)
progress.update(len(bytes_read))
client_socket.close()
s.close()
I am sending the file using send_file_by_socket.py from Host B :
import socket
import tqdm
import os
import sys
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096
host = sys.argv[1] #"192.168.1.1"
print("host=",host)
port = 5201
filename = sys.argv[2] #"twibot20.json"
print("filename=",filename)
filesize = os.path.getsize(filename)
s = socket.socket()
#s.setsockopt(socket.SOL_SOCKET,25,'enp2s0')
print(f"[+] Connecting to {host}:{port}")
s.connect((host,port))
print("[+] Connected.")
s.send(f"{filename}{SEPARATOR}{filesize}".encode())
progress = tqdm.tqdm(range(filesize), f"Sending {filename}", unit="B", unit_scale = True, unit_divisor=1024)
with open(filename, "rb") as f:
while True :
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
break
s.sendall(bytes_read)
progress.update(len(bytes_read))
s.close()
The sender sends the file and the server receives it successfully. The transfer rate is quite high. Now I am mirroring the packets while the transfer is happening. I sniff the packets using sniff_mirrored_packets.py :
def get_if():
ifs=get_if_list()
iface=None
for i in get_if_list():
if "enp1s0f1" in i:
iface=i
break;
if not iface:
print("Cannot find eth0 interface")
exit(1)
return iface
def handle_pkt(pkt):
global file_hash
global counter
try :
setir = pkt[IP].load
except :
setir = ""
if "<SEPARATOR>" in str(setir):
setir = ""
if setir!="" :
file_hash.update(setir)
print("{}. Hash = {} ".format(counter,file_hash.hexdigest()))
#pkt.show2()
sys.stdout.flush()
counter = counter +1
def main():
ifaces = [i for i in os.listdir('/sys/class/net/') ]
iface = get_if()
print(("sniffing on %s" % iface))
sys.stdout.flush()
sniff(filter='tcp and port 5201',iface = iface,
prn = lambda x: handle_pkt(x))
if __name__ == '__main__':
main()
The problem is socket transfer rate is too high, that's why I included :
time.sleep(0.001)
on get_file_by_socket.py on the server-side, since sniffing speed on the mirror side is too slow. When I send a 3MB file from Host B, I get around 200 out of 1000 packets in the mirror-side using the tshark. When I time.sleep(0.001) on the server-side, only then I do receive all 1000 packets on the mirror-side.
My questions are:
How do I get transfer data from mirrored port without establishing TCP/IP handshake python socket? Can I get the mirrored packets the same as on get_file_by_socket.py by ignoring TCP handshake which is happening between Host B and Server A. (I implemented get_file_by_socket.py like code on the mirror-side but it stuck in handshake because the mirrored packets don't have any handshake in it). The sniffing method that I am using is too slow in comparison with socket transfer rate.
What other methods can be used to catch up with the socket transfer rate?
In the beginning, I couldn’t capture all the packets mirrored since some packets were dropped in the kernel. I could sniff it only after increasing the kernel buffer size by:
sudo tcpdump -s 65535 -v -i enp1s0f1 -nn -B 424096 -w dtn4.pcap
After capturing all the packets I am post-processing to build the final version of the text file.
Regarding sniffing the socket without handshake, I couldn't find anything.
I am developing a local network file sharing app in Python with PyQt5 (QtDesigner) and when I launch the reciever (destination host) app, it starts to listen for connection and therefore freezes. How can I make the app usable while the script listens for connections?
Here's my the code for the reciever app.
def listen(s):
ui.connection_bar.setStyleSheet("background-color: #328940")
# Device's IP adress
SERVER_HOST = "0.0.0.0"
SERVER_PORT = 5001
# Recieve 4096 bytes each time
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
# Bind the socket to our local address
s.bind((SERVER_HOST, SERVER_PORT))
# Enabling our server to accept connections
# 5 is the number of unaccepted connections that
# the system will allow before refusing new connections
s.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# Accept connection if there's any
client_socket, address = s.accept()
print(f"[+] {address} is connected")
# Recieve the files' infos
recieved = client_socket.recv(BUFFER_SIZE).decode()
filename, filesize = recieved.split(SEPARATOR)
# Remove absolute path if there's
filename = os.path.basename(filename)
filesize = int(filesize)
# Start recieving the file from the socket
# and writing to the file stream
# progress = tqdm.tqdm(range(filesize), f"Recieving {filename}", unit_scale=True, unit_divisor=1024)
with open (filename, "wb") as f:
while True:
# Recieve 1024 bytes from the socket
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
# Transmittion completed
break
f.write(bytes_read)
# progress.update(len(bytes_read))
# Close the client and server sockets
client_socket.close()
s.close()
if __name__ == '__main__':
#qt mainloop
I have a problem when I try to send an Acknowledgment after receiving data.
To resume the situation, I have a client that have a file to send to my server that is listening. When the server receive this file it returns an acknowledgment file. And here is my problem. When my server send the ACK file my client don't get it and just stand here and don't do anything else.
server.py
import socket
import signal
import os
# "all" interfaces
SERVER_HOST = "0.0.0.0"
SERVER_PORT = 8000
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {3:'HOSPITAL', 4:'RESULTS', 5:'LAB400', 6:'RESULTS',7:'20201030084524',9:'ACK', 10:'1452357', 11:'P',12:'2.2', 15:'AL', 16:'NE', 17:'CHE'}
MSA = {1:'AA', 2:'1722178'}
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE).decode()
filename, fileSize = received.split(SEPARATOR)
#convert to integer
fileSize = int(fileSize)
# remove absolute path if there is
filename = os.path.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(f".\\Files\\{filename}", "wb") as f:
while True:
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
# file transmitting is done
print(f"[+] File transfert is done")
print(f"[+] File saved")
break
# write to the file the bytes we just received
f.write(bytes_read)
myfile = open(".\\Files\\ack.hl7", "rb")
client_socket.send(myfile.read())
client_socket.close()
socket.close
client.py
import socket
import os
import random
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
# fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
fileName = f".\\ClientFiles\\{files[0]}"
filesize = os.path.getsize(fileName)
socket = socket.socket()
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
data = socket.recv(BUFFER_SIZE)
print(data)
print("[*] Closing")
socket.close()
I cannot figure out why it doesn't work
Here is my files that need to be transferred(it's some HL7 V2.5.1 btw)
Ack.hl7
MSH|^~\&|HOSPITAL|RESULTS|LAB400|RESULTS|20201030084524||ACK|1452357|P|2.2
MSA|AA|1722178
testFile.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
What I've tried so far
On my client
I've tried to recreate a socket in the case that it wasn't able to respond
try:
data = socket.recv(BUFFER_SIZE)
print(data)
except:
# recreate the socket and reconnect
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.connect(HOST, PORT)
data = socket.recv(BUFFER_SIZE)
print(data)
I could reproduce and understand your problem.
The underlying cause is that TCP is a stream protocol. That means that the only guarantee is that all the bytes that are sent from one side will the received in the same ordre by the peer. But packets mays be concatenated or splitted by any element on the network, including the protocol stack of either side.
So in server.py, this is wrong:
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE).decode()
filename, fileSize = received.split(SEPARATOR)
#convert to integer
fileSize = int(fileSize)
because received can contain the beginning of the data...
The robust way would be:
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
fileSize = bytes(itertools.takewhile(
lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
And you will have the beginning of the data (as bytes) in bytes_read and should write it immediately:
# start receiving the file from the socket and writing to the file stream
with open(f".\\Files\\{filename}", "wb") as f:
f.write(bytes_read)
while True:
...
But that is not all. Still in server.py the reading loop will not return an empty buffer until the peer closes or better shutdowns the socket.
So in client.py you must signal the end of transmission:
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR) # tells peer that nothing will be sent any more
data = socket.recv(BUFFER_SIZE)