This is my server:
//server.py
import sys
import socket
HOST = sys.argv[1] if len(sys.argv) > 1 else '0.0.0.0'
PORT = int(sys.argv[2] if len(sys.argv) > 2 else 5555)
SIZE = 1024
FORMAT = "utf-8"
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET,
socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
s.listen(1)
client = s.accept()
while True:
print(f"\033[33m[*] Listening as {HOST}:{PORT}\033[m")
print(f"\033[32m[!] Client connected {client[1]}\033[m")
client[0].send('copy trash'.encode())
filename = client[0].recv(SIZE).decode(FORMAT)
print(f"[RECV] Receiving the filename.")
file = open(filename, "w")
client[0].send("Filename received.".encode(FORMAT))
data = client[0].recv(SIZE).decode(FORMAT)
print(f"[RECV] Receiving the file data.")
file.write(data)
client[0].send("File data received".encode(FORMAT))
s.close()
This is my client:
//client.py
import sys
import socket
import subprocess
import tqdm
import shutil
HOST = sys.argv[1] if len(sys.argv) > 1 else '0.0.0.0'
PORT = int(sys.argv[2] if len(sys.argv) > 2 else 5555)
FORMAT = "utf-8"
SIZE = 1024
s = socket.socket()
s.connect((HOST, PORT))
msg = s.recv(1024).decode()
print('[*] server:', msg)
if(msg.lower() == "copy trash"):
shutil.make_archive("/root/trashcopy", 'zip', "/root/.local/share/Trash")
file = open("/root/trashcopy.zip", "r")
data = file.read()
s.send("trashcopy.zip".encode(FORMAT))
msg = s.recv(SIZE).decode(FORMAT)
print(f"[SERVER]: {msg}")
s.send(data.encode(FORMAT))
msg = s.recv(SIZE).decode(FORMAT)
print(f"[SERVER]: {msg}")
What am I trying to do?
I'm trying to make a zip file with files from the recycle bin and send it to the server, however, there's a problem with encoding and an error is thrown at this line:
s.send(data.encode(FORMAT))
This is the error:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x95 in position 124: invalid start byte
How can I fix this? When it's a .txt file, for example, I can send the file without problems.
It's a decoding problem, I've tried to decode in other formats besides utf-8 but it didn't work.
There were several issues with your code. For example, when reading the zip (binary) file, you should read it as bytes, not as string, and send it as bytes. Also, your server needs to know in advance the expected file size to be able to read the file in chunks. You can do that using byte ordering. Below is a working example. Credits go to this for the data receiving approach, which was slightly modified to read the data in chunks, as your code requires.
EDIT: The server has been modified to support reconnection from the client. Thus, client = s.accept() has been moved to inside the while loop and client's connection gets closed after the file has been received.
server.py
import sys
import socket
HOST = sys.argv[1] if len(sys.argv) > 1 else '0.0.0.0'
PORT = int(sys.argv[2] if len(sys.argv) > 2 else 5555)
SIZE = 1024
BYTEORDER_LENGTH = 8
FORMAT = "utf-8"
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST, PORT))
s.listen(1)
while True:
client = s.accept()
print(f"\033[33m[*] Listening as {HOST}:{PORT}\033[m")
print(f"\033[32m[!] Client connected {client[1]}\033[m")
print(f"Sending 'copy trash' msg")
client[0].send('copy trash'.encode())
print(f"[RECV] Receiving the file size")
file_size_in_bytes = client[0].recv(BYTEORDER_LENGTH)
file_size= int.from_bytes(file_size_in_bytes, 'big')
print("File size received:", file_size, " bytes")
client[0].send("File size received.".encode(FORMAT))
print(f"[RECV] Receiving the filename.")
filename = client[0].recv(SIZE).decode(FORMAT)
print(f"[RECV]Filename received:", filename)
client[0].send("Filename received.".encode(FORMAT))
print(f"[RECV] Receiving the file data.")
# Until we've received the expected amount of data, keep receiving
packet = b"" # Use bytes, not str, to accumulate
while len(packet) < file_size:
if(file_size - len(packet)) > SIZE: # if remaining bytes are more than the defined chunk size
buffer = client[0].recv(SIZE) # read SIZE bytes
else:
buffer = client[0].recv(file_size - len(packet)) # read remaining number of bytes
if not buffer:
raise Exception("Incomplete file received")
packet += buffer
with open(filename, 'wb') as f:
f.write(packet)
print(f"[RECV] File data received.")
client[0].send("File data received".encode(FORMAT))
client[0].close()
s.close()
client.py
import sys
import socket
import subprocess
import shutil
import os
HOST = sys.argv[1] if len(sys.argv) > 1 else '127.0.0.1'
PORT = int(sys.argv[2] if len(sys.argv) > 2 else 5555)
FORMAT = "utf-8"
SIZE = 1024
BYTEORDER_LENGTH = 8
s = socket.socket()
s.connect((HOST, PORT))
msg = s.recv(SIZE).decode()
print('[*] server:', msg)
if(msg.lower() == "copy trash"):
#shutil.make_archive("/root/trashcopy", 'zip', "/root/.local/share/Trash")
shutil.make_archive("_trashcopy", 'zip', "Trash")
file_size = os.path.getsize('_trashcopy.zip')
print("File Size is :", file_size, "bytes")
file_size_in_bytes = file_size.to_bytes(BYTEORDER_LENGTH, 'big')
print("Sending the file size")
s.send(file_size_in_bytes)
msg = s.recv(SIZE).decode(FORMAT)
print(f"[SERVER]: {msg}")
print("Sending the file name")
s.send("trashcopy.zip".encode(FORMAT))
msg = s.recv(SIZE).decode(FORMAT)
print(f"[SERVER]: {msg}")
print("Sending the file data")
with open ('_trashcopy.zip','rb') as f1:
s.send(f1.read())
msg = s.recv(SIZE).decode(FORMAT)
print(f"[SERVER]: {msg}")
s.close()
Related
I have a server that's always listening and waiting for a connection. When a client connect then sends a file to it, it receives it, saves it and then send back an ACK file, my client send back a confirmation that he received the ACK and finally my server close the connection and my client too.
I had an issue while attempting to receive the ACK from my client, but now it's resolved.
BUT, now that I shutdown my socket connection with SHUT_WR (that's telling that we will stop sending after this, if I'm right) I cannot resend my confirmation.
I can't figure out how that's working and how can I:
From my client
Send a file to my server
Receive the ACK from my server
Resend a confirmation
From my server
Receive a file
Send an ACK
Receive the confirmation
I'm stuck. My server is working unless I try to receive or send something.
The following code blocks are my actual files
client.py
import socket
import os
import random
from socket import SHUT_WR
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 8192
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
filesize = os.path.getsize(fileName)
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
print("[*] Reading")
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
print("[+] File reading is done.")
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR)
print("[*] Waiting for an acknowledgment")
data = socket.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
print(data.decode())
socket.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
socket.close()
server.py
import itertools
import socket
import signal
import ntpath
from pathlib import Path
from consts import *
from helper import *
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {0:'MSH',2:'', 3:'', 4:'', 5:'', 6:'', 7:'', 8:'', 9:'', 10:'', 11:'',12:'', 13:'', 14:'', 15:'NE', 16:'AL', 17:'', 18:'', 19:'', 20:'', 21:''}
MSA = {0:'MSA', 1:'AA', 2:''}
ACK = {'MSH':{}, 'MSA':{}}
def hl7_ack_generator():
"""Generate the Acknowledgement file and sends it to the client
Returns:
String: Returns the Acknowledgment filename
"""
content = ""
# Switch sender-receiver
MSH[3], MSH[5], MSH[4], MSH[6] = MSH[5], MSH[3], MSH[6], MSH[4]
# Set the message type.
# If possible get the message trigger to return the same trigger
try:
MSH[9] = f"ACK^{check_msh_9_trigger_event(MSH[9].decode().split('^'))}^ACK"
except:
MSH[9] = "ACK"
# Set MSH values
for param in MSH:
ACK['MSH'][param] = MSH.get(param)
# Set MSA values
for param in MSA:
ACK['MSA'][param] = MSA.get(param)
# Create the ACK message
# Handle integers & bytes in each fields
# Get MSH values
for i in range(0,21) :
if i != 1:
if ACK['MSH'][i]:
# Generate a message id based on recieved message timestamp and message id
# not exceeding 20 chars
if i == 10:
ACK['MSH'][10] = ACK['MSH'][7].decode() + ACK['MSH'][10].decode()
if len(ACK['MSH'][10]) > 20:
ACK['MSH'][10] = ACK['MSH'][10][:20]
content += ACK['MSH'][10]
else:
try:
content += ACK['MSH'][i].decode()
except:
if not ACK['MSH'][i] == None:
content += ACK['MSH'][i]
content += '|'
content += "\r"
# Get MSA values
for i in range(0,3):
try:
content += ACK['MSA'][i].decode()
except:
if not ACK['MSA'][i] == None:
content += ACK['MSA'][i]
content += "|"
# Create the ACK filename
filename = ACK['MSH'][10] + "_ACK.HL7"
# create the ACK file and write its content
with open(Path(SERVER_ACK_FILES_FOLDER + filename), "w") as f:
f.write(content)
f.close()
return filename
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
# Handle ctrl+c
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
# get the file size
fileSize = bytes(itertools.takewhile(lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
# remove absolute path if there is
filename = ntpath.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(Path(SERVER_FILES_FOLDER + filename), "wb") as f:
print("[+] File received")
while True:
# write to the file the bytes we just received
f.write(bytes_read)
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j+=1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
if not bytes_read:
# file transmitting is done
print("[+] File transfert is done")
break
with open(Path(SERVER_ACK_FILES_FOLDER + ackFilename), "rb") as f:
while True:
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
print("[+] Acknoledgment Sent")
break
client_socket.sendall(bytes_read)
confirmation = client_socket.recv(BUFFER_SIZE)
print(confirmation)
print("[+] Confirmation received")
print("[*] Closing conneciton")
client_socket.close()
socket.close
testX.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
Thanks for the attention!
I changed my server into this:
final_size = b''
while True:
# write to the file the bytes we just received
f.write(bytes_read)
final_size += bytes_read
if len(final_size) >= fileSize:
# file transmitting is done
print("[+] File transfert is done")
break
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j += 1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
And my client to this:
print(f"[+] File {fileName} sent")
print("[*] Waiting for an acknowledgment")
data = sock.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
sock.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
sock.close()
So now I have a server and client script. I'm trying to upload a file from the client to the server. However, the data from the file in the client will be cut out by the HEADER size. How do I send multiple packets under the same send command to the server?
server.py:
import socket
import threading
HEADER=2048
PORT=5050
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
server=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn,addr):
print(f'[NEW CONNECTION] {addr} connected.')
connected=True
while connected:
data=conn.recv(HEADER).decode(FORMAT)
if data==DISCONNECT_MESSAGE:
connected=False
else:
data=data.split(SEPARATOR)
file=open(data[0],'w')
file.write(data[1])
print('file received')
conn.send('file received'.encode(FORMAT))
conn.close()
print(f'[DISCONNECT] {addr} disconnected')
def start():
server.listen()
print(f'[LISTENING] Server is listening on {SERVER}')
while True:
conn,addr=server.accept()
thread=threading.Thread(target=handle_client,args=(conn,addr))
thread.start()
print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')
print("[STARTING] server is starting...")
start()
client.py:
import socket
HEADER=2048
PORT=5050
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
client=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client.connect(ADDR)
def send(msg):
message=msg.encode(FORMAT)
client.send(message)
print(client.recv(HEADER).decode(FORMAT))
file=open('question_pool.csv','r')
data=file.read()
send(f'question_pool.csv{SEPARATOR}{data}')
file.close()
send(DISCONNECT_MESSAGE)
In short, you want to send multiple chunks of any file that is larger than your HEADER size. Split the file into chunks smaller than the HEADER size, and send each chunk individually. Then when all the chunks are set, send a message that says the whole file has been sent so that the server can save it.
Here is my code for the solution described above:
server.py:
import socket
import threading
HEADER = 2048
PORT = 5050
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'
server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn, addr):
print(f'[NEW CONNECTION] {addr} connected.')
connected = True
current_file = None
while connected:
data = conn.recv(HEADER).decode(FORMAT)
if data == DISCONNECT_MESSAGE:
connected = False
elif data == FILE_FINISHED_SENDING:
current_file.close()
current_file = None
conn.send(b'file received.')
else:
data = data.split(SEPARATOR)
if len(data) == 2 and data[1] == '':
# The name of the file was sent, more will follow.
current_file = open(data[0], 'w')
conn.send(b'filename recieved')
else:
# File data was send, so write it to the current file
current_file.write(data[1])
print('chunk of file recv''d')
conn.send(b'chunk received')
conn.close()
print(f'[DISCONNECT] {addr} disconnected')
def start():
server.listen()
print(f'[LISTENING] Server is listening on {SERVER}')
while True:
conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn,addr))
thread.start()
print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')
print("[STARTING] server is starting...")
start()
client.py:
import socket
from pathlib import Path
HEADER = 2048
PORT = 5050
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)
def chunker(string: str, size: int):
return (string[pos:pos + size] for pos in range(0, len(string), size))
def send(msg):
message = msg.encode(FORMAT)
client.send(message)
print(client.recv(HEADER).decode(FORMAT))
def send_file(filepath: str):
with open(filepath, 'r', encoding=FORMAT) as f:
data = f.read()
first_bits = f'{Path(filepath).name}{SEPARATOR}' # Easy way of getting just a file's name from its path
send(first_bits) # Send the filename to the server
for chunk in chunker(data, HEADER-48): # Leave a safe buffer
# Send each chunk of the file
send(f"{SEPARATOR}{chunk}")
# Tell the server that's all for this file.
# Now it can close the file object.
send(FILE_FINISHED_SENDING)
send_file("/path/to/file.html")
send(DISCONNECT_MESSAGE)
Tips:
make sure that your special messages like SEPARATOR, FILE_FINISHED_SENDING and DISCONNECT_MESSAGE are NOT going to appear in the files you are sending. Otherwise things might get wonky.
You may want to read files as raw bytes when you send them through the socket, instead of reading as strings, encoding, decoding, etc. This way you could send binary files such as .mp3, for example.
I'm new to python socket programming. I want to transfer 5 files (photos) from one host in mininet to another. The name of these files are numbered respectively (I mean 1.jpg, 2.jpg and ...). The problem is that when I run these codes, the first photo is transferred correctly but others become corrupted. What's the problem:
sender.py
import socket
import sys
buf = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('10.0.0.1',12345))
for i in range(1,6):
with open("%d.jpg" % (i),'rb') as f:
data = f.read(buf)
while 1:
if not data:
break
s.sendall(data)
data = f.read(buf)
s.close()
receiver.py:
import socket
import sys
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('10.0.0.1', 12345))
buf = 1024
s.listen(1)
conn , addr = s.accept()
for i in range(6,11):
with open("%d.jpg" % (i),'wb') as f:
while 1:
data = conn.recv(buf)
#print(data[:10])
#print "PACKAGE RECEIVED..."
f.write(data)
if not data: break
#conn.close()
#s.close()
The simple solution to your problem is to create a new connection for each file. The code below does that.
Sender
from __future__ import print_function
import socket
HOST = 'localhost'
PORT = 12345
BUFSIZE = 4096
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
while True:
data = f.read(BUFSIZE)
if not data:
break
while data:
sent = sock.send(data)
data = data[sent:]
sock.close()
fnames = [
'test0.jpg',
'test1.jpg',
'test2.jpg',
'test3.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver
from __future__ import print_function
import socket
HOST = 'localhost'
PORT = 12345
BUFSIZE = 4096
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
file_number = 0
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
fname = 'image%d.jpg' % file_number
with open(fname, 'wb') as f:
while True:
data = conn.recv(BUFSIZE)
if not data:
break
f.write(data)
conn.close()
print(fname, 'saved\n')
file_number += 1
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
You need to hit CtrlC or Break (depending on your OS) to exit the receiver.
But using those numeric file names at the receiver isn't very satisfactory , so I decided to make it a little more complicated. :) In the following version we send the file name before the file data. That's slightly tricky because the receiver needs to separate the file name from the actual file data. If each socket.recv call corresponded to a socket.send call that would be easy, but that's not guaranteed to happen: the received bytes may be split differently from how they were sent. The receiver needs to buffer the bytes so it can break them up correctly. See the Socket Programming HOWTO for details.
So that the receiver knows where the file name ends we first send a single byte that encodes the length of the file name. A byte can hold a number from 0 to 255, so this code can't handle file names longer than that. After the length byte, we send the file name itself, encoded using UTF-8. And then we send the actual file contents.
The receiver uses a class named Receiver to handle the buffering. This class has a .get method which we can use to get a specified number of bytes. We use that method to get the file name length and the file name. And then we use Receiver's .save method to save the received file contents to a new file.
This code is a little bit messy because it's designed to run on Python 2 and Python 3, in any combination. It'd be somewhat neater if it was for Python 3 only. I hard-coded 'localhost' as the host name, since I only have one computer, so I couldn't test it over a network, but I'm confident that it will work correctly on a network.
Here's the sender:
from __future__ import print_function
import socket
from struct import pack
HOST = 'localhost'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'test1.gif',
'test2.jpg',
'test3.png',
'test4.pdf',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
And here's the receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = 'localhost'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
You should use it like:
with open("%d.jpg" % (i),'wb') as f:
while 1:
data = conn.recv(buf)
#print(data[:10])
#print "PACKAGE RECEIVED..."
f.write(data)
if not data: break
this way f gets closed automatically when you leave the with block when you exit the while.
I guess you need to extend the protocol so that the receiver knows what it gets and that you can let it act on that. Something like a 'begin-of-transfer' ... sending the filename, sending the data, sending 'end-of-transfer' that makes the receiver close the file
Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?
I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()
A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git
The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
Here is my server.py:
import socket, atexit
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((socket.gethostname(), 8000))
server.listen(5)
(client,(ip,port))=server.accept()
command = raw_input('> ')
if command.rsplit(' ',1)[0] == 'write':
client.send(command.rsplit(' ',1)[2])
print 'Client # ', ip + ' '
data = client.recv(1024)
file = open(command.rsplit(' ',1)[1],'rb')
bytes = file.read(1024)
while(bytes):
client.send(bytes)
bytes = file.read(1024)
file.close()
client.close()
The client.py:
import socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('MY IP', 8000))
client.send("!")
name = client.recv(1024)
with open(name, 'wb') as file:
while True:
data = client.recv(1024)
if not data:
break
file.write(data)
file.close()
client.close()
The first data transmission in server.py is supposed to send the name of the file I want to the client.py. Where it says:
name = client.recv(1024)
in client.py, it is supposed to receive and make a file using that name. However, the server.py closes, causing the client.py to crash and not give output (host closed). If I open in IDLE to see the output, it doesn't work but nothing shows.
Your server.py needed tweaked;
import socket, atexit
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('', 8000))
server.listen(5)
(client,(ip,port))=server.accept()
command = raw_input('> ')
if command.split(' ')[0] == 'write':
client.send(command.split(' ')[2])
print 'Client # '+str(ip)+':'
data = client.recv(1024)
file = open(command.split(' ')[1],'rb')
bytes = file.read(1024)
while(bytes):
client.send(bytes)
bytes = file.read(1024)
file.close()
client.close()
The rsplit and trailing ,1's were causing the breaks.
Using the input write /Users/Namelessthehckr/Downloads/ucsgflmza.cs /Users/Namelessthehckr/Desktop/Test.txt, the file was successfully CP'd without error.