Develop Reference

I cannot send an ack confirmation from my client after receiving an ACK through socket

I have a server that's always listening and waiting for a connection. When a client connect then sends a file to it, it receives it, saves it and then send back an ACK file, my client send back a confirmation that he received the ACK and finally my server close the connection and my client too.
I had an issue while attempting to receive the ACK from my client, but now it's resolved.
BUT, now that I shutdown my socket connection with SHUT_WR (that's telling that we will stop sending after this, if I'm right) I cannot resend my confirmation.
I can't figure out how that's working and how can I:
From my client
Send a file to my server
Receive the ACK from my server
Resend a confirmation
From my server
Receive a file
Send an ACK
Receive the confirmation
I'm stuck. My server is working unless I try to receive or send something.
The following code blocks are my actual files
client.py
import socket
import os
import random
from socket import SHUT_WR
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 8192
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
filesize = os.path.getsize(fileName)
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
print("[*] Reading")
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
print("[+] File reading is done.")
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR)
print("[*] Waiting for an acknowledgment")
data = socket.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
print(data.decode())
socket.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
socket.close()
server.py
import itertools
import socket
import signal
import ntpath
from pathlib import Path
from consts import *
from helper import *
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {0:'MSH',2:'', 3:'', 4:'', 5:'', 6:'', 7:'', 8:'', 9:'', 10:'', 11:'',12:'', 13:'', 14:'', 15:'NE', 16:'AL', 17:'', 18:'', 19:'', 20:'', 21:''}
MSA = {0:'MSA', 1:'AA', 2:''}
ACK = {'MSH':{}, 'MSA':{}}
def hl7_ack_generator():
"""Generate the Acknowledgement file and sends it to the client
Returns:
String: Returns the Acknowledgment filename
"""
content = ""
# Switch sender-receiver
MSH[3], MSH[5], MSH[4], MSH[6] = MSH[5], MSH[3], MSH[6], MSH[4]
# Set the message type.
# If possible get the message trigger to return the same trigger
try:
MSH[9] = f"ACK^{check_msh_9_trigger_event(MSH[9].decode().split('^'))}^ACK"
except:
MSH[9] = "ACK"
# Set MSH values
for param in MSH:
ACK['MSH'][param] = MSH.get(param)
# Set MSA values
for param in MSA:
ACK['MSA'][param] = MSA.get(param)
# Create the ACK message
# Handle integers & bytes in each fields
# Get MSH values
for i in range(0,21) :
if i != 1:
if ACK['MSH'][i]:
# Generate a message id based on recieved message timestamp and message id
# not exceeding 20 chars
if i == 10:
ACK['MSH'][10] = ACK['MSH'][7].decode() + ACK['MSH'][10].decode()
if len(ACK['MSH'][10]) > 20:
ACK['MSH'][10] = ACK['MSH'][10][:20]
content += ACK['MSH'][10]
else:
try:
content += ACK['MSH'][i].decode()
except:
if not ACK['MSH'][i] == None:
content += ACK['MSH'][i]
content += '|'
content += "\r"
# Get MSA values
for i in range(0,3):
try:
content += ACK['MSA'][i].decode()
except:
if not ACK['MSA'][i] == None:
content += ACK['MSA'][i]
content += "|"
# Create the ACK filename
filename = ACK['MSH'][10] + "_ACK.HL7"
# create the ACK file and write its content
with open(Path(SERVER_ACK_FILES_FOLDER + filename), "w") as f:
f.write(content)
f.close()
return filename
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
# Handle ctrl+c
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
# get the file size
fileSize = bytes(itertools.takewhile(lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
# remove absolute path if there is
filename = ntpath.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(Path(SERVER_FILES_FOLDER + filename), "wb") as f:
print("[+] File received")
while True:
# write to the file the bytes we just received
f.write(bytes_read)
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j+=1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
if not bytes_read:
# file transmitting is done
print("[+] File transfert is done")
break
with open(Path(SERVER_ACK_FILES_FOLDER + ackFilename), "rb") as f:
while True:
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
print("[+] Acknoledgment Sent")
break
client_socket.sendall(bytes_read)
confirmation = client_socket.recv(BUFFER_SIZE)
print(confirmation)
print("[+] Confirmation received")
print("[*] Closing conneciton")
client_socket.close()
socket.close
testX.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
Thanks for the attention!

I changed my server into this:
final_size = b''
while True:
# write to the file the bytes we just received
f.write(bytes_read)
final_size += bytes_read
if len(final_size) >= fileSize:
# file transmitting is done
print("[+] File transfert is done")
break
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j += 1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
And my client to this:
print(f"[+] File {fileName} sent")
print("[*] Waiting for an acknowledgment")
data = sock.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
sock.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
sock.close()

How to make a socket server run forever without closed itself (Python)

Whenever the client disconnect, the server will close itself. How can i make the server to run forever ?
What i'm doing
The server let one client to retrieve files with no issues. But the problem is when the client close the program, the server will also closed itself and wouldn't let another client to establish the connection . I had read a few articles about using while loops to make the session alive. Does anyone know how can I do that ?
Server.py
import socket, os, subprocess, shutil, pickle, struct
# Create a Socket ( connect two computers)
def create_socket():
try:
global host
global port
global s
host = ""
port = 9999
s = socket.socket()
except socket.error as msg:
create_socket()
# Binding the socket and listening for connections
def bind_socket():
try:
global host
global port
global s
s.bind((host, port))
s.listen(5)
except socket.error as msg:
bind_socket()
# send file list
def flist(conn):
try:
arr = pickle.dumps(os.listdir())
conn.send(arr)
except:
conn.send(('Error').encode("utf-8"))
# accept file from server
def fdown(filename, conn):
try:
data = conn.recv(1024).decode("utf-8")
if data[:6] == 'EXISTS':
filesize = data[6:]
conn.send("OK".encode("utf-8"))
f = open(filename, 'wb')
data = (conn.recv(1024))
totalRecv = len(data)
f.write(data)
while int(totalRecv) < int(filesize):
data = conn.recv(1024)
totalRecv += len(data)
f.write(data)
f.close()
except:
conn.send(('Error').encode("utf-8"))
# send file
def fup(filename, conn):
if os.path.isfile(filename):
conn.send(str.encode("EXISTS " + str(os.path.getsize(filename))))
filesize = int(os.path.getsize(filename))
userResponse = conn.recv(1024).decode("utf-8")
if userResponse[:2] == 'OK':
with open(filename, 'rb') as f:
bytesToSend = f.read(1024)
conn.send(bytesToSend)
totalSend = len(bytesToSend)
while int(totalSend) < int(filesize):
bytesToSend = f.read(1024)
totalSend += len(bytesToSend)
conn.send(bytesToSend)
else:
conn.send("ERROR".encode("utf-8"))
# main
def main(s):
while True:
data = (s.recv(1024)).decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
# Establish connection with a client (socket must be listening)
def socket_accept():
conn, address = s.accept()
main(conn)
conn.close()
create_socket()
bind_socket()
socket_accept()

You should put accept in the loop, and you may need use a thread to handle read
sample code:
def handle_read(s):
while True:
data = s.recv(1024)
if not data:
#When the client closed, recv will return an empty data
s.close()
break
data = data.decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
def socket_accept():
while True:
conn, address = s.accept()
t = threading.Thread(target = handle_read, args=(conn, ))
t.start()

Receive multiple files simultaneously

I wrote a script in Python 3.8.6 to send and receive files between different devices. I am sending and receiving files normally, the problem happens when I receive multiple simultaneous connections from different devices.
I configured the script to save the files in folders named with the address of the device that sends the file and what is happening is that when several devices send files at the same time, files from one device go to another folder, randomly. The files are mixed.
I have an idea of why it is happening but not how to solve it.
I thought as an alternative to receive only one connection at a time.
But I don't know how to do this.
sendfile.py
def conectividade(host="192.168.0.13", porta=1218, timeout=5):
while True:
try:
socket.setdefaulttimeout(timeout)
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, porta))
return True
except Exception:
return False
# Function that sends the collected data.
def coleta():
total = []
conectividade()
conect = conectividade()
if conect == False:
print("\nConnectivity is",conect)
pass
else:
data_hoje = datetime.now().strftime('%Y-%b-%d')
direct = f".\\{data_hoje}\\"
arquivos = os.listdir(direct)
for arquivo in arquivos:
try:
sock = socket.socket()
host = "192.168.0.13"
porta = 1218
print("Connectivity is", conect)
sock.connect((host, porta))
total += arquivo.split(',')
sock.send(str.encode(arquivo))
arquivo = str.encode(direct + arquivo)
with open(arquivo, "rb") as file:
data = file.read(1024)
resposta = sock.recv(1024)
if resposta == b'Waiting file':
file_size = os.path.getsize(arquivo)
print(b"sending " + arquivo)
print(file_size,"kb")
sock.send(bytes(str(file_size),'utf-8'))
resposta = sock.recv(1024)
if resposta == b'Waiting...':
while data:
sock.send(data)
data = file.read(1024)
if not data:
print(f"**Sending Complete**\n")
sock.close()
# Delete files after uploads
if total == arquivos:
print(f"Uploaded files:\n",total)
except socket.error:
print("\nSocket Error\n")
receive.py
data_hoje = datetime.now().strftime('%Y-%b-%d')
sock = socket.socket()
host ='192.168.0.13'
porta = 1218
sock.bind((host, porta))
sock.listen(10)
total = []
while True:
conn, addr = sock.accept()
pasta = ".\\" + addr[0].replace('.','_')
subpasta = pasta + "\\" + data_hoje
if os.path.exists(pasta) == False:
os.makedirs(f'{pasta}')
if os.path.exists(subpasta) == False:
os.makedirs(f'{subpasta}')
data = conn.recv(1024)
while data == b'':
conn.close()
conn, addr = sock.accept()
data = conn.recv(1024)
if data == b'Finalizando socket':
print("Finalizando conexão com " + addr[0])
conn.close()
sock.close()
total += data.decode().split(',')
name_file = total[-1]
with open(f"{subpasta}\\{name_file}", "wb") as file:
conn.send(b"Waiting file")
data = conn.recv(1024)
tamanho = data.decode('utf-8')
file_size = int(tamanho)
if file_size != 0:
conn.send(b"Waiting...")
print("HOST <--> " + addr[0] + f" Download de {name_file} - {tamanho} kb")
while data:
data = conn.recv(1024)
file.write(data)
conn.close()

In receive.py you have:
conn, addr = sock.accept()
# [...]
while data == b'':
conn.close()
conn, addr = sock.accept()
data = conn.recv(1024)
The client that connects on the first accept() may be different from the client that connects on the accept() inside the cycle, which could lead to the files mixing up as you described. You should re-write it with a single accept(), to guarantee the whole processing is done for the same client. This approach is described in TCP/IP Client and Server - PyMOTW-3.
Modified receive.py:
data_hoje = datetime.now().strftime("%Y-%b-%d")
sock = socket.socket()
host = "192.168.0.13"
porta = 1218
sock.bind((host, porta))
sock.listen(10)
total = []
try:
while True:
conn, addr = sock.accept()
try:
pasta = ".\\" + addr[0].replace(".", "_")
subpasta = pasta + "\\" + data_hoje
if os.path.exists(pasta) is False:
os.makedirs(f"{pasta}")
if os.path.exists(subpasta) is False:
os.makedirs(f"{subpasta}")
data = b""
while data == b'':
data = conn.recv(1024)
if data == b"Finalizando socket":
print("Finalizando conexão com " + addr[0])
continue
total += data.decode().split(",")
name_file = total[-1]
with open(f"{subpasta}\\{name_file}", "wb") as file:
conn.send(b"Waiting file")
data = conn.recv(1024)
tamanho = data.decode("utf-8")
file_size = int(tamanho)
if file_size != 0:
conn.send(b"Waiting...")
print(
"HOST <--> " + addr[0] + f" Download de {name_file} - {tamanho} kb"
)
while data:
data = conn.recv(1024)
file.write(data)
finally:
conn.close()
finally:
sock.close()

Basically, what was causing the error was the conectividade() function recursively in the original script.
so I needed...
while True:
conn.close()
conn, addr = socket.accept()
data = conn.recv
What caused the files to mix, as said by fzdb ...
How was the new script ...
sendfile.py
import socket
import os
from datetime import datetime
def coleta():
total = []
data_hoje = datetime.now().strftime('%Y-%b-%d')
direct = f".\\{data_hoje}\\"
arquivos = os.listdir(direct)
for arquivo in arquivos:
try:
sock = socket.socket()
host = "192.168.0.13"
porta = 1218
print("Connectivity is",)# conect)
sock.connect((host, porta))
total += arquivo.split(',')
sock.send(str.encode(arquivo))
arquivo = str.encode(direct + arquivo)
with open(arquivo, "rb") as file:
data = file.read(1024)
resposta = sock.recv(1024)
if resposta == b'Waiting file':
file_size = os.path.getsize(arquivo)
print(b"sending " + arquivo)
print(file_size,"kb")
sock.send(bytes(str(file_size),'utf-8'))
resposta = sock.recv(1024)
if resposta == b'Waiting...':
while data:
sock.send(data)
data = file.read(1024)
if not data:
sock.close()
print(f"**Sending Complete**\n")
if total == arquivos:
print(f"Uploaded files:\n",total)
except socket.error:
print("\nSocket Error\n")
receive.py
import socket
import os
from datetime import datetime
data_hoje = datetime.now().strftime('%Y-%b-%d')
host ='192.168.0.13'
porta = 1218
sock.bind((host, porta))
sock.listen(10)
total = []
while True:
conn, addr = sock.accept()
pasta = ".\\" + addr[0].replace('.','_')
subpasta = pasta + "\\" + data_hoje
if os.path.exists(pasta) == False:
os.makedirs(f'{pasta}')
if os.path.exists(subpasta) == False:
os.makedirs(f'{subpasta}')
data = conn.recv(1024)
if data == b'':
conn.close()
total += data.decode().split(',')
name_file = total[-1]
try:
with open(f"{subpasta}\\{name_file}", "wb") as file:
conn.send(b"Waiting file")
data = conn.recv(1024)
tamanho = data.decode('utf-8')
file_size = int(tamanho)
if file_size != 0:
conn.send(b"Waiting...")
print("HOST <--> " + addr[0] + f" Download de {name_file} - {tamanho} bytes")
while data:
data = conn.recv(1024)
file.write(data)
except:
pass
Thank you!
For those who want to help with the original code ...
I have problems with NumPad on pynput
https://github.com/gconelhero/Keylogger

Type Error using Socket communication even while decoding to byte string

Hello I got a problem with a type error I cant get rid off.
I have build a server named processA.py and a client processB.py
I start the server calling processA and it starts up
I start the client calling processB and it starts up
Now I input the name of the file I want to send over a socket.
The image must be in the same folder as the programms.
You can use the sample code from below for reproducing the error.
I choose: lena.jpg
I get an error in line 15, in of my processA.py:
sock.send("EXISTS" + str(os.path.getsize(filenameByte))) TypeError: a
bytes-like object is required, not 'str'
processA.py
'''
Created on 29 Nov 2017
#author: Poor Student
'''
import numpy
import socket
import threading
import os
def RtrFile(name, sock):
filenameByte = sock.recv(1024)
filenameStr = filenameByte.decode('ascii')
if os.path.isfile(filenameStr):
sock.send("EXISTS" + str(os.path.getsize(filenameByte)))
userResponse = sock.recv(1024)
if userResponse[:2] == 'OK':
with open(filenameByte, 'rb') as f:
bytesToSend = f.read(1024)
sock.send(bytesToSend)
while bytesToSend != "":
bytesToSend = f.read(1024)
sock.send(bytesToSend)
else:
sock.send("ERR")
sock.close()
def Main():
host = "127.0.0.1"
port = 5000
s = socket.socket()
s.bind((host,port))
s.listen(5)
print("server started.")
while True:
c, addr =s.accept()
print("client connected ip:< " + str(addr) +">")
t = threading.Thread(target = RtrFile, args=("rtrThread",c))
t.start()
s.close()
if __name__ == "__main__":
Main()
processB.py
'''
Created on 29 Nov 2017
#author: Poor Student
'''
import socket
def Main():
host = "127.0.0.1"
port = 5000
s = socket.socket()
s.connect((host,port))
filename = input("Filename? -> ")
if filename != "q":
s.send(filename.encode())
data = s.recv(1024)
if data[:6] == "EXISTS":
filesize = int(data[6:])
message = input("File Exists, " +str(filesize) + "Bytes, download? (Y/N)? ->")
if message == "Y" or message =="y":
s.send("OK")
f = open("new_" + filename, "wb")
data = s.recv(1024)
totalRecv = len(data)
f.write(data)
while totalRecv < filesize:
data = s.recv(1024)
totalRecv += len(data)
f.write(data)
print("(0:.f)".format((totalRecv)/float(filesize))*100 +"done")
print("Download Complete")
else:
print("File does not exists")#
s.close()
if __name__ == "__main__":
Main()

I found an answer. In python 3 you have allways convert your Strings to byte strings when sending them. After sending them to the receiver you have to decode them again and this must be done for the hole communication which is why the error allways shows up in the same module.
Process A:
'''
Created on 29 Nov 2017
#author: Happy student
'''
import numpy
import socket
import threading
import os
from idlelib.IOBinding import encoding
def RtrFile(name, sock):
filenameByte = sock.recv(1024)
filenameStr = filenameByte.decode('ascii')
print("FilenameStr",filenameStr)
if os.path.isfile(filenameStr):
print("Type:",type(filenameByte))
print(os.path.getsize(filenameByte))#
sendStr = "EXISTS" + str(os.path.getsize(filenameByte))
#Convert the string to byte because otherwise it will not be send
sock.send((sendStr.encode(encoding)))
userResponse = sock.recv(1024)
#the Responce will be received in byte and will be converted to a string to make it checkable
userResponceStr = userResponse.decode('ascii')
if userResponceStr[:2] == 'OK':
with open(filenameByte, 'rb') as f:
bytesToSend = f.read(1024)
sock.send(bytesToSend)
while bytesToSend != "":
bytesToSend = f.read(1024)
sock.send(bytesToSend)
else:
print("User response not known")
else:
sendStr = "ERR"
sock.send(sendStr.encode(encoding))
sock.close()
def Main():
host = "127.0.0.1"
port = 5000
s = socket.socket()
s.bind((host,port))
s.listen(5)
print("server started.")
while True:
c, addr =s.accept()
print("client connected ip:< " + str(addr) +">")
t = threading.Thread(target = RtrFile, args=("rtrThread",c))
t.start()
s.close()
if __name__ == "__main__":
Main()
processB.py
'''
Created on 29 Nov 2017
#author: Happy student
'''
import socket
from idlelib.IOBinding import encoding
def Main():
host = "127.0.0.1"
port = 5000
s = socket.socket()
s.connect((host,port))
filename = input("Filename? -> ")
if filename != "q":
s.send(filename.encode())
data = s.recv(1024)
dataStr = data.decode('ascii')
if dataStr[:6] == "EXISTS":
filesize = int(dataStr[6:])
message = input("File Exists, " +str(filesize) + "Bytes, download? (Y/N)? ->")
if message == "Y" or message =="y":
sendStr = "OK"
s.send(sendStr.encode(encoding))
#create new file new_filename and
f = open("new_" + filename, "wb")
data = s.recv(1024)
totalRecv = len(data)
f.write(data)
while totalRecv < filesize:
data = s.recv(1024)
totalRecv += len(data)
f.write(data)
print("Download Complete")
else:
print(message +"was not noticed")
exit()
else:
print("File does not exists")#
s.close()
if __name__ == "__main__":
Main()

What is the proper way of sending a large amount of data over sockets in Python?

Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?

I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()

A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git

The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()