Develop Reference

why does my python socket just stop working after it is done chunking and receiving data

basically this code is a server for a project that I am doing the client sends over 600 photos which then I want to be stored in a folder called incoming. first error was me running into my pickle data being Truncated while being sent to my server to try rectify this. I added a chunking system where it splits the data up into smaller chunks to make sure there is a smaller change of data being lost. doing this in theory should work. but when it has finished chunking all my data the server just quits out of the 'handle_client' function but gives me no errors and in turn leaves my client hanging and unresponsive until I shut the server down
import pickle
import socket
import os
import threading
HEADER = 64
PORT = 5050
SERVER = 'Left this out for obvious reasons'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "..DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn, addr):
print(f"[NEW CONNECTION] {addr} connected.")
connected = True
while connected:
data_length = conn.recv(HEADER).decode(FORMAT)
if data_length:
data_length = int(data_length)
recv_data = conn.recv(data_length)
bytes_received = 0
while bytes_received < data_length:
chunk = conn.recv(4096)
recv_data += chunk
bytes_received += len(chunk)
data = pickle.loads(recv_data)
if data == DISCONNECT_MESSAGE:
connected = False
try:
for files in os.listdir('incoming'):
file_path = os.path.join('incoming', files)
os.remove(file_path)
except:
print('[SERVER_ERROR] No files inside ./Incoming/ folder')
for file in data:
with open(os.path.join('incoming', file['name']), "wb") as f:
f.write(file['data'])
print('[SAVING] Server has saved file')
with open('siamesemodel.h5','rb') as f:
data = f.read()
pickledData = pickle.dumps(data)
data_length = len(pickledData)
send_length = str(data_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
conn.send(send_length)
conn.send(pickledData)
def start():
server.listen()
print(f"[LISTENING] Server is listening on {SERVER}")
while True:
conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn, addr))
thread.start()
print(f"[ACTIVE CONNECTIONS] {threading.active_count() - 1}")
print("[STARTING] server is starting...")
start()
tried debugging to no avail

I cannot send an ack confirmation from my client after receiving an ACK through socket

I have a server that's always listening and waiting for a connection. When a client connect then sends a file to it, it receives it, saves it and then send back an ACK file, my client send back a confirmation that he received the ACK and finally my server close the connection and my client too.
I had an issue while attempting to receive the ACK from my client, but now it's resolved.
BUT, now that I shutdown my socket connection with SHUT_WR (that's telling that we will stop sending after this, if I'm right) I cannot resend my confirmation.
I can't figure out how that's working and how can I:
From my client
Send a file to my server
Receive the ACK from my server
Resend a confirmation
From my server
Receive a file
Send an ACK
Receive the confirmation
I'm stuck. My server is working unless I try to receive or send something.
The following code blocks are my actual files
client.py
import socket
import os
import random
from socket import SHUT_WR
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 8192
HOST = "127.0.0.1"
PORT = 8000
files = ["test1.HL7","test2.HL7","test3.HL7","test4.HL7","test5.HL7","test6.HL7","test7.HL7","test8.HL7"]
fileName = f".\\ClientFiles\\{files[random.randrange(1,8)]}"
filesize = os.path.getsize(fileName)
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(f"[+] Connecting to {HOST}:{PORT}")
socket.connect((HOST, PORT))
print("[+] Connected.")
socket.send(f"{fileName}{SEPARATOR}{filesize}".encode())
# opening file
with open(fileName, "rb") as f:
print("[*] Reading")
while True:
# reading bytes
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
# Transmitting is done
print("[+] File reading is done.")
break
# send all the buffer
socket.sendall(bytes_read)
print(f"[+] File {fileName} sent")
socket.shutdown(SHUT_WR)
print("[*] Waiting for an acknowledgment")
data = socket.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
print(data.decode())
socket.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
socket.close()
server.py
import itertools
import socket
import signal
import ntpath
from pathlib import Path
from consts import *
from helper import *
# Setting Dicionnaries for HL7 ACK Requirements
MSH = {0:'MSH',2:'', 3:'', 4:'', 5:'', 6:'', 7:'', 8:'', 9:'', 10:'', 11:'',12:'', 13:'', 14:'', 15:'NE', 16:'AL', 17:'', 18:'', 19:'', 20:'', 21:''}
MSA = {0:'MSA', 1:'AA', 2:''}
ACK = {'MSH':{}, 'MSA':{}}
def hl7_ack_generator():
"""Generate the Acknowledgement file and sends it to the client
Returns:
String: Returns the Acknowledgment filename
"""
content = ""
# Switch sender-receiver
MSH[3], MSH[5], MSH[4], MSH[6] = MSH[5], MSH[3], MSH[6], MSH[4]
# Set the message type.
# If possible get the message trigger to return the same trigger
try:
MSH[9] = f"ACK^{check_msh_9_trigger_event(MSH[9].decode().split('^'))}^ACK"
except:
MSH[9] = "ACK"
# Set MSH values
for param in MSH:
ACK['MSH'][param] = MSH.get(param)
# Set MSA values
for param in MSA:
ACK['MSA'][param] = MSA.get(param)
# Create the ACK message
# Handle integers & bytes in each fields
# Get MSH values
for i in range(0,21) :
if i != 1:
if ACK['MSH'][i]:
# Generate a message id based on recieved message timestamp and message id
# not exceeding 20 chars
if i == 10:
ACK['MSH'][10] = ACK['MSH'][7].decode() + ACK['MSH'][10].decode()
if len(ACK['MSH'][10]) > 20:
ACK['MSH'][10] = ACK['MSH'][10][:20]
content += ACK['MSH'][10]
else:
try:
content += ACK['MSH'][i].decode()
except:
if not ACK['MSH'][i] == None:
content += ACK['MSH'][i]
content += '|'
content += "\r"
# Get MSA values
for i in range(0,3):
try:
content += ACK['MSA'][i].decode()
except:
if not ACK['MSA'][i] == None:
content += ACK['MSA'][i]
content += "|"
# Create the ACK filename
filename = ACK['MSH'][10] + "_ACK.HL7"
# create the ACK file and write its content
with open(Path(SERVER_ACK_FILES_FOLDER + filename), "w") as f:
f.write(content)
f.close()
return filename
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# test the binding
try:
socket.bind((SERVER_HOST, SERVER_PORT))
except socket.error as error:
print('Bind failed. Error Code : '
+ str(error[0]) + ' Message '
+ error[1])
exit()
# Handle ctrl+c
def signal_handler(sign, frame):
print('[*] Shutting down')
exit(0)
while signal.signal(signal.SIGINT, signal_handler):
# connection limit(5 connection try then deny)
socket.listen(5)
print(f"[*] Listening as {SERVER_HOST}:{SERVER_PORT}")
# accept the connection if there is any
client_socket, address = socket.accept()
# Below code is executed if sender is connected
print(f"[+] {address} is connected.")
# get what the client is sending
received = client_socket.recv(BUFFER_SIZE)
filename, bytes_read = received.split(SEPARATOR.encode())
# get the file size
fileSize = bytes(itertools.takewhile(lambda i: bytes((i,)).isdigit(), bytes_read))
bytes_read = bytes_read[len(fileSize):]
#convert to integer
fileSize = int(fileSize)
filename = filename.decode()
# remove absolute path if there is
filename = ntpath.basename(filename)
# start receiving the file from the socket and writing to the file stream
with open(Path(SERVER_FILES_FOLDER + filename), "wb") as f:
print("[+] File received")
while True:
# write to the file the bytes we just received
f.write(bytes_read)
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j+=1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
if not bytes_read:
# file transmitting is done
print("[+] File transfert is done")
break
with open(Path(SERVER_ACK_FILES_FOLDER + ackFilename), "rb") as f:
while True:
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
print("[+] Acknoledgment Sent")
break
client_socket.sendall(bytes_read)
confirmation = client_socket.recv(BUFFER_SIZE)
print(confirmation)
print("[+] Confirmation received")
print("[*] Closing conneciton")
client_socket.close()
socket.close
testX.hl7
MSH|^~\&|ADT1|MCM|LABADT|MCM|198808181126|SECURITY|ADT^A04|MSG00001|P|2.4
EVN|A01-|198808181123
PID|||PATID1234^5^M11||JONES^WILLIAM^A^III||19610615|M-||2106-3|1200 N ELM STREET^^GREENSBORO^NC^27401-1020|GL|(919)379-1212|(919)271-3434~(919)277-3114||S||PATID12345001^2^M10|123456789|9-87654^NC
NK1|1|JONES^BARBARA^K|SPO|||||20011105
NK1|1|JONES^MICHAEL^A|FTH
PV1|1|I|2000^2012^01||||004777^LEBAUER^SIDNEY^J.|||SUR||-||1|A0-
AL1|1||^PENICILLIN||PRODUCES HIVES~RASH
AL1|2||^CAT DANDER
DG1|001|I9|1550|MAL NEO LIVER, PRIMARY|19880501103005|F||
PR1|2234|M11|111^CODE151|COMMON PROCEDURES|198809081123
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^SMITH^ELLEN|199505011201
GT1|1122|1519|BILL^GATES^A
IN1|001|A357|1234|BCMD|||||132987
IN2|ID1551001|SSN12345678
ROL|45^RECORDER^ROLE MASTER LIST|AD|CP|KATE^ELLEN|199505011201
Thanks for the attention!

I changed my server into this:
final_size = b''
while True:
# write to the file the bytes we just received
f.write(bytes_read)
final_size += bytes_read
if len(final_size) >= fileSize:
# file transmitting is done
print("[+] File transfert is done")
break
# read 1024 bytes from the socket (receive)
bytes_read = client_socket.recv(BUFFER_SIZE)
if bytes_read.startswith(b'MSH'):
messageHeader = bytes_read.partition(b'\r')[0].split(b'|')
j = 0
i = 2
for i in range(2,17):
j += 1
# Exclude MSH fields(constant and/or unwanted)
if i not in (15,16):
MSH[i]= messageHeader[j]
#Get message ID
if i == 10:
MSA[2] = messageHeader[j]
ackFilename = hl7_ack_generator()
And my client to this:
print(f"[+] File {fileName} sent")
print("[*] Waiting for an acknowledgment")
data = sock.recv(BUFFER_SIZE)
print("[+] Acknoledgment recieved")
sock.sendall(data.decode().split('|')[9].encode())
print("[+] Acknowledgment confirmation sent")
print("[*] Closing")
sock.close()

send multiple packets of data using python socket programming

So now I have a server and client script. I'm trying to upload a file from the client to the server. However, the data from the file in the client will be cut out by the HEADER size. How do I send multiple packets under the same send command to the server?
server.py:
import socket
import threading
HEADER=2048
PORT=5050
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
server=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn,addr):
print(f'[NEW CONNECTION] {addr} connected.')
connected=True
while connected:
data=conn.recv(HEADER).decode(FORMAT)
if data==DISCONNECT_MESSAGE:
connected=False
else:
data=data.split(SEPARATOR)
file=open(data[0],'w')
file.write(data[1])
print('file received')
conn.send('file received'.encode(FORMAT))
conn.close()
print(f'[DISCONNECT] {addr} disconnected')
def start():
server.listen()
print(f'[LISTENING] Server is listening on {SERVER}')
while True:
conn,addr=server.accept()
thread=threading.Thread(target=handle_client,args=(conn,addr))
thread.start()
print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')
print("[STARTING] server is starting...")
start()
client.py:
import socket
HEADER=2048
PORT=5050
FORMAT='utf-8'
DISCONNECT_MESSAGE='!DISCONNECT'
SEPARATOR='<SEPERATE>'
SERVER=socket.gethostbyname(socket.gethostname())
ADDR=(SERVER,PORT)
client=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client.connect(ADDR)
def send(msg):
message=msg.encode(FORMAT)
client.send(message)
print(client.recv(HEADER).decode(FORMAT))
file=open('question_pool.csv','r')
data=file.read()
send(f'question_pool.csv{SEPARATOR}{data}')
file.close()
send(DISCONNECT_MESSAGE)

In short, you want to send multiple chunks of any file that is larger than your HEADER size. Split the file into chunks smaller than the HEADER size, and send each chunk individually. Then when all the chunks are set, send a message that says the whole file has been sent so that the server can save it.
Here is my code for the solution described above:
server.py:
import socket
import threading
HEADER = 2048
PORT = 5050
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'
server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn, addr):
print(f'[NEW CONNECTION] {addr} connected.')
connected = True
current_file = None
while connected:
data = conn.recv(HEADER).decode(FORMAT)
if data == DISCONNECT_MESSAGE:
connected = False
elif data == FILE_FINISHED_SENDING:
current_file.close()
current_file = None
conn.send(b'file received.')
else:
data = data.split(SEPARATOR)
if len(data) == 2 and data[1] == '':
# The name of the file was sent, more will follow.
current_file = open(data[0], 'w')
conn.send(b'filename recieved')
else:
# File data was send, so write it to the current file
current_file.write(data[1])
print('chunk of file recv''d')
conn.send(b'chunk received')
conn.close()
print(f'[DISCONNECT] {addr} disconnected')
def start():
server.listen()
print(f'[LISTENING] Server is listening on {SERVER}')
while True:
conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn,addr))
thread.start()
print(f'[ACTIVE CONNECTIONS] {threading.activeCount()-1}')
print("[STARTING] server is starting...")
start()
client.py:
import socket
from pathlib import Path
HEADER = 2048
PORT = 5050
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = '!DISCONNECT'
SEPARATOR = '<SEPERATE>'
FILE_FINISHED_SENDING = '<!FILE_SENT!>'
SERVER = socket.gethostbyname(socket.gethostname())
ADDR = (SERVER, PORT)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)
def chunker(string: str, size: int):
return (string[pos:pos + size] for pos in range(0, len(string), size))
def send(msg):
message = msg.encode(FORMAT)
client.send(message)
print(client.recv(HEADER).decode(FORMAT))
def send_file(filepath: str):
with open(filepath, 'r', encoding=FORMAT) as f:
data = f.read()
first_bits = f'{Path(filepath).name}{SEPARATOR}' # Easy way of getting just a file's name from its path
send(first_bits) # Send the filename to the server
for chunk in chunker(data, HEADER-48): # Leave a safe buffer
# Send each chunk of the file
send(f"{SEPARATOR}{chunk}")
# Tell the server that's all for this file.
# Now it can close the file object.
send(FILE_FINISHED_SENDING)
send_file("/path/to/file.html")
send(DISCONNECT_MESSAGE)
Tips:
make sure that your special messages like SEPARATOR, FILE_FINISHED_SENDING and DISCONNECT_MESSAGE are NOT going to appear in the files you are sending. Otherwise things might get wonky.
You may want to read files as raw bytes when you send them through the socket, instead of reading as strings, encoding, decoding, etc. This way you could send binary files such as .mp3, for example.

making TCP web Server with python

I have created a multi threaded python web server and it is working fine for my client server.However i want it to take results work for web server as well. Please help me understand how we do that
the server
import socket
import threading
import os
import sys
#to convert bytes into string
def bytestoString(stringToRead):
stringToRead = bytes.decode(stringToRead)
type(stringToRead)
return(stringToRead)
#to conver string into bytes
def stringToBytes(bytesToSend1):
bytesToSend1= str.encode(bytesToSend1)
type (bytes)
return(bytesToSend1)
#to retreive a file
def retrFile(name,sock):
fileName=sock.recv(1024)
fileName_string = bytestoString(fileName)
print(fileName_string)
stringLength=len(fileName)
fileName_string = bytestoString(fileName[4:(stringLength-8)])
if os.path.isfile(fileName_string):
fileSize=str(os.path.getsize(fileName_string))
fileSize_Bytes = stringToBytes(fileSize)
exists_Bytes=stringToBytes('HTTP/1.1 200 OK')
sock.send(exists_Bytes)
ContentLength_Bytes=stringToBytes('Content-Length:')
sock.send(ContentLength_Bytes+fileSize_Bytes)
userResponse=sock.recv(1024)
userResponse=bytestoString(userResponse)
if userResponse[:2]=='Y':
print ('Ready to send the file................')
with open(fileName_string,'rb') as fileRead:
data= fileRead.read(1024)
sock.send(data)
while data!="":
data=fileRead.read(1024)
sock.send(data)
fileRead.close()
print('xyz')
elif userResponse[:2]=='N':
print('User Terminated file download, Thanks for connecting')
else :
httpResponse=stringToBytes('HTTP/1.1 404 not Found')
sock.send(httpResponse)
sock.close()
def Main(serverPort):
#creating a server socket type TCP
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
localHost=socket.gethostname()
#binding the server to the client
serverSocket.bind((localHost,serverPort))
serverSocket.listen(5)
print ('***Server is ready to recieve***')
while True:
connectionSocket, addr = serverSocket.accept()
print ('got connection from:<', addr,'>')
t=threading.Thread(target=retrFile,args=('retrThread',connectionSocket))
t.start()
connectionSocket.send('thank you for connecting')
connectionSocket.close()
if __name__ == '__main__':
#getting server hostname and port number from the user
serverPort=int(sys.argv[1])
Main(serverPort)
The Client
import sys
import socket
#to convert bytes into string
def bytestoString(stringToRead):
stringToRead = bytes.decode(stringToRead)
type(stringToRead)
return(stringToRead)
#to conver string into bytes
def stringToBytes(bytesToSend):
bytesToSend= str.encode(bytesToSend)
type (bytes)
return(bytesToSend)
#def header():
def Main(serverName,serverPort,fileName_bytes,fileName):
print('***Initialising the socket***')
serverAddress=(serverName,serverPort)
#create the TCP/IP socket using user inout for server and port
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientSocket.connect(serverAddress)
httpFileRequest=('GET/'+fileName+' HTTP/1.0')
httpFileRequest=stringToBytes(httpFileRequest)
#Reading the Hostname,socket family, socket type, protocol for the server
connectionDetails= socket.getaddrinfo(serverName,serverPort)
#printing the values to the screen
print (connectionDetails)
#print('Family:',families)
#print('type:',types)
#print('protocol:',protocols)
#Sending the request to recieve the file from the server
if fileName_bytes!="q":
clientSocket.send(httpFileRequest)
serverResponse=clientSocket.recv(1024)
serverResponse_string = bytestoString(serverResponse)
print(serverResponse_string)
fileLength=clientSocket.recv(1024)
fileLength_string = bytestoString(fileLength)
print(fileLength_string)
#checking the server response and downloading the file
if serverResponse_string[13:15]=='OK':
fileSize= float(int(fileLength_string[15:17]))
print(('Size of file you wish to download is:'),fileSize)
clientChoice=input('Please Enter Your Choice (Y/N)?:')
if clientChoice=='Y':
clientChoice=stringToBytes(clientChoice)
clientSocket.send(clientChoice)
#recieveing the file from the server
file = open('new_'+fileName,'wb')
print ('file opened.............')
data=clientSocket.recv(1024)
totalRecv=float(len(data))
file.write(data)
while totalRecv<fileSize:
data=clientSocket.recv(1024)
totalRecv+=len(data)
file.write(data)
percentComplte=int(int((totalRecv)/int(fileSize)*100))
print (('we have completed'),percentComplte,('%'))
file.close()
print ('Download complete')
elif clientChoice=='N':
clientChoice=stringToBytes(clientChoice)
clientSocket.send(clientChoice)
print ('File download terminated')
else :
print ('Please enter the correct choice')
else:
print (serverResponse_string)
clientSocket.close()
#Getting Hostname,socket family, socket type, protocol for the server
def get_constants(prefix):
"""Create a dictionary mapping socket module constants to their names."""
return dict( (getattr(socket, n), n)
for n in dir(socket)
if n.startswith(prefix)
)
#calling main function
if __name__ == '__main__':
#getting server hostname and port number from the user
serverName=sys.argv[1]
serverPort=int(sys.argv[2])
fileName= str(sys.argv[3])
fileName_bytes = str.encode(fileName)
type(bytes)
Main(serverName,serverPort,fileName_bytes,fileName)

python socket programming-How to implement multi-threading using the code below?

This is a simple server. When you open the browser type into the address of the server, and it will response a status code and the content of the requested html.
#import socket module
from socket import *
serverSocket = socket(AF_INET, SOCK_STREAM)
#Prepare a sever socket
serverSocket.bind((socket.gethostname(), 4501))#Fill in start
serverSocket.listen(5)#Fill in end
while True:
#Establish the connection
print 'Ready to serve...'
connectionSocket, addr = serverSocket.accept()#Accepts a TCP client connection, waiting until connection arrives
print 'Required connection', addr
try:
message = connectionSocket.recv(32)#Fill in start #Fill in end
filename = message.split()[1]
f = open(filename[1:])
outputdata = f.read()#Fill in start #Fill in end
#Send one HTTP header line into socket
connectionSocket.send('HTTP/1.0 200 OK\r\n\r\n')#Fill in start
#Send the content of the requested file to the client
for i in range(0, len(outputdata)):
connectionSocket.send(outputdata[i])
connectionSocket.close()
except IOError:
#Send response message for file not found
connectionSocket.send('404 Not Found')#Fill in start
#Fill in end
#Close client socket
connectionSocket.close()#Fill in start
serverSocket.close()#Fill in end

There are many ways to do this. Here's a way to use a pool of worker threads:
import Queue
import threading
num_workers = 10
work_q = Queue.Queue()
def worker(work_q):
while True:
connection_socket = work_q.get()
if connection_socket is None:
break
try:
message = connectionSocket.recv()
filename = message.split()[1]
f = open(filename[1:])
outputdata = f.read()
connectionSocket.send('HTTP/1.0 200 OK\r\n\r\n')
connectionSocket.send(outputdata)
except IOError:
connectionSocket.send('404 Not Found')
finally:
connectionSocket.close()
workers = []
for i in range(num_workers):
t = threading.Thread(target=worker, args=(work_q,))
t.start()
workers.append(t)
while True:
#Establish the connection
print 'Ready to serve...'
connectionSocket, addr = serverSocket.accept()
print 'Required connection', addr
work_q.put(connectionSocket)