Develop Reference

why does my python socket just stop working after it is done chunking and receiving data

basically this code is a server for a project that I am doing the client sends over 600 photos which then I want to be stored in a folder called incoming. first error was me running into my pickle data being Truncated while being sent to my server to try rectify this. I added a chunking system where it splits the data up into smaller chunks to make sure there is a smaller change of data being lost. doing this in theory should work. but when it has finished chunking all my data the server just quits out of the 'handle_client' function but gives me no errors and in turn leaves my client hanging and unresponsive until I shut the server down
import pickle
import socket
import os
import threading
HEADER = 64
PORT = 5050
SERVER = 'Left this out for obvious reasons'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "..DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn, addr):
print(f"[NEW CONNECTION] {addr} connected.")
connected = True
while connected:
data_length = conn.recv(HEADER).decode(FORMAT)
if data_length:
data_length = int(data_length)
recv_data = conn.recv(data_length)
bytes_received = 0
while bytes_received < data_length:
chunk = conn.recv(4096)
recv_data += chunk
bytes_received += len(chunk)
data = pickle.loads(recv_data)
if data == DISCONNECT_MESSAGE:
connected = False
try:
for files in os.listdir('incoming'):
file_path = os.path.join('incoming', files)
os.remove(file_path)
except:
print('[SERVER_ERROR] No files inside ./Incoming/ folder')
for file in data:
with open(os.path.join('incoming', file['name']), "wb") as f:
f.write(file['data'])
print('[SAVING] Server has saved file')
with open('siamesemodel.h5','rb') as f:
data = f.read()
pickledData = pickle.dumps(data)
data_length = len(pickledData)
send_length = str(data_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
conn.send(send_length)
conn.send(pickledData)
def start():
server.listen()
print(f"[LISTENING] Server is listening on {SERVER}")
while True:
conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn, addr))
thread.start()
print(f"[ACTIVE CONNECTIONS] {threading.active_count() - 1}")
print("[STARTING] server is starting...")
start()
tried debugging to no avail

Python UDP File Transfer

I'm creating a udp file transfer using python with a client (requests file from server), a server (receives request from client and passes it to the relevant worker) and worker1/worker2 (recieve request from server, if file exists, sends to server to pass back to client) and its all ran in docker containers with an ubuntu image*
Currently, when I type the name of the file I want in the client container, nothing works. I think the file name isn't actually being sent to the server but I can't figure out why at all. I was wondering if anyone can spot my mistake?
Server:
from fileinput import filename
import socket
import time
localIP = "127.0.0.1"
localPort = 50001
bufSize = 1024
msg = "Server is connecting..."
print(msg)
workerAddressPorts = [('127.0.0.1', 50002), ('127.0.0.1', 50003)]
# Create datagram sockets and bind to address and ip
UDPServerSocket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
UDPServerSocket.bind((localIP, localPort))
print("Server is waiting for packet...")
while True:
bytesAddressPair = UDPServerSocket.recvfrom(bufSize)
clientBuf = bytesAddressPair[0]
clientAddr = bytesAddressPair[1]
msgFrom = 'Message from Client:{}'.format(clientBuf.decode('utf-8'))
print(msgFrom)
for workerAddressPort in workerAddressPorts:
UDPServerSocket.sendto(clientBuf, workerAddressPort) # send file name to worker
print(f'Clients request has been sent to Worker {workerAddressPort[0]}')
workerBuf = UDPServerSocket.recvfrom(bufSize)[0] # saving data from worker
print(f'Request recieved.')
UDPServerSocket.sendto(workerBuf, clientAddr) # sending data to client
print('Packet from Worker has been sent to Client')
while not workerBuf: # split file to prevent buffer overflow
workerBuf = UDPServerSocket.recvfrom(bufSize)[0]
print(f'Packet received.')
UDPServerSocket.sendto(workerBuf, clientAddr)
print('Packet from worker has been sent to Client')
print('File has been sent to Client.')
Client:
import sys
msgFrom = input('Name of file: ')
print(msgFrom)
bytesToSend = str.encode(msgFrom, 'utf-8')
serverAddressPort = ("127.0.0.1", 50001)
bufSize = 1024
# create UDP Client Socket and send to server
UDPClientSocket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
UDPClientSocket.sendto(bytesToSend, serverAddressPort)
while (True):
try:
serverBuf = UDPClientSocket.recvfrom(bufSize)[0]
if len(serverBuf) > 0:
print('Packet incoming...')
msg = serverBuf.decode('utf-8')
if msg == 'NO_FILE':
print('No such file exists.')
break
elif msg == 'END_OF_FILE':
print('Empty file.')
break
else:
f = open(msgFrom, 'wb')
f.write(serverBuf)
f.close()
except Exception as e:
print(e)
Worker:
import socket
import time
from os.path import exists
bufSize = 1024
msg = "Worker is connecting..."
print(msg)
serverAddressPort = ("127.0.0.1", 50001)
# Create a datagram socket and bind to address and ip
UDPWorkerSocket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
UDPWorkerSocket.bind(('127.0.0.1', 50002))
print("Worker is connected and waiting...")
while True:
time.sleep(1)
bytesAddressPair = UDPWorkerSocket.recvfrom(bufSize)
msg = bytesAddressPair[0]
addr = bytesAddressPair[1]
print('Packet Incoming...')
file_name = msg.decode('utf-8')
script_dir = os.path.dirname("C:/Users/Vic/Documents/3rd Year TCD/CSU33031 Computer Networks/Assignments/Assignment1.3/Files to send")
abs_file_path = os.path.join(script_dir, file_name)
if exists(file_name):
f = open(file_name, 'rb')
data = f.read(bufSize)
while data:
if UDPWorkerSocket.sendto(data, serverAddressPort):
data = f.read(bufSize)
time.sleep(1)
print('Sent to Server.')
else:
UDPWorkerSocket.sendto(str.encode('NO_FILE'), serverAddressPort)
print(f'{file_name} was not found. Moving to next available worker..')
Hoping that the for statement in the server, iterates the two worker files(only included one) so if it can't find the file in one worker it moves to the other
*the client is connected to a different network as the two workers. The server is connected to both networks
Thanks in advance!!

Network server only sending data to client once instead of looping

I'm sorry if this is a really dumb question, I'm sure someone could probably find the answer in a minute, I've just recently been getting into Python sockets.
I want my server to continually send a stream of data to my client, but for some reason, after receiving the first piece of data my client just does not receive/print out any more data.
My simplified server.py:
while True:
#do some stuff with dfwebsites here
senddata = True
#time.sleep(1)
#Starting the sending data part
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((socket.gethostname(),1236))
s.listen(5) #queue of five
while senddata==True:
clientsocket, address = s.accept()
print(f"Connection from {address} has been established!")
d = pd.DataFrame(dfwebsites)
msg = pickle.dumps(d)
#header to specify length
#msg = "Welcome to the server!"
msg = bytes(f'{len(msg):<{HEADERSIZE}}','utf-8')+msg
clientsocket.send(msg) #type of bytes is utf-8
#clientsocket.close()
senddata = False
My client.py:
import socket
import pickle
import time
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((socket.gethostname(), 1236))
while True:
full_msg = b''
new_msg = True
while True:
msg = s.recv(1024)
if new_msg:
print("new msg len:",msg[:HEADERSIZE])
msglen = int(msg[:HEADERSIZE])
new_msg = False
print(f"full message length: {msglen}")
full_msg += msg
print(len(full_msg))
if len(full_msg)-HEADERSIZE == msglen:
print("full msg recvd")
print(full_msg[HEADERSIZE:])
print(pickle.loads(full_msg[HEADERSIZE:]))
new_msg = True
full_msg = b""
Why can it not receive more than one peice of data?
Thank you so much for your help! I would really love even a comment telling me how to improve my qeustion!

To send more than one message to each client, you need a loop after the accept() has happened.
#!/usr/bin/env python
import socket
import pickle
import pandas as pd
HEADERSIZE = 10
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((socket.gethostname(),1236))
s.listen(5) # only one client at a time, but let up to five wait in line
while True:
clientsocket, address = s.accept()
print(f"Connection from {address} has been established!")
while senddata:
# FIXME: refresh dfwebsites every time through this loop?
d = pd.DataFrame(dfwebsites)
msg = pickle.dumps(d)
msg = bytes(f'{len(msg):<{HEADERSIZE}}','utf-8')+msg
try:
clientsocket.send(msg) #type of bytes is utf-8
except socket.error as exc:
print(f"Ending connection from client {address} due to {exc}")
# FIXME: Do the below only when you want to disconnect a client
#senddata = False
clientsocket.close()

Threaded socket doesn't transfer all data using sendall

I am currently running into an issue where the following code only reads the first 1028 bytes from a socket, and then hangs waiting for more even though the rest has been sent. I thought it could be the code causing the problem, but it only happens when I am receiving data from the client to the server (which is multi-threaded). When I run the code in reverse (server sending data to client) it runs without an issue.
This is what is currently failing:
The clients code:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
serverSocket.sendall(str(len(data)).encode())
# Sends the data to the server
data = file.read(1024)
while data:
data = file.read(1024)
serverSocket.sendall(data)
Along with the server code:
# Gets the total length of the file
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
while len(totalData) < dataLen:
# Reads in and appends the data to the variable
data = clientsocket.recv(1024).decode()
totalData += data
What else have I tried?
The original code for the client was:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
serverSocket.sendall(str(len(data)).encode())
# Sends the data to the server
serverSocket.sendall(data)
Which I changed because I thought there may be an issue with the recv not getting all of the packets from the client, but the client believes that everything was sent.
Minimum Reproducible Code:
SERVER.PY
from socket import socket, AF_INET, SOCK_STREAM, error
from threading import Thread, Lock
from _thread import interrupt_main
import logging
from sys import stdout
from os import makedirs
from pathlib import Path
class Server:
def __init__(self, host, port):
logging.basicConfig(stream=stdout, level=logging.INFO, format='%(asctime)s - %(message)s')
self.host = host
self.port = port
self.quitVal = False
self.threads = []
self.lock = Lock()
self.sock = None
self.open()
def open(self):
sock = socket(AF_INET, SOCK_STREAM)
sock.bind((self.host, self.port))
sock.listen(5)
self.sock = sock
def listen(self):
try:
while self.quitVal != True:
(clientsocket, address) = self.sock.accept()
# This creates a threaded connection as this is the choke for the connection
thread = Thread(target=self.connection, args=(clientsocket, address), daemon=True)
self.threads.append(thread)
thread.start()
except OSError:
quit()
def connection(self, clientsocket, address):
while self.quitVal != True:
# We are going to allow multiple file transfers at one time on a connection
requestThreads = []
# The client innitially sends a command "s<command> [<fileLocation>]"
data = clientsocket.recv(4096).decode("utf-8")
data = data.split()
data[0] = data[0].lower()
if data[0] == "get":
thread = Thread(target=self.get, args=(clientsocket, data[1]), daemon=True)
requestThreads.append(thread)
thread.start()
elif data[0] == "put":
thread = Thread(target=self.put, args=(clientsocket, data[1]), daemon=True)
requestThreads.append(thread)
thread.start()
clientsocket.close()
def get(self, clientsocket, fileLocation):
# Tries to get the file, and sends it all to the client
with self.lock:
try:
with open(fileLocation, "rb") as file:
data = file.read()
# Sends the length of the file to the client
clientsocket.sendall(str(len(data)).encode())
# Byte to separate the input from length and file
response = self.sock.recv(1).decode()
# Sends the data to the client
clientsocket.sendall(data)
except IOError as e:
print("\nFile doesn't exist")
def put(self, clientsocket, fileLocation):
# Gets the total length of the file
with self.lock:
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
# Gets the total length of the file
dataLen = int(clientsocket.recv(1024).decode())
totalData = ""
# Byte to separate the input from length and file
clientsocket.sendall("0".encode())
while len(totalData) < dataLen:
# Reads in and appends the data to the variable
data = clientsocket.recv(2048).decode()
totalData += data
logging.info(totalData)
server = Server("127.0.0.1", 10002)
server.listen()
CLIENT.PY
from socket import socket, AF_INET, SOCK_STREAM, error
from threading import Thread, Lock
from _thread import interrupt_main
import logging
from sys import stdout
from os import makedirs
from pathlib import Path
class Client:
def __init__(self, host, port):
self.host = host
self.port = port
self.quitVal = False
self.sock = self.open()
self.threads = []
self.lock = Lock()
def open(self):
try:
sock = socket(AF_INET, SOCK_STREAM)
sock.connect((self.host, self.port))
except error as err:
return None
return sock
def get(self, commands):
# Sends the request for the file to the server using the main Sock
with self.lock:
command = "GET " + commands[0]
self.sock.send(command.encode())
dataLen = int(self.sock.recv(1024).decode())
totalData = ""
self.sock.sendall("0".encode())
while len(totalData) < dataLen:
data = self.sock.recv(2048).decode()
totalData += data
def put(self, commands):
with self.lock:
# Sends the put call command to the server
command = "PUT " + commands[1]
self.sock.sendall(command.encode())
try:
with open(commands[1], "rb") as file:
data = file.read()
# Sends the length of the file to the server
self.sock.sendall(str(len(data)).encode())
# Byte to separate the input from length and file
response = self.sock.recv(1).decode()
# Sends the data to the server
self.sock.sendall(data)
except IOError as e:
logging.info("\nFile doesn't exist")
client = Client("127.0.0.1", 10002)
print("foo")
client.get(["foo.txt", "bar.txt"])
client.put(["foo.txt", "bar1.txt"])

making TCP web Server with python

I have created a multi threaded python web server and it is working fine for my client server.However i want it to take results work for web server as well. Please help me understand how we do that
the server
import socket
import threading
import os
import sys
#to convert bytes into string
def bytestoString(stringToRead):
stringToRead = bytes.decode(stringToRead)
type(stringToRead)
return(stringToRead)
#to conver string into bytes
def stringToBytes(bytesToSend1):
bytesToSend1= str.encode(bytesToSend1)
type (bytes)
return(bytesToSend1)
#to retreive a file
def retrFile(name,sock):
fileName=sock.recv(1024)
fileName_string = bytestoString(fileName)
print(fileName_string)
stringLength=len(fileName)
fileName_string = bytestoString(fileName[4:(stringLength-8)])
if os.path.isfile(fileName_string):
fileSize=str(os.path.getsize(fileName_string))
fileSize_Bytes = stringToBytes(fileSize)
exists_Bytes=stringToBytes('HTTP/1.1 200 OK')
sock.send(exists_Bytes)
ContentLength_Bytes=stringToBytes('Content-Length:')
sock.send(ContentLength_Bytes+fileSize_Bytes)
userResponse=sock.recv(1024)
userResponse=bytestoString(userResponse)
if userResponse[:2]=='Y':
print ('Ready to send the file................')
with open(fileName_string,'rb') as fileRead:
data= fileRead.read(1024)
sock.send(data)
while data!="":
data=fileRead.read(1024)
sock.send(data)
fileRead.close()
print('xyz')
elif userResponse[:2]=='N':
print('User Terminated file download, Thanks for connecting')
else :
httpResponse=stringToBytes('HTTP/1.1 404 not Found')
sock.send(httpResponse)
sock.close()
def Main(serverPort):
#creating a server socket type TCP
serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
localHost=socket.gethostname()
#binding the server to the client
serverSocket.bind((localHost,serverPort))
serverSocket.listen(5)
print ('***Server is ready to recieve***')
while True:
connectionSocket, addr = serverSocket.accept()
print ('got connection from:<', addr,'>')
t=threading.Thread(target=retrFile,args=('retrThread',connectionSocket))
t.start()
connectionSocket.send('thank you for connecting')
connectionSocket.close()
if __name__ == '__main__':
#getting server hostname and port number from the user
serverPort=int(sys.argv[1])
Main(serverPort)
The Client
import sys
import socket
#to convert bytes into string
def bytestoString(stringToRead):
stringToRead = bytes.decode(stringToRead)
type(stringToRead)
return(stringToRead)
#to conver string into bytes
def stringToBytes(bytesToSend):
bytesToSend= str.encode(bytesToSend)
type (bytes)
return(bytesToSend)
#def header():
def Main(serverName,serverPort,fileName_bytes,fileName):
print('***Initialising the socket***')
serverAddress=(serverName,serverPort)
#create the TCP/IP socket using user inout for server and port
clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientSocket.connect(serverAddress)
httpFileRequest=('GET/'+fileName+' HTTP/1.0')
httpFileRequest=stringToBytes(httpFileRequest)
#Reading the Hostname,socket family, socket type, protocol for the server
connectionDetails= socket.getaddrinfo(serverName,serverPort)
#printing the values to the screen
print (connectionDetails)
#print('Family:',families)
#print('type:',types)
#print('protocol:',protocols)
#Sending the request to recieve the file from the server
if fileName_bytes!="q":
clientSocket.send(httpFileRequest)
serverResponse=clientSocket.recv(1024)
serverResponse_string = bytestoString(serverResponse)
print(serverResponse_string)
fileLength=clientSocket.recv(1024)
fileLength_string = bytestoString(fileLength)
print(fileLength_string)
#checking the server response and downloading the file
if serverResponse_string[13:15]=='OK':
fileSize= float(int(fileLength_string[15:17]))
print(('Size of file you wish to download is:'),fileSize)
clientChoice=input('Please Enter Your Choice (Y/N)?:')
if clientChoice=='Y':
clientChoice=stringToBytes(clientChoice)
clientSocket.send(clientChoice)
#recieveing the file from the server
file = open('new_'+fileName,'wb')
print ('file opened.............')
data=clientSocket.recv(1024)
totalRecv=float(len(data))
file.write(data)
while totalRecv<fileSize:
data=clientSocket.recv(1024)
totalRecv+=len(data)
file.write(data)
percentComplte=int(int((totalRecv)/int(fileSize)*100))
print (('we have completed'),percentComplte,('%'))
file.close()
print ('Download complete')
elif clientChoice=='N':
clientChoice=stringToBytes(clientChoice)
clientSocket.send(clientChoice)
print ('File download terminated')
else :
print ('Please enter the correct choice')
else:
print (serverResponse_string)
clientSocket.close()
#Getting Hostname,socket family, socket type, protocol for the server
def get_constants(prefix):
"""Create a dictionary mapping socket module constants to their names."""
return dict( (getattr(socket, n), n)
for n in dir(socket)
if n.startswith(prefix)
)
#calling main function
if __name__ == '__main__':
#getting server hostname and port number from the user
serverName=sys.argv[1]
serverPort=int(sys.argv[2])
fileName= str(sys.argv[3])
fileName_bytes = str.encode(fileName)
type(bytes)
Main(serverName,serverPort,fileName_bytes,fileName)