Develop Reference

why does my python socket just stop working after it is done chunking and receiving data

basically this code is a server for a project that I am doing the client sends over 600 photos which then I want to be stored in a folder called incoming. first error was me running into my pickle data being Truncated while being sent to my server to try rectify this. I added a chunking system where it splits the data up into smaller chunks to make sure there is a smaller change of data being lost. doing this in theory should work. but when it has finished chunking all my data the server just quits out of the 'handle_client' function but gives me no errors and in turn leaves my client hanging and unresponsive until I shut the server down
import pickle
import socket
import os
import threading
HEADER = 64
PORT = 5050
SERVER = 'Left this out for obvious reasons'
ADDR = (SERVER, PORT)
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "..DISCONNECT"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(ADDR)
def handle_client(conn, addr):
print(f"[NEW CONNECTION] {addr} connected.")
connected = True
while connected:
data_length = conn.recv(HEADER).decode(FORMAT)
if data_length:
data_length = int(data_length)
recv_data = conn.recv(data_length)
bytes_received = 0
while bytes_received < data_length:
chunk = conn.recv(4096)
recv_data += chunk
bytes_received += len(chunk)
data = pickle.loads(recv_data)
if data == DISCONNECT_MESSAGE:
connected = False
try:
for files in os.listdir('incoming'):
file_path = os.path.join('incoming', files)
os.remove(file_path)
except:
print('[SERVER_ERROR] No files inside ./Incoming/ folder')
for file in data:
with open(os.path.join('incoming', file['name']), "wb") as f:
f.write(file['data'])
print('[SAVING] Server has saved file')
with open('siamesemodel.h5','rb') as f:
data = f.read()
pickledData = pickle.dumps(data)
data_length = len(pickledData)
send_length = str(data_length).encode(FORMAT)
send_length += b' ' * (HEADER - len(send_length))
conn.send(send_length)
conn.send(pickledData)
def start():
server.listen()
print(f"[LISTENING] Server is listening on {SERVER}")
while True:
conn, addr = server.accept()
thread = threading.Thread(target=handle_client, args=(conn, addr))
thread.start()
print(f"[ACTIVE CONNECTIONS] {threading.active_count() - 1}")
print("[STARTING] server is starting...")
start()
tried debugging to no avail

proxy server with python

i have an university project with python which i have to write a proxy server that waits for a request from a client and then connects the client to the server. i searched the net and found an already-written code from this site:
https://www.geeksforgeeks.org/creating-a-proxy-webserver-in-python-set-1/
so i used it and made some changes in the code and add the public server and port to it
but when i run it i get this error:
line 33, in main
request = conn.recv(4096)
NameError: name 'conn' is not defined
so i'm not very familiar with sockets and python so if there are obvious mistakes in the code i would be happy if u guys could explain them in a very basic way so my amateur butt would understand it lol
this is the code:
import signal
import socket
import threading
class Proxy:
def __init__(self):
# creating a tcp socket
self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reuse the socket
self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ip = 'localhost'
self.port = 8080
self.serverSocket.bind((self.ip, self.port))
self.serverSocket.listen(10)
self.__clients = {}
def shutdown(self):
# shutdown on cntrl c
signal.signal(signal.SIGINT, self.shutdown)
def multirequest (self):
while True:
# establish the connection
(clientSocket, client_address) = self.serverSocket.accept()
d = threading.Thread(name=self._getclientname(client_address),
target=self.proxy_thread,
args=(clientSocket, client_address))
d.setDaemon(True)
d.start()
def main(self, conn):
# get the request from browser
request = conn.recv(4096)
# parse the first line
first_line = request.split('\n')[0]
# get url
url = first_line.split(' ')[1]
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
webserver = ""
port = -1
port_pos = temp.find(":")
# find end of web server
webserver_pos = temp.find("/")
if webserver_pos == -1:
webserver_pos = len(temp)
if port_pos == -1 or webserver_pos < port_pos:
# default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int((temp[(port_pos + 1):])[:webserver_pos - port_pos - 1])
webserver = temp[:port_pos]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
s.connect((webserver, port))
s.sendall(request)
while 1:
# receive data from web server
data = s.recv(4096)
if len(data) > 0:
conn.send(data) # send to browser/client
else:
break
p = Proxy()
p.main()

In python, the common pattern for executables is like this:
def main(): # Or whatever name you want to use
"""Your code here"""
# If you are importing the code, the condition will evaluate to false.
if __name__ == "__main__":
main()
Have in mind that you can use whatever function name you want.

You need to put your code inside of a class like this. Any variables you plan to use through your code, define inside of the __init__ method, this could be for example your self.serverSocket, ip, port number, etc.
The init method is deigned to run once, once you create an instance of the class, it usually stores variables. The main method or any other methods you define, would be where you put the rest of your code.
import signal
import socket
import threading
class proxy():
def __init__(self):
# creating a tcp socket
self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# reuse the socket
self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.your_ip = "127.0.0.1" # loop back address
self.your_port = 80 # use a port thats open like port 80
def shutdown(self):
# add your signal shutdown code here
pass
def main(self):
# shutdown on cntrl c
signal.signal(signal.SIGINT, self.shutdown)
self.serverSocket.bind((self.your_ip, self.your_port))
self.serverSocket.listen(10)
self.__clients = {}
while True:
# establish the connection
(clientSocket, client_address) = self.serverSocket.accept()
d = threading.Thread(name=self._getClientName(client_address),
target=self.proxy_thread,
args=(clientSocket, client_address))
d.setDaemon(True)
d.start()
# get the request from browser
request = conn.recv(4096)
# parse the first line
first_line = request.split('\n')[0]
# get url
url = first_line.split(' ')[1]
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
port_pos = temp.find(":")
# find end of web server
webserver_pos = temp.find("/")
if webserver_pos == -1:
webserver_pos = len(temp)
webserver = ""
port = -1
if port_pos == -1 or webserver_pos < port_pos:
# default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int((temp[(port_pos + 1):])[:webserver_pos - port_pos - 1])
webserver = temp[:port_pos]
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
s.connect((webserver, port))
s.sendall(request)
while 1:
# receive data from web server
data = s.recv(4096)
if len(data) > 0:
conn.send(data) # send to browser/client
else:
break
p = proxy()
p.main()

I successful make a local python proxy via socket module: https://github.com/wayne931121/Python_Proxy_Server/blob/main/Proxy.py
It can run http and https request.
Code:
#!/usr/bin/env python
# coding: utf-8
# In[ ]:
# 參考資料
# https://docs.python.org/3/library/socket.html
# https://stackoverflow.com/questions/24218058/python-https-proxy-tunnelling
# https://stackoverflow.com/questions/68008233/proxy-server-with-python/73851150#73851150
import sys
#import ssl
import time
import signal
import socket
#import certifi
import threading
with open("log.txt", "w") as f:
f.write("")
def signal_handler(sig, frame):
print('Proxy is Stopped.')
sys.exit(0)
def write(*content, prt=False):
if prt :
if len(content[0])<100:
print(*content)
else:
print("This message is too long not print in cmd but will store at log.txt.")
if type(content[0])==bytes:
content = b" ".join(content)
else:
content = bytes(" ".join(content), encoding="utf-8")
with open("log.txt", "ab") as f:
f.write(content+b"\n")
class Proxy:
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # creating a tcp socket
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # reuse the socket
self.ip = "127.0.0.1"
self.port = 8080
# self.host = socket.gethostbyname(socket.gethostname())+":%s"%self.port
self.sock.bind((self.ip, self.port))
self.sock.listen(10)
print("Proxy Server Is Start, See log.txt get log.")
print("Press Ctrl+C to Stop.")
start_multirequest = threading.Thread(target=self.multirequest)
start_multirequest.setDaemon(True)
start_multirequest.start()
while 1:
time.sleep(0.01)
signal.signal(signal.SIGINT, signal_handler)
def multirequest(self):
while True:
(clientSocket, client_address) = self.sock.accept() # establish the connection
client_process = threading.Thread(target=self.main, args=(clientSocket, client_address))
client_process.setDaemon(True)
client_process.start()
def main(self, client_conn, client_addr): # client_conn is the connection by proxy client like browser.
origin_request = client_conn.recv(4096)
request = origin_request.decode(encoding="utf-8") # get the request from browser
first_line = request.split("\r\n")[0] # parse the first line
url = first_line.split(" ")[1] # get url
http_pos = url.find("://")
if http_pos == -1:
temp = url
else:
temp = url[(http_pos + 3):]
webserver = ""
port = -1
port_pos = temp.find(":")
webserver_pos = temp.find("/") # find end of web server
if webserver_pos == -1:
webserver_pos = len(temp)
if port_pos == -1 or webserver_pos < port_pos: # default port
port = 80
webserver = temp[:webserver_pos]
else: # specific port
port = int(temp[(port_pos + 1):])
webserver = temp[:port_pos]
write("Connected by", str(client_addr))
write("ClientSocket", str(client_conn))
write("Browser Request:")
write(request)
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_conn.settimeout(1000)
try:
server_conn.connect((webserver, port)) # "server_conn" connect to public web server, like www.google.com:443.
except: # socket.gaierror: [Errno 11001] getaddrinfo failed
client_conn.close()
server_conn.close()
return
if port==443:
client_conn.send(b"HTTP/1.1 200 Connection established\r\n\r\n")
client_conn.setblocking(0)
server_conn.setblocking(0)
write("Connection established")
# now = time.time()
client_browser_message = b""
website_server_message = b""
error = ""
while 1:
# if time.time()-now>1: # SET TIMEOUT
# server_conn.close()
# client_conn.close()
# break
try:
reply = client_conn.recv(1024)
if not reply: break
server_conn.send(reply)
client_browser_message += reply
except Exception as e:
pass
# error += str(e)
try:
reply = server_conn.recv(1024)
if not reply: break
client_conn.send(reply)
website_server_message += reply
except Exception as e:
pass
# error += str(e)
write("Client Browser Message:")
write(client_browser_message+b"\n")
write("Website Server Message:")
write(website_server_message+b"\n")
# write("Error:")
# write(error+"\n")
server_conn.shutdown(socket.SHUT_RDWR)
server_conn.close()
client_conn.close()
return
server_conn.sendall(origin_request)
write("Website Host Result:")
while 1:
# receive data from web server
data = server_conn.recv(4096)
try:
write(data.decode(encoding="utf-8"))
except:
write(data)
if len(data) > 0:
client_conn.send(data) # send to browser/client
else:
break
server_conn.shutdown(socket.SHUT_RDWR)
server_conn.close()
client_conn.close()
Proxy()

How to make a socket server run forever without closed itself (Python)

Whenever the client disconnect, the server will close itself. How can i make the server to run forever ?
What i'm doing
The server let one client to retrieve files with no issues. But the problem is when the client close the program, the server will also closed itself and wouldn't let another client to establish the connection . I had read a few articles about using while loops to make the session alive. Does anyone know how can I do that ?
Server.py
import socket, os, subprocess, shutil, pickle, struct
# Create a Socket ( connect two computers)
def create_socket():
try:
global host
global port
global s
host = ""
port = 9999
s = socket.socket()
except socket.error as msg:
create_socket()
# Binding the socket and listening for connections
def bind_socket():
try:
global host
global port
global s
s.bind((host, port))
s.listen(5)
except socket.error as msg:
bind_socket()
# send file list
def flist(conn):
try:
arr = pickle.dumps(os.listdir())
conn.send(arr)
except:
conn.send(('Error').encode("utf-8"))
# accept file from server
def fdown(filename, conn):
try:
data = conn.recv(1024).decode("utf-8")
if data[:6] == 'EXISTS':
filesize = data[6:]
conn.send("OK".encode("utf-8"))
f = open(filename, 'wb')
data = (conn.recv(1024))
totalRecv = len(data)
f.write(data)
while int(totalRecv) < int(filesize):
data = conn.recv(1024)
totalRecv += len(data)
f.write(data)
f.close()
except:
conn.send(('Error').encode("utf-8"))
# send file
def fup(filename, conn):
if os.path.isfile(filename):
conn.send(str.encode("EXISTS " + str(os.path.getsize(filename))))
filesize = int(os.path.getsize(filename))
userResponse = conn.recv(1024).decode("utf-8")
if userResponse[:2] == 'OK':
with open(filename, 'rb') as f:
bytesToSend = f.read(1024)
conn.send(bytesToSend)
totalSend = len(bytesToSend)
while int(totalSend) < int(filesize):
bytesToSend = f.read(1024)
totalSend += len(bytesToSend)
conn.send(bytesToSend)
else:
conn.send("ERROR".encode("utf-8"))
# main
def main(s):
while True:
data = (s.recv(1024)).decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
# Establish connection with a client (socket must be listening)
def socket_accept():
conn, address = s.accept()
main(conn)
conn.close()
create_socket()
bind_socket()
socket_accept()

You should put accept in the loop, and you may need use a thread to handle read
sample code:
def handle_read(s):
while True:
data = s.recv(1024)
if not data:
#When the client closed, recv will return an empty data
s.close()
break
data = data.decode("utf-8").split('~')
if data[0] == 'fdown':
fup(data[1], s)
elif data[0] == 'fup':
fdown(data[1], s)
elif data[0] == 'flist':
flist(s)
else:
s.send(".".encode('utf-8'))
def socket_accept():
while True:
conn, address = s.accept()
t = threading.Thread(target = handle_read, args=(conn, ))
t.start()

What is the proper way of sending a large amount of data over sockets in Python?

Recently I wrote some code (client and server) to send an image - the client simply uploads the image to the server, just using the socket module: Sending image over sockets (ONLY) in Python, image can not be open.
However, the image sending part is now what I am concerned with. This is the original image I'm using:
In my server code (which receives the images), I have these lines:
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
data = sock.recv(40960000)
if not data:
myfile.close()
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
But I don't think this is the best way of doing it. I think I should instead implement the server such that it receives the data in chunks:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = 0
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()
But when I do this, the server prints:
got size 54674
4096
8192
12288
16384
20480
24576
28672
32768
36864
40960
45056
49152
50578
And then seems to hang, and the client hangs too. However, at the server's side I can see only a piece of the image the client wanted to send:
It seems like there are some bytes missing. What is the proper way of sending a huge amount of data (images, other type of file) using ONLY sockets?

I'm assuming that you have a particular reason for doing this with naked sockets, such as self-edification, which means that I won't answer by saying "You accidentally forgot to just use HTTP and Twisted", which perhaps you've heard before :-P. But really you should look at higher-level libraries at some point as they're a lot easier!
Define a protocol
If all you want is to send an image, then it can be simple:
Client -> server: 8 bytes: big endian, length of image.
Client -> server: length bytes: all image data.
(Client <- server: 1 byte, value 0: indicate transmission received - optional step you may not care if you're using TCP and just assume that it's reliable.)
Code it
server.py
import os
from socket import *
from struct import unpack
class ServerProtocol:
def __init__(self):
self.socket = None
self.output_dir = '.'
self.file_num = 1
def listen(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.bind((server_ip, server_port))
self.socket.listen(1)
def handle_images(self):
try:
while True:
(connection, addr) = self.socket.accept()
try:
bs = connection.recv(8)
(length,) = unpack('>Q', bs)
data = b''
while len(data) < length:
# doing it in batches is generally better than trying
# to do it all in one go, so I believe.
to_read = length - len(data)
data += connection.recv(
4096 if to_read > 4096 else to_read)
# send our 0 ack
assert len(b'\00') == 1
connection.sendall(b'\00')
finally:
connection.shutdown(SHUT_WR)
connection.close()
with open(os.path.join(
self.output_dir, '%06d.jpg' % self.file_num), 'w'
) as fp:
fp.write(data)
self.file_num += 1
finally:
self.close()
def close(self):
self.socket.close()
self.socket = None
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
sp = ServerProtocol()
sp.listen('127.0.0.1', 55555)
sp.handle_images()
client.py
from socket import *
from struct import pack
class ClientProtocol:
def __init__(self):
self.socket = None
def connect(self, server_ip, server_port):
self.socket = socket(AF_INET, SOCK_STREAM)
self.socket.connect((server_ip, server_port))
def close(self):
self.socket.shutdown(SHUT_WR)
self.socket.close()
self.socket = None
def send_image(self, image_data):
# use struct to make sure we have a consistent endianness on the length
length = pack('>Q', len(image_data))
# sendall to make sure it blocks if there's back-pressure on the socket
self.socket.sendall(length)
self.socket.sendall(image_data)
ack = self.socket.recv(1)
# could handle a bad ack here, but we'll assume it's fine.
if __name__ == '__main__':
cp = ClientProtocol()
image_data = None
with open('IMG_0077.jpg', 'r') as fp:
image_data = fp.read()
assert(len(image_data))
cp.connect('127.0.0.1', 55555)
cp.send_image(image_data)
cp.close()

A simple way is to send data size as the first 4 bytes of your data and then read complete data in one shot. Use the below functions on both client and server-side to send and receive data.
def send_data(conn, data):
serialized_data = pickle.dumps(data)
conn.sendall(struct.pack('>I', len(serialized_data)))
conn.sendall(serialized_data)
def receive_data(conn):
data_size = struct.unpack('>I', conn.recv(4))[0]
received_payload = b""
reamining_payload_size = data_size
while reamining_payload_size != 0:
received_payload += conn.recv(reamining_payload_size)
reamining_payload_size = data_size - len(received_payload)
data = pickle.loads(received_payload)
return data
you could find sample program at https://github.com/vijendra1125/Python-Socket-Programming.git

The problem is you are not incrementing amount_received for the first chunk of the data received.
Fix below:
#!/usr/bin/env python
import random
import socket, select
from time import gmtime, strftime
from random import randint
imgcounter = 1
basename = "image%s.png"
HOST = '127.0.0.1'
PORT = 2905
connected_clients_sockets = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
connected_clients_sockets.append(server_socket)
while True:
read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])
for sock in read_sockets:
if sock == server_socket:
sockfd, client_address = server_socket.accept()
connected_clients_sockets.append(sockfd)
else:
try:
data = sock.recv(4096)
txt = str(data)
if data:
if data.startswith('SIZE'):
tmp = txt.split()
size = int(tmp[1])
print 'got size %s' % size
sock.sendall("GOT SIZE")
elif data.startswith('BYE'):
sock.shutdown()
else :
myfile = open(basename % imgcounter, 'wb')
myfile.write(data)
amount_received = len(data) # The fix!
while amount_received < size:
data = sock.recv(4096)
amount_received += len(data)
print amount_received
if not data:
break
myfile.write(data)
myfile.close()
sock.sendall("GOT IMAGE")
sock.shutdown()
except:
sock.close()
connected_clients_sockets.remove(sock)
continue
imgcounter += 1
server_socket.close()

How to keep socket alive?

Should i keep alive sockets on my server if I interact with known clients for a long time and all of them stays online and max amount of them is 10?
If yes what the best way to do this? Something like:
class mysocket:
'''demonstration class only
- coded for clarity, not efficiency
'''
def __init__(self, sock=None):
if sock is None:
self.sock = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
else:
self.sock = sock
def connect(self, host, port):
self.sock.connect((host, port))
def mysend(self, msg):
totalsent = 0
while totalsent < MSGLEN:
sent = self.sock.send(msg[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent = totalsent + sent
def myreceive(self):
chunks = []
bytes_recd = 0
while bytes_recd < MSGLEN:
chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048))
if chunk == '':
raise RuntimeError("socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
return ''.join(chunks)
will be good?
Also what happens if I specify more size for recv() then socket have?